Feat: protobuf support (#831)
* feat: protobuf support * Wuxiaworld (#31) * wuxiaworld * cleanup * make .proto smaller * add status * fix date and partial freechapter * fetchrpoto * no more JSON.parse * fix typeerrors * fetchProto as type * final draft * cleanup summary --------- Co-authored-by: K1ngfish3r <trulyihu@gmail.com>
This commit is contained in:
nyagami
@@ -1,3 +1,5 @@
|
||||
import { parse as parseProto } from 'protobufjs';
|
||||
|
||||
type FetchInit = {
|
||||
headers?: Record<string, string | undefined> | Headers;
|
||||
method?: string;
|
||||
@@ -106,3 +108,51 @@ export const fetchText = async function (
|
||||
return "";
|
||||
}
|
||||
};
|
||||
|
||||
interface ProtoRequestInit {
|
||||
// merged .proto file
|
||||
proto: string;
|
||||
requestType: string;
|
||||
requestData?: any;
|
||||
responseType: string;
|
||||
}
|
||||
|
||||
const BYTE_MARK = BigInt((1 << 8) - 1);
|
||||
|
||||
export const fetchProto = async function<ReturnType> (
|
||||
protoInit: ProtoRequestInit,
|
||||
url: string,
|
||||
init?: FetchInit,
|
||||
) {
|
||||
const protoRoot = parseProto(protoInit.proto).root;
|
||||
const RequestMessge = protoRoot.lookupType(protoInit.requestType);
|
||||
if (RequestMessge.verify(protoInit.requestData)){
|
||||
throw new Error('Invalid Proto');
|
||||
}
|
||||
// encode request data
|
||||
const encodedrequest = RequestMessge.encode(protoInit.requestData).finish();
|
||||
const requestLength = BigInt(encodedrequest.length);
|
||||
const headers = new Uint8Array(
|
||||
Array(5).fill(0).map((v, idx) => {
|
||||
if(idx === 0) return 0;
|
||||
return Number((requestLength >> BigInt(8 * (5 - idx - 1))) & BYTE_MARK);
|
||||
})
|
||||
);
|
||||
init = await makeInit(init);
|
||||
const bodyArray = new Uint8Array(headers.length + encodedrequest.length)
|
||||
bodyArray.set(headers, 0)
|
||||
bodyArray.set(encodedrequest, headers.length)
|
||||
return fetch(url, {
|
||||
method: 'POST',
|
||||
...init,
|
||||
body: bodyArray
|
||||
} as RequestInit
|
||||
).then(r => r.arrayBuffer())
|
||||
.then(arr => {
|
||||
// decode response data
|
||||
const payload = new Uint8Array(arr);
|
||||
const length = Number(BigInt(payload[1] << 24) | BigInt(payload[2] << 16) | BigInt(payload[3] << 8) | BigInt(payload[4]));
|
||||
const ResponseMessage = protoRoot.lookupType(protoInit.responseType);
|
||||
return ResponseMessage.decode(payload.slice(5, 5 + length));
|
||||
}) as ReturnType
|
||||
}
|
||||
|
||||
Generated
+161
@@ -17,6 +17,7 @@
|
||||
"express": "^4.18.2",
|
||||
"less": "^4.2.0",
|
||||
"module-alias": "^2.2.3",
|
||||
"protobufjs": "^7.2.6",
|
||||
"qs": "^6.11.2",
|
||||
"sanitize-html": "^2.12.1",
|
||||
"terser": "^5.28.1",
|
||||
@@ -128,6 +129,60 @@
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/@protobufjs/aspromise": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
|
||||
"integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="
|
||||
},
|
||||
"node_modules/@protobufjs/base64": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
|
||||
"integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="
|
||||
},
|
||||
"node_modules/@protobufjs/codegen": {
|
||||
"version": "2.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz",
|
||||
"integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg=="
|
||||
},
|
||||
"node_modules/@protobufjs/eventemitter": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz",
|
||||
"integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q=="
|
||||
},
|
||||
"node_modules/@protobufjs/fetch": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz",
|
||||
"integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==",
|
||||
"dependencies": {
|
||||
"@protobufjs/aspromise": "^1.1.1",
|
||||
"@protobufjs/inquire": "^1.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@protobufjs/float": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
|
||||
"integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="
|
||||
},
|
||||
"node_modules/@protobufjs/inquire": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz",
|
||||
"integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q=="
|
||||
},
|
||||
"node_modules/@protobufjs/path": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
|
||||
"integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="
|
||||
},
|
||||
"node_modules/@protobufjs/pool": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
|
||||
"integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="
|
||||
},
|
||||
"node_modules/@protobufjs/utf8": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
|
||||
"integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="
|
||||
},
|
||||
"node_modules/@tsconfig/node10": {
|
||||
"version": "1.0.9",
|
||||
"resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz",
|
||||
@@ -2212,6 +2267,11 @@
|
||||
"resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-3.0.4.tgz",
|
||||
"integrity": "sha512-eDn9kqrAmVUC1wmZvlQ6Uhde44n+tXpqPrN8olQJbttgh0oKclk+SF54P47VEGE9CEiMeRwAP8BaM7UHvBkz2A=="
|
||||
},
|
||||
"node_modules/long": {
|
||||
"version": "5.2.3",
|
||||
"resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz",
|
||||
"integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q=="
|
||||
},
|
||||
"node_modules/lru-cache": {
|
||||
"version": "4.1.5",
|
||||
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz",
|
||||
@@ -2827,6 +2887,29 @@
|
||||
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
|
||||
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
|
||||
},
|
||||
"node_modules/protobufjs": {
|
||||
"version": "7.2.6",
|
||||
"resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.6.tgz",
|
||||
"integrity": "sha512-dgJaEDDL6x8ASUZ1YqWciTRrdOuYNzoOf27oHNfdyvKqHr5i0FV7FSLU+aIeFjyFgVxrpTOtQUi0BLLBymZaBw==",
|
||||
"hasInstallScript": true,
|
||||
"dependencies": {
|
||||
"@protobufjs/aspromise": "^1.1.2",
|
||||
"@protobufjs/base64": "^1.1.2",
|
||||
"@protobufjs/codegen": "^2.0.4",
|
||||
"@protobufjs/eventemitter": "^1.1.0",
|
||||
"@protobufjs/fetch": "^1.1.0",
|
||||
"@protobufjs/float": "^1.0.2",
|
||||
"@protobufjs/inquire": "^1.1.0",
|
||||
"@protobufjs/path": "^1.1.2",
|
||||
"@protobufjs/pool": "^1.1.0",
|
||||
"@protobufjs/utf8": "^1.1.0",
|
||||
"@types/node": ">=13.7.0",
|
||||
"long": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/proxy-addr": {
|
||||
"version": "2.0.7",
|
||||
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
|
||||
@@ -3952,6 +4035,60 @@
|
||||
"fastq": "^1.6.0"
|
||||
}
|
||||
},
|
||||
"@protobufjs/aspromise": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
|
||||
"integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="
|
||||
},
|
||||
"@protobufjs/base64": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
|
||||
"integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="
|
||||
},
|
||||
"@protobufjs/codegen": {
|
||||
"version": "2.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz",
|
||||
"integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg=="
|
||||
},
|
||||
"@protobufjs/eventemitter": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz",
|
||||
"integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q=="
|
||||
},
|
||||
"@protobufjs/fetch": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz",
|
||||
"integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==",
|
||||
"requires": {
|
||||
"@protobufjs/aspromise": "^1.1.1",
|
||||
"@protobufjs/inquire": "^1.1.0"
|
||||
}
|
||||
},
|
||||
"@protobufjs/float": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
|
||||
"integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="
|
||||
},
|
||||
"@protobufjs/inquire": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz",
|
||||
"integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q=="
|
||||
},
|
||||
"@protobufjs/path": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
|
||||
"integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="
|
||||
},
|
||||
"@protobufjs/pool": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
|
||||
"integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="
|
||||
},
|
||||
"@protobufjs/utf8": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
|
||||
"integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="
|
||||
},
|
||||
"@tsconfig/node10": {
|
||||
"version": "1.0.9",
|
||||
"resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz",
|
||||
@@ -5636,6 +5773,11 @@
|
||||
"resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-3.0.4.tgz",
|
||||
"integrity": "sha512-eDn9kqrAmVUC1wmZvlQ6Uhde44n+tXpqPrN8olQJbttgh0oKclk+SF54P47VEGE9CEiMeRwAP8BaM7UHvBkz2A=="
|
||||
},
|
||||
"long": {
|
||||
"version": "5.2.3",
|
||||
"resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz",
|
||||
"integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q=="
|
||||
},
|
||||
"lru-cache": {
|
||||
"version": "4.1.5",
|
||||
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz",
|
||||
@@ -6101,6 +6243,25 @@
|
||||
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
|
||||
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
|
||||
},
|
||||
"protobufjs": {
|
||||
"version": "7.2.6",
|
||||
"resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.6.tgz",
|
||||
"integrity": "sha512-dgJaEDDL6x8ASUZ1YqWciTRrdOuYNzoOf27oHNfdyvKqHr5i0FV7FSLU+aIeFjyFgVxrpTOtQUi0BLLBymZaBw==",
|
||||
"requires": {
|
||||
"@protobufjs/aspromise": "^1.1.2",
|
||||
"@protobufjs/base64": "^1.1.2",
|
||||
"@protobufjs/codegen": "^2.0.4",
|
||||
"@protobufjs/eventemitter": "^1.1.0",
|
||||
"@protobufjs/fetch": "^1.1.0",
|
||||
"@protobufjs/float": "^1.0.2",
|
||||
"@protobufjs/inquire": "^1.1.0",
|
||||
"@protobufjs/path": "^1.1.2",
|
||||
"@protobufjs/pool": "^1.1.0",
|
||||
"@protobufjs/utf8": "^1.1.0",
|
||||
"@types/node": ">=13.7.0",
|
||||
"long": "^5.0.0"
|
||||
}
|
||||
},
|
||||
"proxy-addr": {
|
||||
"version": "2.0.7",
|
||||
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
"express": "^4.18.2",
|
||||
"less": "^4.2.0",
|
||||
"module-alias": "^2.2.3",
|
||||
"protobufjs": "^7.2.6",
|
||||
"qs": "^6.11.2",
|
||||
"sanitize-html": "^2.12.1",
|
||||
"terser": "^5.28.1",
|
||||
|
||||
@@ -1,145 +0,0 @@
|
||||
import { load as parseHTML} from "cheerio";
|
||||
import { fetchApi, fetchFile } from "@libs/fetch";
|
||||
import { Plugin } from "@typings/plugin";
|
||||
import { Filters } from "@libs/filterInputs";
|
||||
|
||||
interface NovelEntry {
|
||||
name: string;
|
||||
coverUrl: string;
|
||||
slug: string;
|
||||
}
|
||||
|
||||
class WuxiaWorld implements Plugin.PluginBase {
|
||||
id = "wuxiaworld";
|
||||
name = "Wuxia World";
|
||||
icon = "src/en/wuxiaworld/icon.png";
|
||||
site = "https://www.wuxiaworld.com/";
|
||||
filters?: Filters | undefined;
|
||||
version = "0.5.0";
|
||||
baseUrl = this.site;
|
||||
async popularNovels(pageNo: number, options: Plugin.PopularNovelsOptions<Filters>): Promise<Plugin.NovelItem[]> {
|
||||
const link = `${this.baseUrl}api/novels`;
|
||||
|
||||
const result = await fetch(link);
|
||||
const data = await result.json();
|
||||
|
||||
let novels: Plugin.NovelItem[] = [];
|
||||
|
||||
data.items.map((novel: NovelEntry) => {
|
||||
let name = novel.name;
|
||||
let cover = novel.coverUrl;
|
||||
let url = this.baseUrl + "novel/" + novel.slug + "/";
|
||||
|
||||
novels.push({
|
||||
name,
|
||||
cover,
|
||||
url,
|
||||
});
|
||||
});
|
||||
|
||||
return novels;
|
||||
}
|
||||
async parseNovelAndChapters(novelUrl: string): Promise<Plugin.SourceNovel> {
|
||||
const url = novelUrl;
|
||||
const result = await fetchApi(url);
|
||||
const body = await result.text();
|
||||
|
||||
const loadedCheerio = parseHTML(body);
|
||||
|
||||
const novel: Plugin.SourceNovel = {
|
||||
url,
|
||||
chapters: [],
|
||||
};
|
||||
|
||||
novel.name = loadedCheerio("h1.line-clamp-2").text();
|
||||
|
||||
novel.cover = loadedCheerio("img.absolute").attr("src");
|
||||
|
||||
novel.summary = loadedCheerio(
|
||||
"div.flex-col:nth-child(4) > div > div > span > span"
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
novel.author = loadedCheerio("div.MuiGrid-container > div > div > div")
|
||||
.filter(function () {
|
||||
return loadedCheerio(this).text().trim() === "Author:";
|
||||
})
|
||||
.next()
|
||||
.text();
|
||||
|
||||
let genres: string[] = [];
|
||||
|
||||
loadedCheerio("a.MuiLink-underlineNone").each(function () {
|
||||
genres.push(loadedCheerio(this).find("div > div").text());
|
||||
});
|
||||
|
||||
novel.genres = genres.join(",");
|
||||
|
||||
novel.status = loadedCheerio("div.font-set-b10")
|
||||
.text();
|
||||
|
||||
let chapter: Plugin.ChapterItem[] = [];
|
||||
|
||||
loadedCheerio("div.border-b.border-gray-line-base").each((idx, ele) => {
|
||||
const name = loadedCheerio(ele)
|
||||
.find("a > div > div > div > span")
|
||||
.text();
|
||||
const releaseTime = loadedCheerio(ele)
|
||||
.find("a > div > div > div > div > span")
|
||||
.text();
|
||||
let url = loadedCheerio(ele).find("a").attr("href")?.slice(1);
|
||||
url = `${this.baseUrl}${url}`;
|
||||
|
||||
chapter.push({ name, releaseTime, url });
|
||||
});
|
||||
|
||||
novel.chapters = chapter;
|
||||
|
||||
return novel;
|
||||
}
|
||||
async parseChapter(chapterUrl: string): Promise<string> {
|
||||
const result = await fetchApi(chapterUrl);
|
||||
const body = await result.text();
|
||||
|
||||
const loadedCheerio = parseHTML(body);
|
||||
|
||||
loadedCheerio(".chapter-nav").remove();
|
||||
|
||||
loadedCheerio("#chapter-content > script").remove();
|
||||
|
||||
let chapterText = loadedCheerio("#chapter-content").html() || '';
|
||||
|
||||
return chapterText;
|
||||
}
|
||||
async searchNovels(searchTerm: string, pageNo: number): Promise<Plugin.NovelItem[]> {
|
||||
const searchUrl = "https://www.wuxiaworld.com/api/novels/search?query=";
|
||||
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const result = await fetchApi(url);
|
||||
const data = await result.json();
|
||||
|
||||
const novels: Plugin.NovelItem[] = [];
|
||||
|
||||
data.items.map((novel: NovelEntry) => {
|
||||
let name = novel.name;
|
||||
let cover = novel.coverUrl;
|
||||
let url = this.baseUrl + "novel/" + novel.slug + "/";
|
||||
|
||||
novels.push({
|
||||
name,
|
||||
url,
|
||||
cover,
|
||||
});
|
||||
});
|
||||
|
||||
return novels;
|
||||
}
|
||||
fetchImage(url: string): Promise<string | undefined> {
|
||||
return fetchFile(url);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export default new WuxiaWorld();
|
||||
@@ -0,0 +1,466 @@
|
||||
import { load as parseHTML } from "cheerio";
|
||||
import { fetchApi, fetchFile, fetchProto } from "@libs/fetch";
|
||||
import { Plugin } from "@typings/plugin";
|
||||
import { Filters } from "@libs/filterInputs";
|
||||
import { NovelStatus } from "@libs/novelStatus";
|
||||
|
||||
interface StringValue {
|
||||
value: string;
|
||||
}
|
||||
|
||||
interface BoolValue {
|
||||
value: boolean;
|
||||
}
|
||||
|
||||
interface DecimalValue {
|
||||
units: bigint;
|
||||
nanos: number;
|
||||
}
|
||||
|
||||
interface Timestamp {
|
||||
seconds: number;
|
||||
nanos: number;
|
||||
}
|
||||
|
||||
interface NovelEntry {
|
||||
name: string;
|
||||
coverUrl: string;
|
||||
slug: string;
|
||||
}
|
||||
|
||||
interface RelatedChapterUserInfo {
|
||||
isChapterUnlocked?: BoolValue;
|
||||
}
|
||||
|
||||
interface ChapterItem {
|
||||
entityId: number;
|
||||
name: string;
|
||||
slug: string;
|
||||
number?: DecimalValue;
|
||||
content?: StringValue;
|
||||
relatedUserInfo?: RelatedChapterUserInfo;
|
||||
offset: number;
|
||||
publishedAt?: Timestamp;
|
||||
}
|
||||
|
||||
interface GetChapterResponse {
|
||||
item?: ChapterItem;
|
||||
}
|
||||
|
||||
interface ChapterGroupCounts {
|
||||
total: number;
|
||||
advance: number;
|
||||
normal: number;
|
||||
}
|
||||
|
||||
interface ChapterGroupItem {
|
||||
id: number;
|
||||
title: string;
|
||||
order: number;
|
||||
fromChapterNumber?: DecimalValue;
|
||||
toChapterNumber?: DecimalValue;
|
||||
chapterList: ChapterItem[];
|
||||
counts?: ChapterGroupCounts;
|
||||
}
|
||||
|
||||
interface GetChapterListResponse {
|
||||
items: ChapterGroupItem[];
|
||||
}
|
||||
|
||||
enum NovelItem_Status {
|
||||
Finished = 0,
|
||||
Active = 1,
|
||||
Hiatus = 2,
|
||||
All = -1
|
||||
}
|
||||
|
||||
interface NovelKarmaInfo {
|
||||
isActive: boolean;
|
||||
isFree: boolean;
|
||||
maxFreeChapter?: DecimalValue;
|
||||
canUnlockWithVip: boolean;
|
||||
}
|
||||
|
||||
interface NovelItem {
|
||||
id: number;
|
||||
name: string;
|
||||
slug: string;
|
||||
status: NovelItem_Status;
|
||||
visible: boolean;
|
||||
description?: StringValue;
|
||||
synopsis?: StringValue;
|
||||
coverUrl?: StringValue;
|
||||
translatorName?: StringValue;
|
||||
authorName?: StringValue;
|
||||
karmaInfo?: NovelKarmaInfo;
|
||||
genres: string[];
|
||||
}
|
||||
|
||||
interface GetNovelResponse {
|
||||
item?: NovelItem;
|
||||
}
|
||||
|
||||
class WuxiaWorld implements Plugin.PluginBase {
|
||||
id = "wuxiaworld";
|
||||
name = "Wuxia World";
|
||||
icon = "src/en/wuxiaworld/icon.png";
|
||||
site = "https://www.wuxiaworld.com/";
|
||||
filters?: Filters | undefined;
|
||||
version = "0.5.0";
|
||||
|
||||
parseNovels(data: {items: NovelEntry[]}){
|
||||
const novels: Plugin.NovelItem[] = [];
|
||||
|
||||
data.items.map((novel: NovelEntry) => {
|
||||
const name = novel.name;
|
||||
const cover = novel.coverUrl;
|
||||
const path = `novel/${novel.slug}/`;
|
||||
|
||||
novels.push({
|
||||
name,
|
||||
cover,
|
||||
path,
|
||||
});
|
||||
});
|
||||
|
||||
return novels;
|
||||
}
|
||||
|
||||
async popularNovels(pageNo: number, options: Plugin.PopularNovelsOptions<Filters>): Promise<Plugin.NovelItem[]> {
|
||||
const link = `${this.site}api/novels`;
|
||||
|
||||
const result = await fetchApi(link);
|
||||
const data = await result.json();
|
||||
|
||||
return this.parseNovels(data)
|
||||
}
|
||||
|
||||
async parseNovel(novelPath: string): Promise<Plugin.SourceNovel> {
|
||||
const data = await fetchProto<GetNovelResponse>(
|
||||
{
|
||||
proto: this.proto,
|
||||
requestType: 'GetNovelRequest',
|
||||
responseType: 'GetNovelResponse',
|
||||
requestData: {slug: novelPath.split('/')[1]},
|
||||
},
|
||||
'https://api2.wuxiaworld.com/wuxiaworld.api.v2.Novels/GetNovel',
|
||||
{
|
||||
headers: {
|
||||
"Content-Type": "application/grpc-web+proto",
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
const novel: Plugin.SourceNovel = {
|
||||
path: novelPath,
|
||||
name: data.item?.name || "Untitled",
|
||||
cover: data.item?.coverUrl?.value,
|
||||
summary: parseHTML(data.item?.description?.value + '\n\n' + data.item?.synopsis?.value).text(),
|
||||
author: data.item?.authorName?.value,
|
||||
genres: data.item?.genres.join(','),
|
||||
chapters: [],
|
||||
};
|
||||
|
||||
const status = data.item?.status;
|
||||
switch (status){
|
||||
case NovelItem_Status.Active:
|
||||
novel.status = NovelStatus.Ongoing;
|
||||
break;
|
||||
case NovelItem_Status.Hiatus:
|
||||
novel.status = NovelStatus.OnHiatus;
|
||||
break;
|
||||
case NovelItem_Status.All:
|
||||
novel.status = NovelStatus.Unknown;
|
||||
break;
|
||||
default: novel.status = NovelStatus.Completed;
|
||||
}
|
||||
|
||||
// novel.summary = loadedCheerio('.relative > .absolute:first')
|
||||
// .children('span')
|
||||
// .map((i,el) => loadedCheerio(el).text().trim())
|
||||
// .toArray()
|
||||
// .join('\n');
|
||||
|
||||
// novel.genres = loadedCheerio("a.MuiLink-underlineNone")
|
||||
// .map((i,el) => loadedCheerio(el).text())
|
||||
// .toArray()
|
||||
// .join(',');
|
||||
|
||||
// novel.status = loadedCheerio("div.font-set-b10").text();
|
||||
|
||||
const list = await fetchProto<GetChapterListResponse>(
|
||||
{
|
||||
proto: this.proto,
|
||||
requestType: 'GetChapterListRequest',
|
||||
responseType: 'GetChapterListResponse',
|
||||
requestData: {novelId: data.item?.id},
|
||||
},
|
||||
'https://api2.wuxiaworld.com/wuxiaworld.api.v2.Chapters/GetChapterList',
|
||||
{
|
||||
headers: {
|
||||
"Content-Type": "application/grpc-web+proto",
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
const freeChapter = Number(data.item?.karmaInfo?.maxFreeChapter?.units) +
|
||||
(data.item?.karmaInfo?.maxFreeChapter?.nanos || 0) / 1000000000 || 50;
|
||||
|
||||
const chapter: Plugin.ChapterItem[] = list.items.flatMap((ChapterGroupItem: ChapterGroupItem) =>
|
||||
ChapterGroupItem.chapterList.map((chapterItem: ChapterItem) => ({
|
||||
page: ChapterGroupItem.title,
|
||||
name: chapterItem.name + (
|
||||
(chapterItem.relatedUserInfo?.isChapterUnlocked?.value === false) ||
|
||||
(!chapterItem.relatedUserInfo &&
|
||||
(Number(chapterItem.number?.units) +
|
||||
(chapterItem.number?.nanos || 0) / 1000000000) > freeChapter)
|
||||
? ' 🔒'
|
||||
: ''
|
||||
),
|
||||
path: novelPath + chapterItem.slug,
|
||||
chapterNumber: chapterItem.offset,
|
||||
releaseTime: new Date(
|
||||
(chapterItem.publishedAt?.seconds || 0) * 1000 +
|
||||
(chapterItem.publishedAt?.nanos || 0) / 1000000
|
||||
).toISOString(),
|
||||
})
|
||||
));
|
||||
|
||||
novel.chapters = chapter;
|
||||
|
||||
return novel;
|
||||
}
|
||||
|
||||
async parseChapter(chapterPath: string): Promise<string> {
|
||||
const paths = chapterPath.split('/');
|
||||
const data = await fetchProto<GetChapterResponse>(
|
||||
{
|
||||
proto: this.proto,
|
||||
requestType: 'GetChapterRequest',
|
||||
responseType: 'GetChapterResponse',
|
||||
requestData: {
|
||||
chapterProperty: {
|
||||
slugs: {
|
||||
novelSlug: paths[1],
|
||||
chapterSlug: paths[2],
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
'https://api2.wuxiaworld.com/wuxiaworld.api.v2.Chapters/GetChapter',
|
||||
{
|
||||
headers: {
|
||||
"Content-Type": "application/grpc-web+proto",
|
||||
}
|
||||
}
|
||||
)
|
||||
// loadedCheerio(".chapter-nav").remove();
|
||||
// loadedCheerio("#chapter-content > script").remove();
|
||||
// const chapterText = loadedCheerio("#chapter-content").html() || '';
|
||||
|
||||
const chapterText = data.item?.content?.value || "";
|
||||
return chapterText
|
||||
}
|
||||
|
||||
async searchNovels(searchTerm: string, pageNo: number): Promise<Plugin.NovelItem[]> {
|
||||
const searchUrl = "https://www.wuxiaworld.com/api/novels/search?query=";
|
||||
|
||||
const url = searchUrl + searchTerm;
|
||||
|
||||
const result = await fetchApi(url);
|
||||
const data = await result.json();
|
||||
|
||||
return this.parseNovels(data);
|
||||
}
|
||||
|
||||
fetchImage(url: string): Promise<string | undefined> {
|
||||
return fetchFile(url);
|
||||
}
|
||||
|
||||
proto = `
|
||||
syntax = "proto3";
|
||||
option optimize_for = CODE_SIZE;
|
||||
package wuxiaworld.api.v2;
|
||||
|
||||
import public "google/protobuf/wrappers.proto";
|
||||
import public "google/protobuf/timestamp.proto";
|
||||
|
||||
message StringValue {
|
||||
// The string value.
|
||||
string value = 1;
|
||||
}
|
||||
|
||||
message BoolValue {
|
||||
// The bool value.
|
||||
bool value = 1;
|
||||
}
|
||||
|
||||
message Int32Value {
|
||||
// The int32 value.
|
||||
int32 value = 1;
|
||||
}
|
||||
|
||||
message DecimalValue {
|
||||
// Whole units part of the amount
|
||||
int64 units = 1;
|
||||
// Nano units of the amount (10^-9)
|
||||
// Must be the same sign as units
|
||||
sfixed32 nanos = 2;
|
||||
}
|
||||
|
||||
message Timestamp {
|
||||
int64 seconds = 1;
|
||||
int32 nanos = 2;
|
||||
}
|
||||
|
||||
message RelatedChapterUserInfo {
|
||||
optional BoolValue isChapterUnlocked = 1;
|
||||
optional BoolValue isNovelUnlocked = 2;
|
||||
optional BoolValue isChapterFavorite = 3;
|
||||
optional BoolValue isNovelOwned = 4;
|
||||
optional BoolValue isChapterOwned = 5;
|
||||
}
|
||||
|
||||
message ChapterNovelInfo {
|
||||
int32 id = 1;
|
||||
string name = 2;
|
||||
optional StringValue coverUrl = 3;
|
||||
string slug = 4;
|
||||
}
|
||||
|
||||
message ChapterParagraph {
|
||||
string id = 1;
|
||||
int32 chapterId = 2;
|
||||
int32 totalComments = 3;
|
||||
optional StringValue content = 4;
|
||||
}
|
||||
|
||||
message ChapterItem {
|
||||
int32 entityId = 1;
|
||||
string name = 2;
|
||||
string slug = 3;
|
||||
optional DecimalValue number = 4;
|
||||
optional StringValue content = 5;
|
||||
int32 novelId = 6;
|
||||
bool visible = 7;
|
||||
bool isTeaser = 8;
|
||||
optional Timestamp whenToPublish = 9;
|
||||
bool spoilerTitle = 10;
|
||||
bool allowComments = 11;
|
||||
optional ChapterNovelInfo novelInfo = 14;
|
||||
optional RelatedChapterUserInfo relatedUserInfo = 16;
|
||||
int32 offset = 17;
|
||||
optional Timestamp publishedAt = 18;
|
||||
optional StringValue translatorThoughts = 19;
|
||||
repeated ChapterParagraph paragraphs = 21;
|
||||
}
|
||||
|
||||
message ChapterGroupCounts {
|
||||
int32 total = 1;
|
||||
int32 advance = 2;
|
||||
int32 normal = 3;
|
||||
}
|
||||
|
||||
message ChapterGroupItem {
|
||||
int32 id = 1;
|
||||
string title = 2;
|
||||
int32 order = 3;
|
||||
optional DecimalValue fromChapterNumber = 4;
|
||||
optional DecimalValue toChapterNumber = 5;
|
||||
repeated ChapterItem chapterList = 6;
|
||||
optional ChapterGroupCounts counts = 7;
|
||||
}
|
||||
|
||||
message GetChapterListRequest {
|
||||
int32 novelId = 1;
|
||||
message BaseChapterInfo {
|
||||
oneof chapterInfo {
|
||||
int32 chapterId = 1;
|
||||
string slug = 2;
|
||||
int32 offset = 3;
|
||||
}
|
||||
}
|
||||
message FilterChapters {
|
||||
optional Int32Value chapterGroupId = 1;
|
||||
optional BoolValue isAdvanceChapter = 2;
|
||||
optional BaseChapterInfo baseChapter = 3;
|
||||
}
|
||||
optional FilterChapters filter = 2;
|
||||
optional Int32Value count = 3;
|
||||
}
|
||||
|
||||
message GetChapterListResponse {
|
||||
repeated ChapterGroupItem items = 1;
|
||||
optional ChapterNovelInfo novelInfo = 2;
|
||||
}
|
||||
|
||||
message GetChapterByProperty {
|
||||
message ByNovelAndChapterSlug {
|
||||
string novelSlug = 1;
|
||||
string chapterSlug = 2;
|
||||
}
|
||||
oneof byProperty {
|
||||
int32 chapterId = 1;
|
||||
ByNovelAndChapterSlug slugs = 2;
|
||||
}
|
||||
}
|
||||
|
||||
message GetChapterRequest {
|
||||
optional GetChapterByProperty chapterProperty = 1;
|
||||
}
|
||||
|
||||
message GetChapterResponse {
|
||||
optional ChapterItem item = 1;
|
||||
}
|
||||
|
||||
message NovelKarmaInfo {
|
||||
bool isActive = 1;
|
||||
bool isFree = 2;
|
||||
optional DecimalValue maxFreeChapter = 3;
|
||||
bool canUnlockWithVip = 4;
|
||||
}
|
||||
|
||||
message NovelItem {
|
||||
int32 id = 1;
|
||||
string name = 2;
|
||||
string slug = 3;
|
||||
enum Status {
|
||||
Finished = 0;
|
||||
Active = 1;
|
||||
Hiatus = 2;
|
||||
All = -1;
|
||||
}
|
||||
Status status = 4;
|
||||
bool visible = 7;
|
||||
optional StringValue description = 8;
|
||||
optional StringValue synopsis = 9;
|
||||
optional StringValue coverUrl = 10;
|
||||
optional StringValue translatorName = 11;
|
||||
optional StringValue authorName = 13;
|
||||
optional NovelKarmaInfo karmaInfo = 14;
|
||||
repeated string genres = 16;
|
||||
}
|
||||
|
||||
message GetNovelRequest {
|
||||
oneof selector {
|
||||
int32 id = 1;
|
||||
string slug = 2;
|
||||
}
|
||||
}
|
||||
|
||||
message GetNovelResponse {
|
||||
optional NovelItem item = 1;
|
||||
}
|
||||
|
||||
service Chapters {
|
||||
rpc GetChapterList(GetChapterListRequest) returns (GetChapterListResponse);
|
||||
rpc GetChapter(GetChapterRequest) returns (GetChapterResponse);
|
||||
}
|
||||
|
||||
service Novels {
|
||||
rpc GetNovel(GetNovelRequest) returns (GetNovelResponse);
|
||||
}
|
||||
`
|
||||
}
|
||||
|
||||
export default new WuxiaWorld();
|
||||
Reference in New Issue
Block a user