Divine Dao Library: update all data fetching (#1320)
* feat: update Divine Dao Library Add novel filters, support both popular and latest novels, get covers, and find the actual last published chapter. * feat: introduce caching to Divine Dao Library
This commit is contained in:
Martijn van der Ven
committed by
GitHub
GitHub
parent
87a9f3c2cb
commit
e1567d6615
@@ -1,125 +1,298 @@
|
|||||||
import { CheerioAPI, load as parseHTML } from 'cheerio';
|
|
||||||
import { fetchApi } from '@libs/fetch';
|
|
||||||
import { Plugin } from '@typings/plugin';
|
|
||||||
import { defaultCover } from '@libs/defaultCover';
|
import { defaultCover } from '@libs/defaultCover';
|
||||||
|
import { fetchApi } from '@libs/fetch';
|
||||||
|
import { Filters, FilterTypes, FilterValueWithType } from '@libs/filterInputs';
|
||||||
|
import { Plugin } from '@typings/plugin';
|
||||||
|
import { load as parseHTML } from 'cheerio';
|
||||||
|
|
||||||
class DDLPlugin implements Plugin.PluginBase {
|
class DDLPlugin implements Plugin.PluginBase {
|
||||||
id = 'DDL.com';
|
id = 'DDL.com';
|
||||||
name = 'Divine Dao Library';
|
name = 'Divine Dao Library';
|
||||||
site = 'https://www.divinedaolibrary.com/';
|
site = 'https://www.divinedaolibrary.com/';
|
||||||
version = '1.0.1';
|
version = '1.1.0';
|
||||||
icon = 'src/en/divinedaolibrary/icon.png';
|
icon = 'src/en/divinedaolibrary/icon.png';
|
||||||
|
|
||||||
parseNovels(loadedCheerio: CheerioAPI, searchTerm?: string) {
|
filters = {
|
||||||
let novels: Plugin.NovelItem[] = [];
|
category: {
|
||||||
|
type: FilterTypes.CheckboxGroup,
|
||||||
|
label: 'State',
|
||||||
|
value: ['Completed', 'Translating', 'Lost in Voting Poll', 'Dropped'],
|
||||||
|
options: [
|
||||||
|
{ label: 'Completed', value: 'Completed' },
|
||||||
|
{ label: 'Translating', value: 'Translating' },
|
||||||
|
{ label: 'Lost in Voting Poll', value: 'Lost in Voting Poll' },
|
||||||
|
{ label: 'Dropped', value: 'Dropped' },
|
||||||
|
{ label: 'Personally Written', value: 'Personally Written' },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
} satisfies Filters;
|
||||||
|
|
||||||
loadedCheerio('#main')
|
allNovelsCache: readonly (readonly [string, string, string])[] | undefined;
|
||||||
.find('li')
|
novelItemCache = new Map<string, Required<Plugin.NovelItem>>();
|
||||||
.each((i, el) => {
|
|
||||||
const novelName = loadedCheerio(el).find('a').text();
|
|
||||||
const novelCover = defaultCover;
|
|
||||||
const novelUrl = loadedCheerio(el).find('a').attr('href');
|
|
||||||
|
|
||||||
if (!novelUrl) return;
|
/**
|
||||||
|
* Safely extract the pathname from any URL on {@link site}. Check the root
|
||||||
const novel = {
|
* site as there are novels linking off-site (to Patreon).
|
||||||
name: novelName,
|
*
|
||||||
cover: novelCover,
|
* @private
|
||||||
path: novelUrl.replace(this.site, ''),
|
*/
|
||||||
};
|
getPath(url: string): string | undefined {
|
||||||
|
if (!url.startsWith(this.site)) {
|
||||||
novels.push(novel);
|
return undefined;
|
||||||
});
|
|
||||||
|
|
||||||
if (searchTerm) {
|
|
||||||
novels = novels.filter(novel =>
|
|
||||||
novel.name.toLowerCase().includes(searchTerm.toLowerCase()),
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
return novels;
|
const trimmed = url.substring(this.site.length).replace(/(^\/+|\/+$)/g, '');
|
||||||
|
if (trimmed.length === 0) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
return trimmed;
|
||||||
}
|
}
|
||||||
|
|
||||||
async popularNovels(): Promise<Plugin.NovelItem[]> {
|
/**
|
||||||
const link = this.site + 'novels';
|
* Map an array with an asynchronous function and only return the array items
|
||||||
|
* that successfully were fulfilled with values other than undefined.
|
||||||
const body = await fetchApi(link).then(res => res.text());
|
*
|
||||||
|
* @private
|
||||||
const loadedCheerio = parseHTML(body);
|
*/
|
||||||
return this.parseNovels(loadedCheerio);
|
async asyncMap<T, U>(
|
||||||
|
collection: readonly T[],
|
||||||
|
callbackfn: (value: T, index: number, array: readonly T[]) => Promise<U>,
|
||||||
|
): Promise<Exclude<U, undefined>[]> {
|
||||||
|
return (await Promise.allSettled(collection.map(callbackfn)))
|
||||||
|
.filter(
|
||||||
|
<U>(
|
||||||
|
p: PromiseSettledResult<U>,
|
||||||
|
): p is PromiseFulfilledResult<Exclude<U, undefined>> =>
|
||||||
|
p.status === 'fulfilled' && p.value !== undefined,
|
||||||
|
)
|
||||||
|
.map(({ value }) => value);
|
||||||
}
|
}
|
||||||
|
|
||||||
async parseNovel(novelPath: string): Promise<Plugin.SourceNovel> {
|
/**
|
||||||
const result = await fetchApi(this.site + novelPath);
|
* DDL links to future (unpublished) chapters from its novel pages. To be
|
||||||
const body = await result.text();
|
* able to report updates correctly, try to figure out which chapter was the
|
||||||
|
* actual latest one to be published.
|
||||||
|
*
|
||||||
|
* @private
|
||||||
|
* @returns the path value of the latest published chapter (or undefined)
|
||||||
|
*/
|
||||||
|
async findLatestChapter(novelPath: string): Promise<string | undefined> {
|
||||||
|
const link = `${this.site}wp-json/wp/v2/categories?slug=${novelPath}`;
|
||||||
|
const guessCategory = await fetchApi(link).then(res => res.json());
|
||||||
|
if (guessCategory.length !== 1) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
const categoryId = guessCategory[0].id;
|
||||||
|
const chapterLink = `${this.site}wp-json/wp/v2/posts?categories=${categoryId}&per_page=1`;
|
||||||
|
const lastChapter = await fetchApi(chapterLink).then(res => res.json());
|
||||||
|
if (lastChapter.length !== 1) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
return lastChapter[0].slug;
|
||||||
|
}
|
||||||
|
|
||||||
const loadedCheerio = parseHTML(body);
|
/**
|
||||||
|
* Based on the path, grab all the available information about a novel. Used
|
||||||
const novel: Plugin.SourceNovel = {
|
* to seed the {@link novelItemCache} as well as to fetch the actual single
|
||||||
path: novelPath,
|
* novel views with chapter list.
|
||||||
name: loadedCheerio('h1.entry-title').text().trim() || 'Untitled',
|
*
|
||||||
cover:
|
* @private
|
||||||
loadedCheerio('.entry-content').find('img').attr('data-ezsrc') ||
|
*/
|
||||||
defaultCover,
|
async grabNovel(
|
||||||
chapters: [],
|
path: string,
|
||||||
|
getChapters = false,
|
||||||
|
): Promise<(Plugin.SourceNovel & Required<Plugin.NovelItem>) | undefined> {
|
||||||
|
const link = `${this.site}wp-json/wp/v2/pages?slug=${path}`;
|
||||||
|
const data = await fetchApi(link).then(res => res.json());
|
||||||
|
if (data.length !== 1) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
const content = parseHTML(data[0].content.rendered);
|
||||||
|
const excerpt = parseHTML(data[0].excerpt.rendered);
|
||||||
|
const image = content('img').first();
|
||||||
|
let chapters: Plugin.ChapterItem[] = [];
|
||||||
|
if (getChapters) {
|
||||||
|
const linkedChapters = content('li > span > a')
|
||||||
|
.map((_, anchorEl) => {
|
||||||
|
const chapterPath = this.getPath(anchorEl.attribs['href']);
|
||||||
|
if (!chapterPath) return;
|
||||||
|
return {
|
||||||
|
name: content(anchorEl).text(),
|
||||||
|
path: chapterPath,
|
||||||
|
} satisfies Plugin.ChapterItem;
|
||||||
|
})
|
||||||
|
.toArray();
|
||||||
|
const lastChapterPath = await this.findLatestChapter(path);
|
||||||
|
if (lastChapterPath) {
|
||||||
|
chapters = linkedChapters.slice(
|
||||||
|
0,
|
||||||
|
1 +
|
||||||
|
linkedChapters.findIndex(
|
||||||
|
chapter => chapter.path === lastChapterPath,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
chapters = linkedChapters;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
name: data[0].title.rendered,
|
||||||
|
path,
|
||||||
|
cover: image.attr('data-lazy-src') ?? image.attr('src') ?? defaultCover,
|
||||||
|
author: content('h3')
|
||||||
|
.first()
|
||||||
|
.text()
|
||||||
|
.replace(/^Author:\s*/g, ''),
|
||||||
|
summary: excerpt('p')
|
||||||
|
.first()
|
||||||
|
.text()
|
||||||
|
.replace(/^.+Description\s*/g, ''),
|
||||||
|
chapters,
|
||||||
};
|
};
|
||||||
|
}
|
||||||
|
|
||||||
novel.summary = loadedCheerio('#main > article > div > p:nth-child(6)')
|
/**
|
||||||
.text()
|
* Based on the path, grab the {@link Plugin.NovelItem} information from the
|
||||||
.trim();
|
* cache. If not available in the cache, it will fetch the information.
|
||||||
|
*
|
||||||
novel.author = loadedCheerio('#main > article > div > h3:nth-child(2)')
|
* @private
|
||||||
.text()
|
*/
|
||||||
.replace(/Author:/g, '')
|
async grabCachedNovel(path: string): Promise<Plugin.NovelItem | undefined> {
|
||||||
.trim();
|
const fromCache = this.novelItemCache.get(path);
|
||||||
|
if (fromCache) {
|
||||||
const chapter: Plugin.ChapterItem[] = [];
|
return fromCache;
|
||||||
|
}
|
||||||
loadedCheerio('#main')
|
const sourceNovel = await this.grabNovel(path, false);
|
||||||
.find('li > span > a')
|
if (sourceNovel === undefined) {
|
||||||
.each((i, el) => {
|
return undefined;
|
||||||
const chapterName = loadedCheerio(el).text().trim();
|
}
|
||||||
const chapterUrl = loadedCheerio(el).attr('href');
|
const novel = {
|
||||||
|
name: sourceNovel.name,
|
||||||
if (!chapterUrl) return;
|
path: sourceNovel.path,
|
||||||
|
cover: sourceNovel.cover,
|
||||||
chapter.push({
|
};
|
||||||
name: chapterName,
|
this.novelItemCache.set(path, novel);
|
||||||
path: chapterUrl.replace(this.site, ''),
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
novel.chapters = chapter;
|
|
||||||
|
|
||||||
return novel;
|
return novel;
|
||||||
}
|
}
|
||||||
|
|
||||||
async parseChapter(chapterPath: string): Promise<string> {
|
/**
|
||||||
const result = await fetchApi(this.site + chapterPath);
|
* Get the list of all novels listed on the novels page. Cache it so filters
|
||||||
const body = await result.text();
|
* do not have to refetch the same HTML again.
|
||||||
|
*
|
||||||
const loadedCheerio = parseHTML(body);
|
* @private
|
||||||
|
*/
|
||||||
const chapterName = loadedCheerio('.entry-title').text().trim();
|
async grabCachedNovels(): Promise<
|
||||||
|
readonly (readonly [string, string, string])[] | undefined
|
||||||
let chapterText = loadedCheerio('.entry-content').html();
|
> {
|
||||||
|
if (this.allNovelsCache) {
|
||||||
if (!chapterText) {
|
return this.allNovelsCache;
|
||||||
chapterText = loadedCheerio('.page-header').html();
|
|
||||||
}
|
}
|
||||||
|
const body = await fetchApi(this.site + 'novels').then(res => res.text());
|
||||||
chapterText = `<p><h1>${chapterName}</h1></p>` + chapterText;
|
const loadedCheerio = parseHTML(body);
|
||||||
|
const novels = loadedCheerio('.entry-content ul')
|
||||||
return chapterText;
|
.map((_, listEl) => {
|
||||||
|
const list = loadedCheerio(listEl);
|
||||||
|
const category = list.prev().text();
|
||||||
|
return list
|
||||||
|
.find('a')
|
||||||
|
.map((_, anchorEl) => {
|
||||||
|
const path = this.getPath(anchorEl.attribs['href']);
|
||||||
|
if (!path) return;
|
||||||
|
const name = loadedCheerio(anchorEl).text();
|
||||||
|
return [[category, name, path]] as const;
|
||||||
|
})
|
||||||
|
.toArray();
|
||||||
|
})
|
||||||
|
.toArray();
|
||||||
|
this.allNovelsCache = novels;
|
||||||
|
return novels;
|
||||||
}
|
}
|
||||||
|
|
||||||
async searchNovels(searchTerm: string): Promise<Plugin.NovelItem[]> {
|
/**
|
||||||
const url = this.site + 'novels';
|
* Parse list of paths of recently updated novels from the homepage.
|
||||||
|
*
|
||||||
const result = await fetchApi(url);
|
* @private
|
||||||
const body = await result.text();
|
*/
|
||||||
|
async latestNovels(): Promise<readonly string[]> {
|
||||||
|
const body = await fetchApi(this.site).then(res => res.text());
|
||||||
const loadedCheerio = parseHTML(body);
|
const loadedCheerio = parseHTML(body);
|
||||||
return this.parseNovels(loadedCheerio, searchTerm);
|
const novelPaths = loadedCheerio('#main')
|
||||||
|
.find('a[rel="category tag"]')
|
||||||
|
.map((_, anchorEl) => {
|
||||||
|
const path = this.getPath(anchorEl.attribs['href']);
|
||||||
|
return path;
|
||||||
|
})
|
||||||
|
.toArray();
|
||||||
|
const uniqueNovelPaths = new Set(novelPaths);
|
||||||
|
return Array.from(uniqueNovelPaths);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse list of paths from novels list, optionally filtered.
|
||||||
|
*
|
||||||
|
* @private
|
||||||
|
*/
|
||||||
|
async allNovels(
|
||||||
|
categoryFilter: FilterValueWithType<FilterTypes.CheckboxGroup>,
|
||||||
|
) {
|
||||||
|
const allNovels = await this.grabCachedNovels();
|
||||||
|
if (!allNovels) return [];
|
||||||
|
return allNovels
|
||||||
|
.filter(([category]) => categoryFilter.value.includes(category))
|
||||||
|
.map(([, , path]) => path);
|
||||||
|
}
|
||||||
|
|
||||||
|
async popularNovels(
|
||||||
|
pageNo: number,
|
||||||
|
options: Plugin.PopularNovelsOptions<typeof this.filters>,
|
||||||
|
): Promise<Plugin.NovelItem[]> {
|
||||||
|
if (pageNo !== 1) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
const novelsList = options.showLatestNovels
|
||||||
|
? this.latestNovels()
|
||||||
|
: this.allNovels(options.filters.category);
|
||||||
|
return await this.asyncMap(
|
||||||
|
await novelsList,
|
||||||
|
this.grabCachedNovel.bind(this),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async parseNovel(novelPath: string): Promise<Plugin.SourceNovel> {
|
||||||
|
const sourceNovel = await this.grabNovel(novelPath, true);
|
||||||
|
if (sourceNovel === undefined) {
|
||||||
|
throw new Error(`The path "${novelPath}" could not be resolved.`);
|
||||||
|
}
|
||||||
|
return sourceNovel;
|
||||||
|
}
|
||||||
|
|
||||||
|
async parseChapter(chapterPath: string): Promise<string> {
|
||||||
|
const chapterLink = `${this.site}wp-json/wp/v2/posts?slug=${chapterPath}`;
|
||||||
|
const chapter = await fetchApi(chapterLink).then(res => res.json());
|
||||||
|
if (chapter.length !== 1) {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
const title = `<h1>${chapter[0].title.rendered}</h1>`;
|
||||||
|
const content = chapter[0].content.rendered;
|
||||||
|
return `${title}${content}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async searchNovels(
|
||||||
|
searchTerm: string,
|
||||||
|
pageNo: number,
|
||||||
|
): Promise<Plugin.NovelItem[]> {
|
||||||
|
if (pageNo !== 1) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
const allNovels = await this.grabCachedNovels();
|
||||||
|
if (!allNovels) return [];
|
||||||
|
const foundNovels = allNovels
|
||||||
|
.filter(([, name]) =>
|
||||||
|
name.toLocaleLowerCase().includes(searchTerm.toLocaleLowerCase()),
|
||||||
|
)
|
||||||
|
.map(([, , path]) => path);
|
||||||
|
return await this.asyncMap(
|
||||||
|
await foundNovels,
|
||||||
|
this.grabCachedNovel.bind(this),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user