fix: Update NovelBuddy URL and Fix Scraping Logic (#2135)

* fix(novelbuddy): update site URL and fix scraping logic

* feat(novelbuddy): filter webnovel watermark and bump version to 2.0.1

* feat(novelbuddy): simplify search, fix summary formatting, version 2.0.0

* chore: restore package-lock.json libc platform constraints

Reverts lockfile pruning from commit 18ca477 to recover deleted glibc and musl entries for cross-platform compatibility.

* update imports

---------

Co-authored-by: root <root@localhost.localdomain>
This commit is contained in:
7ui77
2026-04-22 10:29:18 +07:00
committed by GitHub
parent ecf871ed0d
commit d483a6413d
+311 -300
View File
@@ -1,353 +1,364 @@
import { CheerioAPI, load as parseHTML } from 'cheerio';
import { load as parseHTML } from 'cheerio';
import { fetchApi } from '@libs/fetch';
import { Plugin } from '@/types/plugin';
import { Filters, FilterTypes } from '@libs/filterInputs';
class NovelBuddy implements Plugin.PluginBase {
id = 'novelbuddy';
name = 'NovelBuddy.io';
site = 'https://novelbuddy.io/';
version = '1.0.4';
name = 'NovelBuddy';
site = 'https://novelbuddy.com/';
version = '2.0.0'; // Bumped version
icon = 'src/en/novelbuddy/icon.png';
parseNovels(loadedCheerio: CheerioAPI) {
const novels: Plugin.NovelItem[] = [];
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Referer': 'https://novelbuddy.com/',
};
loadedCheerio('.book-item').each((idx, ele) => {
const novelName = loadedCheerio(ele).find('.title').text();
const novelCover =
'https:' + loadedCheerio(ele).find('img').attr('data-src');
const novelUrl = loadedCheerio(ele)
.find('.title a')
.attr('href')
?.substring(1);
async popularNovels(pageNo: number): Promise<Plugin.NovelItem[]> {
const url = `https://api.novelbuddy.com/titles?sort=popular&page=${pageNo}`;
if (!novelUrl) return;
try {
const result = await fetchApi(url, { headers: this.headers });
const json = await result.json();
const novel = { name: novelName, cover: novelCover, path: novelUrl };
novels.push(novel);
});
return novels;
}
async popularNovels(
pageNo: number,
{ filters }: Plugin.PopularNovelsOptions<Filters>,
): Promise<Plugin.NovelItem[]> {
// create empty query params
const params = new URLSearchParams();
// apply all filters
params.append('sort', filters.orderBy.value.toString());
params.append('status', filters.status.value.toString());
if (filters.genre.value instanceof Array) {
filters.genre.value.forEach(genre => {
params.append('genre[]', genre.toString());
});
if (json?.data?.items) {
return json.data.items.map((item: any) => ({
name: item.name,
cover: item.cover,
path: item.url.replace(/^\//, ''),
}));
}
} catch (e) {
// Fallback to HTML
}
params.append('q', filters.keyword.value.toString());
params.append('page', pageNo.toString());
const url = `${this.site}search?${params.toString()}`;
const result = await fetchApi(url);
const body = await result.text();
const loadedCheerio = parseHTML(body);
return this.parseNovels(loadedCheerio);
const htmlUrl = `${this.site}popular?page=${pageNo}`;
const htmlRes = await fetchApi(htmlUrl, { headers: this.headers });
const htmlBody = await htmlRes.text();
const $ = parseHTML(htmlBody);
const script = $('#__NEXT_DATA__').html();
if (script) {
const data = JSON.parse(script);
const items = data.props.pageProps.items || [];
return items.map((item: any) => ({
name: item.name,
cover: item.cover,
path: item.url.replace(/^\//, ''),
}));
}
return [];
}
async parseNovel(novelPath: string): Promise<Plugin.SourceNovel> {
const response = await fetchApi(this.site + novelPath);
const response = await fetchApi(this.site + novelPath, {
headers: this.headers,
});
const body = await response.text();
const $ = parseHTML(body);
const coverSrc =
$('.img-cover img').attr('data-src') ||
$('.img-cover img').attr('src') ||
'';
const script = $('#__NEXT_DATA__').html();
if (!script) throw new Error('Could not find __NEXT_DATA__');
const normalizeUrl = (url?: string): string => {
if (!url) return '';
if (url.startsWith('//')) return `https:${url}`;
if (url.startsWith('/')) return `${this.site}${url.replace(/^\//, '')}`;
return url;
};
const data = JSON.parse(script);
const initialManga = data.props.pageProps.initialManga;
const normalizeText = (text: string): string =>
text.replace(/\s+/g, ' ').replace(/\s*,\s*$/, '').trim();
if (!initialManga) throw new Error('Could not find initialManga data');
const getLinkTextList = (root: ReturnType<typeof $>): string[] =>
root
.find('a')
.map((_, el) => normalizeText($(el).text()))
.toArray()
.filter(Boolean);
// Fix summary formatting by preserving line breaks before stripping HTML
let formattedSummary = initialManga.summary || '';
formattedSummary = formattedSummary
.replace(/<br\s*\/?>/gi, '\n') // Replace <br> or <br/> with newline
.replace(/<\/p>/gi, '\n\n') // Replace </p> with double newline for paragraphs
.replace(/<[^>]*>?/gm, '') // Strip all remaining HTML tags
.trim(); // Remove extra whitespace at start/end
const novel: Plugin.SourceNovel = {
path: novelPath,
name: normalizeText($('.name h1').first().text()) || 'Untitled',
cover: normalizeUrl(coverSrc),
summary: normalizeText($('.section-body.summary .content').text()),
name: initialManga.name,
cover: initialManga.cover,
summary: formattedSummary,
author: initialManga.authors?.map((a: any) => a.name).join(', ') || '',
artist: initialManga.artists?.map((a: any) => a.name).join(', ') || '',
status: initialManga.status,
genres: initialManga.genres?.map((g: any) => g.name).join(',') || '',
chapters: [],
};
$('.meta.box p').each((_, el) => {
const row = $(el);
const label = normalizeText(row.find('strong').first().text());
const linkTexts = getLinkTextList(row);
switch (label) {
case 'Authors :':
novel.author = linkTexts.join(', ');
break;
case 'Artists :':
case 'Artist :':
novel.artist = linkTexts.join(', ');
break;
case 'Status :':
novel.status = linkTexts[0] || normalizeText(row.text().replace(label, ''));
break;
case 'Genres :':
novel.genres = linkTexts.join(',');
break;
}
});
const ratingText = normalizeText($('.rating .score').first().text()).replace(
/[^0-9.]/g,
'',
);
const rating = Number.parseFloat(ratingText);
if (!Number.isNaN(rating)) {
novel.rating = rating;
if (initialManga.ratingStats) {
novel.rating = initialManga.ratingStats.average;
}
const scriptText = $('script')
.map((_, el) => $(el).html() || '')
.toArray()
.join('\n');
const novelIdMatch = scriptText.match(/bookId\s*=\s*(\d+)\s*;/);
if (!novelIdMatch) {
return novel;
}
const novelId = novelIdMatch[1];
const getChapters = async (id: string): Promise<Plugin.ChapterItem[]> => {
const chapterListUrl = `${this.site}api/manga/${id}/chapters?source=detail`;
const chapterResponse = await fetchApi(chapterListUrl);
const chapterHtml = await chapterResponse.text();
const $$ = parseHTML(chapterHtml);
const chapters: Plugin.ChapterItem[] = [];
const months = [
'jan',
'feb',
'mar',
'apr',
'may',
'jun',
'jul',
'aug',
'sep',
'oct',
'nov',
'dec',
];
const dateRegex = new RegExp(
`(${months.join('|')})\\s+(\\d{1,2}),\\s+(\\d{4})`,
'i',
);
$$('li').each((_, el) => {
const item = $$(el);
const chapterName = normalizeText(item.find('.chapter-title').text());
const releaseDateText = normalizeText(item.find('.chapter-update').text());
const chapterHref = item.find('a').attr('href');
if (!chapterHref) return;
const chapterPath = chapterHref.startsWith('/')
? chapterHref.slice(1)
: chapterHref;
const chapterItem: Plugin.ChapterItem = {
name: chapterName || 'Untitled',
path: chapterPath,
};
const dateMatch = dateRegex.exec(releaseDateText);
if (dateMatch) {
const year = Number.parseInt(dateMatch[3], 10);
const month = months.indexOf(dateMatch[1].toLowerCase());
const day = Number.parseInt(dateMatch[2], 10);
if (month >= 0) {
chapterItem.releaseTime = new Date(year, month, day).toISOString();
}
} else if (releaseDateText) {
chapterItem.releaseTime = releaseDateText;
}
const chapterNumberMatch = chapterName.match(
/chapter\s+(\d+(?:\.\d+)?)/i,
);
if (chapterNumberMatch) {
chapterItem.chapterNumber = Number.parseFloat(chapterNumberMatch[1]);
}
chapters.push(chapterItem);
// Fetch full chapter list from API
const chaptersUrl = `https://api.novelbuddy.com/titles/${initialManga.id}/chapters`;
try {
const chaptersResponse = await fetchApi(chaptersUrl, {
headers: this.headers,
});
const chaptersJson = await chaptersResponse.json();
return chapters;
};
novel.chapters = (await getChapters(novelId)).reverse();
if (chaptersJson?.success && chaptersJson?.data?.chapters) {
novel.chapters = chaptersJson.data.chapters
.map((chapter: any) => ({
name: chapter.name,
path: chapter.url.replace(/^\//, ''),
releaseTime: chapter.updated_at,
}))
.reverse();
} else if (initialManga.chapters) {
novel.chapters = initialManga.chapters
.map((chapter: any) => ({
name: chapter.name,
path: chapter.url.replace(/^\//, ''),
releaseTime: chapter.updatedAt,
}))
.reverse();
}
} catch (e) {
if (initialManga.chapters) {
novel.chapters = initialManga.chapters
.map((chapter: any) => ({
name: chapter.name,
path: chapter.url.replace(/^\//, ''),
releaseTime: chapter.updatedAt,
}))
.reverse();
}
}
return novel;
}
async parseChapter(chapterPath: string): Promise<string> {
const result = await fetchApi(this.site + chapterPath);
const result = await fetchApi(this.site + chapterPath, {
headers: this.headers,
});
const body = await result.text();
const $ = parseHTML(body);
const loadedCheerio = parseHTML(body);
const script = $('#__NEXT_DATA__').html();
if (!script) throw new Error('Could not find __NEXT_DATA__');
loadedCheerio('#listen-chapter').remove();
loadedCheerio('#google_translate_element').remove();
const data = JSON.parse(script);
const initialChapter = data.props.pageProps.initialChapter;
if (!initialChapter) throw new Error('Could not find chapter content');
const chapterText = loadedCheerio('.chapter__content').html() || '';
let content = initialChapter.content;
return chapterText;
if (content) {
// Remove Webnovel watermarks/ads
content = content.replace(
/Find authorized novels in Webnovel.*?faster updates, better experience.*?Please click www\.webnovel\.com for visiting\./gi,
'',
);
// Remove obfuscated freewebnovel watermarks (e.g., free𝑤𝑒𝑏novel.com)
content = content.replace(/free.*?novel\.com/gi, '');
}
return content;
}
async searchNovels(
searchTerm: string,
page: number,
): Promise<Plugin.NovelItem[]> {
const url = `${this.site}search?q=${encodeURIComponent(searchTerm)}&page=${page}`;
const url = `https://api.novelbuddy.com/titles?q=${encodeURIComponent(searchTerm)}&page=${page}`;
const result = await fetchApi(url, { headers: this.headers });
const json = await result.json();
const result = await fetchApi(url);
const body = await result.text();
if (!json || !json.data || !json.data.items) {
return [];
}
const loadedCheerio = parseHTML(body);
return this.parseNovels(loadedCheerio);
return json.data.items.map((item: any) => ({
name: item.name,
cover: item.cover,
path: item.url.replace(/^\//, ''),
}));
}
}
export default new NovelBuddy();
import { load as parseHTML } from 'cheerio';
import { fetchApi } from '@libs/fetch';
import { Plugin } from '@/types/plugin';
class NovelBuddy implements Plugin.PluginBase {
id = 'novelbuddy';
name = 'NovelBuddy';
site = 'https://novelbuddy.com/';
version = '2.0.1'; // Bumped version
icon = 'src/en/novelbuddy/icon.png';
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Referer': 'https://novelbuddy.com/',
};
async popularNovels(pageNo: number): Promise<Plugin.NovelItem[]> {
const url = `https://api.novelbuddy.com/titles?sort=popular&page=${pageNo}`;
try {
const result = await fetchApi(url, { headers: this.headers });
const json = await result.json();
if (json?.data?.items) {
return json.data.items.map((item: any) => ({
name: item.name,
cover: item.cover,
path: item.url.replace(/^\//, ''),
}));
}
} catch (e) {
// Fallback to HTML
}
const htmlUrl = `${this.site}popular?page=${pageNo}`;
const htmlRes = await fetchApi(htmlUrl, { headers: this.headers });
const htmlBody = await htmlRes.text();
const $ = parseHTML(htmlBody);
const script = $('#__NEXT_DATA__').html();
if (script) {
const data = JSON.parse(script);
const items = data.props.pageProps.items || [];
return items.map((item: any) => ({
name: item.name,
cover: item.cover,
path: item.url.replace(/^\//, ''),
}));
}
return [];
}
async parseNovel(novelPath: string): Promise<Plugin.SourceNovel> {
const response = await fetchApi(this.site + novelPath, {
headers: this.headers,
});
const body = await response.text();
const $ = parseHTML(body);
const script = $('#__NEXT_DATA__').html();
if (!script) throw new Error('Could not find __NEXT_DATA__');
const data = JSON.parse(script);
const initialManga = data.props.pageProps.initialManga;
if (!initialManga) throw new Error('Could not find initialManga data');
// Fix summary formatting by preserving line breaks before stripping HTML
let formattedSummary = initialManga.summary || '';
formattedSummary = formattedSummary
.replace(/<br\s*\/?>/gi, '\n') // Replace <br> or <br/> with newline
.replace(/<\/p>/gi, '\n\n') // Replace </p> with double newline for paragraphs
.replace(/<[^>]*>?/gm, '') // Strip all remaining HTML tags
.trim(); // Remove extra whitespace at start/end
const novel: Plugin.SourceNovel = {
path: novelPath,
name: initialManga.name,
cover: initialManga.cover,
summary: formattedSummary,
author: initialManga.authors?.map((a: any) => a.name).join(', ') || '',
artist: initialManga.artists?.map((a: any) => a.name).join(', ') || '',
status: initialManga.status,
genres: initialManga.genres?.map((g: any) => g.name).join(',') || '',
chapters: [],
};
if (initialManga.ratingStats) {
novel.rating = initialManga.ratingStats.average;
}
// Fetch full chapter list from API
const chaptersUrl = `https://api.novelbuddy.com/titles/${initialManga.id}/chapters`;
try {
const chaptersResponse = await fetchApi(chaptersUrl, {
headers: this.headers,
});
const chaptersJson = await chaptersResponse.json();
if (chaptersJson?.success && chaptersJson?.data?.chapters) {
novel.chapters = chaptersJson.data.chapters
.map((chapter: any) => ({
name: chapter.name,
path: chapter.url.replace(/^\//, ''),
releaseTime: chapter.updated_at,
}))
.reverse();
} else if (initialManga.chapters) {
novel.chapters = initialManga.chapters
.map((chapter: any) => ({
name: chapter.name,
path: chapter.url.replace(/^\//, ''),
releaseTime: chapter.updatedAt,
}))
.reverse();
}
} catch (e) {
if (initialManga.chapters) {
novel.chapters = initialManga.chapters
.map((chapter: any) => ({
name: chapter.name,
path: chapter.url.replace(/^\//, ''),
releaseTime: chapter.updatedAt,
}))
.reverse();
}
}
return novel;
}
async parseChapter(chapterPath: string): Promise<string> {
const result = await fetchApi(this.site + chapterPath, {
headers: this.headers,
});
const body = await result.text();
const $ = parseHTML(body);
const script = $('#__NEXT_DATA__').html();
if (!script) throw new Error('Could not find __NEXT_DATA__');
const data = JSON.parse(script);
const initialChapter = data.props.pageProps.initialChapter;
if (!initialChapter) throw new Error('Could not find chapter content');
let content = initialChapter.content;
if (content) {
// Remove Webnovel watermarks/ads
content = content.replace(
/Find authorized novels in Webnovel.*?faster updates, better experience.*?Please click www\.webnovel\.com for visiting\./gi,
'',
);
// Remove obfuscated freewebnovel watermarks (e.g., free𝑤𝑒𝑏novel.com)
content = content.replace(/free.*?novel\.com/gi, '');
}
return content;
}
async searchNovels(
searchTerm: string,
page: number,
): Promise<Plugin.NovelItem[]> {
const url = `https://api.novelbuddy.com/titles?q=${encodeURIComponent(searchTerm)}&page=${page}`;
const result = await fetchApi(url, { headers: this.headers });
const json = await result.json();
if (!json || !json.data || !json.data.items) {
return [];
}
return json.data.items.map((item: any) => ({
name: item.name,
cover: item.cover,
path: item.url.replace(/^\//, ''),
}));
}
filters = {
orderBy: {
value: 'views',
label: 'Order by',
options: [
{ label: 'Views', value: 'views' },
{ label: 'Updated At', value: 'updated_at' },
{ label: 'Created At', value: 'created_at' },
{ label: 'Name', value: 'name' },
{ label: 'Rating', value: 'rating' },
],
type: FilterTypes.Picker,
},
keyword: {
value: '',
label: 'Keywords',
type: FilterTypes.TextInput,
},
status: {
value: 'all',
label: 'Status',
options: [
{ label: 'All', value: 'all' },
{ label: 'Ongoing', value: 'ongoing' },
{ label: 'Completed', value: 'completed' },
],
type: FilterTypes.Picker,
},
genre: {
value: [],
label: 'Genres (OR, not AND)',
options: [
{ label: 'Action', value: 'action' },
{ label: 'Action Adventure', value: 'action-adventure' },
{ label: 'Adult', value: 'adult' },
{ label: 'Adventcure', value: 'adventcure' },
{ label: 'Adventure', value: 'adventure' },
{ label: 'Adventurer', value: 'adventurer' },
{ label: 'Bender', value: 'bender' },
{ label: 'Chinese', value: 'chinese' },
{ label: 'Comedy', value: 'comedy' },
{ label: 'Cultivation', value: 'cultivation' },
{ label: 'Drama', value: 'drama' },
{ label: 'Eastern', value: 'eastern' },
{ label: 'Ecchi', value: 'ecchi' },
{ label: 'Fan Fiction', value: 'fan-fiction' },
{ label: 'Fanfiction', value: 'fanfiction' },
{ label: 'Fantas', value: 'fantas' },
{ label: 'Fantasy', value: 'fantasy' },
{ label: 'Game', value: 'game' },
{ label: 'Gender', value: 'gender' },
{ label: 'Gender Bender', value: 'gender-bender' },
{ label: 'Harem', value: 'harem' },
{ label: 'HaremAction', value: 'haremaction' },
{ label: 'Haremv', value: 'haremv' },
{ label: 'Historica', value: 'historica' },
{ label: 'Historical', value: 'historical' },
{ label: 'History', value: 'history' },
{ label: 'Horror', value: 'horror' },
{ label: 'Isekai', value: 'isekai' },
{ label: 'Josei', value: 'josei' },
{ label: 'Lolicon', value: 'lolicon' },
{ label: 'Magic', value: 'magic' },
{ label: 'Martial', value: 'martial' },
{ label: 'Martial Arts', value: 'martial-arts' },
{ label: 'Mature', value: 'mature' },
{ label: 'Mecha', value: 'mecha' },
{ label: 'Military', value: 'military' },
{ label: 'Modern Life', value: 'modern-life' },
{ label: 'Mystery', value: 'mystery' },
{ label: 'Mystery Adventure', value: 'mystery-adventure' },
{ label: 'Psychologic', value: 'psychologic' },
{ label: 'Psychological', value: 'psychological' },
{ label: 'Reincarnation', value: 'reincarnation' },
{ label: 'Romance', value: 'romance' },
{ label: 'Romance Adventure', value: 'romance-adventure' },
{ label: 'Romance Harem', value: 'romance-harem' },
{ label: 'Romancem', value: 'romancem' },
{ label: 'School Life', value: 'school-life' },
{ label: 'Sci-fi', value: 'sci-fi' },
{ label: 'Seinen', value: 'seinen' },
{ label: 'Shoujo', value: 'shoujo' },
{ label: 'Shoujo Ai', value: 'shoujo-ai' },
{ label: 'Shounen', value: 'shounen' },
{ label: 'Shounen Ai', value: 'shounen-ai' },
{ label: 'Slice of Life', value: 'slice-of-life' },
{ label: 'Smut', value: 'smut' },
{ label: 'Sports', value: 'sports' },
{ label: 'Superna', value: 'superna' },
{ label: 'Supernatural', value: 'supernatural' },
{ label: 'System', value: 'system' },
{ label: 'Tragedy', value: 'tragedy' },
{ label: 'Urban', value: 'urban' },
{ label: 'Urban Life', value: 'urban-life' },
{ label: 'Wuxia', value: 'wuxia' },
{ label: 'Xianxia', value: 'xianxia' },
{ label: 'Xuanhuan', value: 'xuanhuan' },
{ label: 'Yaoi', value: 'yaoi' },
{ label: 'Yuri', value: 'yuri' },
],
type: FilterTypes.CheckboxGroup,
},
} satisfies Filters;
}
export default new NovelBuddy();