masi
5c444fc35a
- load-plugins: bundle once per source revision with an id index; loadPlugin(id) loads one plugin (144 ms / 87 MB instead of ~15 s / 1.2 GB). - fetch-novel: keep only the contiguous run of free chapters (no locked teasers). - update-novels: append newly free chapters to our EPUBs without moving positions. - find-novel --json for programs; shared EPUB pieces in epub.js. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
87 lines
2.7 KiB
JavaScript
87 lines
2.7 KiB
JavaScript
// Search every plugin of the given languages for a title and list the sources
|
||
// that carry it. Usage: node scripts/find-novel.js [--json] "<title>" [lang...]
|
||
// Languages are plugin folder names (default: english multi). --json prints one JSON
|
||
// array of {source, id, name, path, url, exact} for programs (the request page).
|
||
import { loadPlugins, print } from './load-plugins.js';
|
||
|
||
const TIMEOUT_MS = 20_000;
|
||
const CONCURRENCY = 16;
|
||
const PER_SOURCE = 3;
|
||
|
||
const args = process.argv.slice(2);
|
||
const asJson = args[0] === '--json';
|
||
const [term, ...langArgs] = asJson ? args.slice(1) : args;
|
||
if (!term) {
|
||
console.error('Usage: node scripts/find-novel.js "<title>" [lang...]');
|
||
process.exit(2);
|
||
}
|
||
|
||
const plugins = await loadPlugins(
|
||
langArgs.length ? langArgs : ['english', 'multi'],
|
||
);
|
||
process.stderr.write(`Searching ${plugins.length} sources for "${term}"...\n`);
|
||
|
||
const hits = [];
|
||
let failed = 0;
|
||
const queue = [...plugins];
|
||
const worker = async () => {
|
||
for (let plugin; (plugin = queue.shift());) {
|
||
try {
|
||
const found = await Promise.race([
|
||
plugin.searchNovels(term, 1),
|
||
new Promise((_, reject) =>
|
||
setTimeout(() => reject(new Error('timeout')), TIMEOUT_MS),
|
||
),
|
||
]);
|
||
for (const novel of (found || []).slice(0, PER_SOURCE)) {
|
||
const url = plugin.resolveUrl
|
||
? plugin.resolveUrl(novel.path, true)
|
||
: new URL(novel.path, plugin.site).href;
|
||
hits.push({
|
||
source: plugin.name,
|
||
id: plugin.id,
|
||
name: novel.name,
|
||
path: novel.path,
|
||
url,
|
||
});
|
||
}
|
||
} catch {
|
||
failed++;
|
||
}
|
||
}
|
||
};
|
||
await Promise.all(Array.from({ length: CONCURRENCY }, worker));
|
||
|
||
// Keep titles containing every word of the query (quotes/punctuation ignored).
|
||
const norm = t =>
|
||
t
|
||
.toLowerCase()
|
||
.replace(/[’']/g, '')
|
||
.replace(/[^a-z0-9]+/g, ' ')
|
||
.trim();
|
||
const words = norm(term).split(' ');
|
||
const exact = norm(term);
|
||
const matches = hits.filter(h =>
|
||
words.every(w => norm(h.name).split(' ').includes(w)),
|
||
);
|
||
const rank = h => (norm(h.name) === exact ? 0 : 1);
|
||
matches.sort((a, b) => rank(a) - rank(b) || a.source.localeCompare(b.source));
|
||
|
||
if (asJson) {
|
||
print(JSON.stringify(matches.map(h => ({ ...h, exact: rank(h) === 0 }))));
|
||
process.exit(0);
|
||
}
|
||
for (const h of matches) {
|
||
print(
|
||
`${rank(h) === 0 ? '*' : ' '} ${h.source.padEnd(28)} ${h.name}\n` +
|
||
` ${''.padEnd(28)} ${h.url}\n` +
|
||
` ${''.padEnd(28)} novel fetch ${h.id} ${h.path}`,
|
||
);
|
||
}
|
||
process.stderr.write(
|
||
`${matches.length} matches (* = exact title), ${hits.length - matches.length} loose results hidden; ` +
|
||
`${plugins.length - failed}/${plugins.length} sources answered.\n`,
|
||
);
|
||
// Keep-alive sockets can hold the event loop open; results are already printed.
|
||
process.exit(0);
|