diff --git a/.gitignore b/.gitignore index ca63257..99be8a8 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ dist total.svg src/plugins/index.ts .history +broken-sites-report.json \ No newline at end of file diff --git a/package.json b/package.json index 92c4f2e..1d30a90 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "lint:fix": "npx eslint . --fix", "format": "prettier --write \"./**/*.{js,ts}\"", "format:check": "prettier --check \"./**/*.{js,ts}\"", + "check:sites": "node scripts/check-plugin-sites.js", "prepare": "husky" }, "author": "LNReader", diff --git a/scripts/check-plugin-sites.js b/scripts/check-plugin-sites.js new file mode 100644 index 0000000..f68cebd --- /dev/null +++ b/scripts/check-plugin-sites.js @@ -0,0 +1,340 @@ +#!/usr/bin/env node + +import https from 'https'; +import http from 'http'; +import { URL, fileURLToPath } from 'url'; +import fs from 'fs'; +import path, { dirname } from 'path'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +const TIMEOUT = 10000; +const CONCURRENT_REQUESTS = 10; +const PLUGINS_JSON_URL = + 'https://raw.githubusercontent.com/LNReader/lnreader-plugins/refs/heads/plugins/v3.0.0/.dist/plugins.json'; + +const results = { + accessible: [], + inaccessible: [], + cloudflare: [], + errors: [], +}; + +let totalSites = 0; +let checkedSites = 0; +let activeRequests = 0; + +function checkSite(url, pluginInfo) { + return new Promise(resolve => { + const parsedUrl = new URL(url); + const isHttps = parsedUrl.protocol === 'https:'; + const client = isHttps ? https : http; + + const options = { + hostname: parsedUrl.hostname, + port: parsedUrl.port || (isHttps ? 443 : 80), + path: parsedUrl.pathname + parsedUrl.search, + method: 'HEAD', + timeout: TIMEOUT, + headers: { + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + }, + }; + + const req = client.request(options, res => { + const headers = res.headers || {}; + const isCloudflare = + headers['cf-ray'] || + headers['server']?.toLowerCase().includes('cloudflare') || + headers['cf-cache-status'] || + headers['cf-request-id'] || + (res.statusCode === 403 && + (headers['server']?.toLowerCase().includes('cloudflare') || + headers['cf-ray'])); + + if (res.statusCode >= 200 && res.statusCode < 400) { + resolve({ + status: 'accessible', + statusCode: res.statusCode, + url, + pluginInfo, + }); + } else if (res.statusCode === 403 && isCloudflare) { + resolve({ + status: 'cloudflare', + statusCode: res.statusCode, + url, + pluginInfo, + reason: 'HTTP 403 (Cloudflare protected)', + }); + } else { + resolve({ + status: 'inaccessible', + statusCode: res.statusCode, + url, + pluginInfo, + reason: `HTTP ${res.statusCode}`, + }); + } + res.destroy(); + }); + + req.on('error', error => { + resolve({ + status: 'inaccessible', + url, + pluginInfo, + reason: error.message || 'Connection error', + error: error.code || 'UNKNOWN', + }); + }); + + req.on('timeout', () => { + req.destroy(); + resolve({ + status: 'inaccessible', + url, + pluginInfo, + reason: 'Request timeout', + error: 'TIMEOUT', + }); + }); + + req.setTimeout(TIMEOUT); + req.end(); + }); +} + +async function processSites(sites) { + const queue = [...sites]; + + async function processNext() { + if (queue.length === 0) { + return; + } + + activeRequests++; + const { url, pluginInfo } = queue.shift(); + + try { + const result = await checkSite(url, pluginInfo); + checkedSites++; + + if (result.status === 'accessible') { + results.accessible.push(result); + process.stdout.write( + `\r✓ ${checkedSites}/${totalSites} - ${pluginInfo.name || url}`, + ); + } else if (result.status === 'cloudflare') { + results.cloudflare.push(result); + process.stdout.write( + `\r⚠ ${checkedSites}/${totalSites} - ${pluginInfo.name || url} (Cloudflare)`, + ); + } else { + results.inaccessible.push(result); + process.stdout.write( + `\r✗ ${checkedSites}/${totalSites} - ${pluginInfo.name || url} (${result.reason})`, + ); + } + } catch (error) { + checkedSites++; + results.errors.push({ + url, + pluginInfo, + error: error.message, + }); + process.stdout.write( + `\r✗ ${checkedSites}/${totalSites} - ${pluginInfo.name || url} (Error)`, + ); + } finally { + activeRequests--; + await processNext(); + } + } + + const promises = []; + for (let i = 0; i < CONCURRENT_REQUESTS && i < sites.length; i++) { + promises.push(processNext()); + } + + await Promise.all(promises); +} + +function fetchJson(url) { + return new Promise((resolve, reject) => { + const parsedUrl = new URL(url); + const client = parsedUrl.protocol === 'https:' ? https : http; + + const options = { + hostname: parsedUrl.hostname, + port: parsedUrl.port || (parsedUrl.protocol === 'https:' ? 443 : 80), + path: parsedUrl.pathname + parsedUrl.search, + method: 'GET', + headers: { + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + }, + }; + + const req = client.request(options, res => { + if (res.statusCode !== 200) { + reject( + new Error(`Failed to fetch plugins.json: HTTP ${res.statusCode}`), + ); + return; + } + + let data = ''; + res.on('data', chunk => { + data += chunk; + }); + + res.on('end', () => { + try { + resolve(JSON.parse(data)); + } catch (error) { + reject(new Error(`Failed to parse JSON: ${error.message}`)); + } + }); + }); + + req.on('error', error => { + reject(new Error(`Request failed: ${error.message}`)); + }); + + req.setTimeout(30000, () => { + req.destroy(); + reject(new Error('Request timeout')); + }); + + req.end(); + }); +} + +async function main() { + console.log('Fetching plugins.json from GitHub...\n'); + + try { + const plugins = await fetchJson(PLUGINS_JSON_URL); + console.log(`Successfully fetched ${plugins.length} plugins.\n`); + + const siteMap = new Map(); + plugins.forEach(plugin => { + const site = plugin.site; + if (site && site !== 'url' && site.trim() !== '') { + if (!siteMap.has(site)) { + siteMap.set(site, { + url: site, + pluginInfo: { + id: plugin.id, + name: plugin.name, + lang: plugin.lang, + }, + }); + } + } + }); + + const sites = Array.from(siteMap.values()); + totalSites = sites.length; + + console.log(`Found ${totalSites} unique sites to check.\n`); + console.log('Checking sites...\n'); + + await processSites(sites); + + console.log('\n\n' + '='.repeat(80)); + console.log('RESULTS SUMMARY'); + console.log('='.repeat(80)); + console.log(`Total sites checked: ${totalSites}`); + console.log(`Accessible: ${results.accessible.length}`); + console.log(`Cloudflare protected: ${results.cloudflare.length}`); + console.log(`Inaccessible/Broken: ${results.inaccessible.length}`); + console.log(`Errors: ${results.errors.length}`); + + if (results.cloudflare.length > 0) { + console.warn('\n' + '='.repeat(80)); + console.warn('CLOUDFLARE PROTECTED SITES (excluded from broken list):'); + console.warn('='.repeat(80)); + results.cloudflare.forEach((item, index) => { + console.warn( + `\n${index + 1}. ${item.pluginInfo.name || 'Unknown'} (${item.pluginInfo.id})`, + ); + console.warn(` URL: ${item.url}`); + console.warn(` Language: ${item.pluginInfo.lang || 'N/A'}`); + }); + } + + if (results.inaccessible.length > 0 || results.errors.length > 0) { + console.log('\n' + '='.repeat(80)); + console.log('INACCESSIBLE OR BROKEN SITES:'); + console.log('='.repeat(80)); + + const allFailed = [...results.inaccessible, ...results.errors]; + allFailed.forEach((item, index) => { + console.log( + `\n${index + 1}. ${item.pluginInfo.name || 'Unknown'} (${item.pluginInfo.id})`, + ); + console.log(` URL: ${item.url}`); + console.log(` Language: ${item.pluginInfo.lang || 'N/A'}`); + if (item.statusCode) { + console.log(` Status Code: ${item.statusCode}`); + } + if (item.reason) { + console.log(` Reason: ${item.reason}`); + } + if (item.error) { + console.log(` Error: ${item.error}`); + } + }); + + const reportFile = path.join(__dirname, '..', 'broken-sites-report.json'); + fs.writeFileSync( + reportFile, + JSON.stringify( + { + timestamp: new Date().toISOString(), + total: totalSites, + accessible: results.accessible.length, + cloudflare: results.cloudflare.length, + inaccessible: results.inaccessible.length, + errors: results.errors.length, + brokenSites: allFailed.map(item => ({ + id: item.pluginInfo.id, + name: item.pluginInfo.name, + url: item.url, + lang: item.pluginInfo.lang, + statusCode: item.statusCode, + reason: item.reason, + error: item.error, + })), + cloudflareSites: results.cloudflare.map(item => ({ + id: item.pluginInfo.id, + name: item.pluginInfo.name, + url: item.url, + lang: item.pluginInfo.lang, + statusCode: item.statusCode, + reason: item.reason, + })), + }, + null, + 2, + ), + ); + + console.log(`\n\nDetailed report saved to: ${reportFile}`); + } else { + console.log('\n✓ All sites are accessible!'); + } + } catch (error) { + console.error('Error:', error.message); + process.exit(1); + } +} + +main().catch(error => { + console.error('Fatal error:', error); + process.exit(1); +});