lnreader/scripts/generateSource.cjs

/**
 * This script can be used to generate sources
 */

const fs = require('fs');
const path = require('path');
const prettier = require('prettier');
const prettierConfig = require('../.prettierrc');

const AllSources = require('../src/sources/sources.json');

const getLanguageCode = language => {
  const codes = {
    English: 'en',
    Arabic: 'ar',
    French: 'fr',
    Turkish: 'tr',
    Chinese: 'cn',
    Japanese: 'jp',
    Vietnamese: 'vi',
    Korean: 'kr',
  };

  return codes[language];
};

try {
  if (process.argv[2]) {
    const source = JSON.parse(
      fs.readFileSync(path.resolve(process.cwd(), process.argv[2]), 'utf8'),
    );

    const isDuplicate = AllSources.some(
      item => item.sourceId === source.sourceId,
    );

    const content = `
    import * as cheerio from 'cheerio';
    import { isUrlAbsolute } from '../../utils/isAbsoluteUrl';
    ${
      source?.options?.imports?.toaster
        ? "import { showToast } from '../../hooks/showToast'"
        : ''
    };
    import {
      SourceChapter,
      SourceChapterItem,
      SourceNovel,
      SourceNovelItem,
    } from '../types';
    
    const sourceId = ${source.sourceId};
    const sourceName = "${source.sourceName}";
    const baseUrl = "${source.baseUrl}";
    
    ${
      !source.popularNovels.url.includes('${page}')
        ? '// eslint-disable-next-line @typescript-eslint/no-unused-vars'
        : ''
    }
    const popularNovels = async (page: number) => {
      const url = \`${source.popularNovels.url}\`;
    
      const result = await fetch(url);
      const body = await result.text();
    
      const loadedCheerio = cheerio.load(body);
    
      const novels: SourceNovelItem[] = [];
    
      loadedCheerio('${
        source.popularNovels.selectors.novelList
      }').each(function () {
        let novelUrl = loadedCheerio(this).find('${
          source.popularNovels.selectors.novelUrl
        }').attr('href');
    
        if (novelUrl && !isUrlAbsolute(novelUrl)) {
          novelUrl = baseUrl + novelUrl;
        }
    
        if (novelUrl) {
          const novelName = loadedCheerio(this).find('${
            source.popularNovels.selectors.novelName
          }').text().trim();
          let novelCover = loadedCheerio(this).find('${
            source.popularNovels.selectors.novelCover
          }').attr('src');
    
          if (novelCover && !isUrlAbsolute(novelCover)) {
            novelCover = baseUrl + novelCover;
          }
    
          const novel = {
            sourceId,
            novelUrl,
            novelName,
            novelCover,
          };
    
          novels.push(novel);
        }
      });
    
      return { novels };
    
    const parseNovelAndChapters = async (novelUrl: string) => {
      const url = novelUrl;
    
      const result = await fetch(url);
      const body = await result.text();
    
      let loadedCheerio = cheerio.load(body);
    
      const novel: SourceNovel = {
        sourceId,
        sourceName,
        url: novelUrl,
        novelUrl,
        chapters: [],
      };
    
      novel.novelName = loadedCheerio('${
        source.novelAndChapters.selectors.novelName
      }').text();
    
      let novelCover = loadedCheerio('${
        source.novelAndChapters.selectors.novelCover
      }').attr('src');
    
      novel.novelCover = novelCover
        ? isUrlAbsolute(novelCover)
          ? novelCover
          : baseUrl + novelCover
        : undefined;
    
      novel.summary = loadedCheerio('${
        source.novelAndChapters.selectors.summary
      }').text().trim();
    
      novel.author = loadedCheerio('${
        source.novelAndChapters.selectors.author
      }').text().trim();
    
      novel.genre = loadedCheerio('${
        source.novelAndChapters.selectors.genre
      }').text().trim()${source?.options?.status?.replace || ''};

    
      novel.status = loadedCheerio('${
        source.novelAndChapters.selectors.status
      }').text().trim();
    
      loadedCheerio('${
        source.novelAndChapters.selectors.chapterList
      }').each(function () {
        let chapterUrl = loadedCheerio(this).find('a').attr('href');
    
        if (chapterUrl && !isUrlAbsolute(chapterUrl)) {
          chapterUrl = baseUrl + chapterUrl;
        }
    
        if (chapterUrl) {
          const chapterName = loadedCheerio(this).find('${
            source.novelAndChapters.selectors.chapterName
          }').text().trim();
          const releaseDate = ${
            source.novelAndChapters.selectors.releaseDate
              ? `loadedCheerio(this).find('${source.novelAndChapters.selectors.releaseDate}').text();`
              : null
          }
    
          const chapter: SourceChapterItem = {
            chapterName,
            releaseDate,
            chapterUrl,
          };
    
          novel.chapters?.push(chapter);
        }
      });
    
      return novel;
    };
    
    const parseChapter = async (novelUrl: string, chapterUrl: string) => {
      const url = chapterUrl;
    
      const result = await fetch(url);
      const body = await result.text();
    
      const loadedCheerio = cheerio.load(body);
    
      const chapterName = loadedCheerio('${
        source.chapter.selectors.chapterName
      }').text();
      const chapterText = loadedCheerio('${
        source.chapter.selectors.chapterText
      }').html() || '';
    
      const chapter: SourceChapter = {
        sourceId,
        novelUrl,
        chapterUrl,
        chapterName,
        chapterText,
      };
    
      return chapter;
    };
    
    ${
      !source.search.url.includes('${searchTerm}')
        ? '// eslint-disable-next-line @typescript-eslint/no-unused-vars'
        : ''
    }
    const searchNovels = async (searchTerm: string) => {
    ${
      source.search
        ? `
      const url = \`${source.search.url}\`;
    
      const result = await fetch(url);
      const body = await result.text();
    
      const loadedCheerio = cheerio.load(body);
    
      const novels: SourceNovelItem[] = [];
    
      loadedCheerio('${source.search.selectors.novelList}').each(function () {
        let novelUrl = loadedCheerio(this).find('${source.search.selectors.novelUrl}').attr('href');

        if (novelUrl && !isUrlAbsolute(novelUrl)) {
          novelUrl = baseUrl + novelUrl;
        }
  
        if (novelUrl) {
          const novelName = loadedCheerio(this).find('${source.search.selectors.novelName}').text();
          let novelCover = loadedCheerio(this).find('${source.search.selectors.novelCover}').attr('src');
    
          if (novelCover && !isUrlAbsolute(novelCover)) {
            novelCover = baseUrl + novelCover;
          }
    
          novels.push({
            sourceId,
            novelUrl,
            novelName,
            novelCover,
          });
        }
      });
    
      return novels;`
        : `
      showToast('${source.sourceName}: Search is not available in this source')
      
      return [];
      `
    }
    };
    
    const ${source.sourceName}Scraper = {
      popularNovels,
      parseNovelAndChapters,
      parseChapter,
      searchNovels,
    };
    
    export default ${source.sourceName}Scraper;
    
    `;

    const formatContent = prettier.format(content, {
      parser: 'typescript',
      ...prettierConfig,
    });

    fs.writeFile(
      path.resolve(
        process.cwd(),
        `src/sources/${getLanguageCode(source.lang)}/${source.sourceName}.ts`,
      ),
      formatContent,
      error => {
        if (error) {
          console.log(error);
        }
      },
    );

    if (!isDuplicate) {
      let allSourcesContent = AllSources;

      allSourcesContent.push({
        sourceId: source.sourceId,
        sourceName: source.sourceName,
        icon: `https://github.com/LNReader/lnreader-sources/blob/main/src/${getLanguageCode(
          source.lang,
        )}/${source.sourceName.toLowerCase()}/icon.png?raw=true`,
        url: source.baseUrl,
        lang: source.lang || 'English',
      });

      allSourcesContent = prettier.format(JSON.stringify(allSourcesContent), {
        parser: 'json',
        ...prettierConfig,
      });

      fs.writeFile(
        path.resolve(process.cwd(), 'src/sources/sources.json'),
        allSourcesContent,
        error => {
          if (error) {
            console.log(error);
          }
        },
      );
    } else {
      console.log('Source with same id already exists');
    }
  } else {
    console.log('Template path not provided');
  }
} catch (e) {
  console.log(e);
}