Fix MTLNovel & LightNovelPub (#20)

* Test user-agent with fetch

* Use fetch.Headers

* Use puppeteer for MTLNovel & LightNovelPub

* Import scraper in LightNovelPub

* Update status

README.md

@@ -10,7 +10,7 @@ Sources for [LNReader](https://github.com/rajarsheechatterjee/lnreader)
 | ReadLightNovel  | :heavy_check_mark: |
 | FastNovel       |        :x:         |
 | ReadNovelFull   | :heavy_check_mark: |
-| MTLNovel        |        :x:         |
+| MTLNovel        | :heavy_check_mark: |
 | NovelHall       | :heavy_check_mark: |
 | WuxiaWorld      | :heavy_check_mark: |
 | NovelFull       | :heavy_check_mark: |

sources.js

@@ -42,7 +42,7 @@ extensions = [
         sourceCover:
             "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/mtlnovel/icon.png?raw=true",
         sourceLanguage: "English",
-        status: 0,
+        status: 1,
     },
     {
         sourceId: 6,
@@ -132,7 +132,7 @@ extensions = [
         sourceCover:
             "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/lightnovelpub/icon.png?raw=true",
         sourceLanguage: "English",
-        status: 0,
+        status: 1,
     },
 ];
 

src/en/lightnovelpub/LightNovelPubScraper.js

@@ -1,14 +1,14 @@
 const cheerio = require("cheerio");
 const fetch = require("node-fetch");
 const { htmlToText } = require("html-to-text");
+const { scraper } = require("../../helper");
 
 const baseUrl = "https://www.lightnovelpub.com/";
 
 const novelsScraper = async (req, res) => {
     let url = baseUrl + "browse/all/popular/all/1";
 
-    const result = await fetch(url);
-    const body = await result.text();
+    const body = await scraper(url);
 
     $ = cheerio.load(body);
 
@@ -47,8 +47,7 @@ const novelScraper = async (req, res) => {
     const novelUrl = req.params.novelUrl;
     const url = `${baseUrl}novel/${novelUrl}/`;
 
-    const result = await fetch(url);
-    const body = await result.text();
+    const body = await scraper(url);
 
     $ = cheerio.load(body);
 
@@ -121,8 +120,7 @@ const chapterScraper = async (req, res) => {
 
     const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`;
 
-    const result = await fetch(url);
-    const body = await result.text();
+    const body = await scraper(url);
 
     $ = cheerio.load(body);
 
@@ -169,6 +167,7 @@ const searchScraper = async (req, res) => {
 
     const result = await fetch(url);
     const body = await result.json();
+
     const data = body.resultview;
 
     $ = cheerio.load(data);

src/en/mtlnovel/MTLNovelScraper.js

@@ -1,5 +1,7 @@
 const cheerio = require("cheerio");
 const fetch = require("node-fetch");
+const UserAgent = require("user-agents");
+
 const { scraper } = require("../../helper");
 
 const baseUrl = "https://mtlnovel.com";
@@ -124,8 +126,7 @@ const chapterScraper = async (req, res) => {
 
     const url = `${baseUrl}/${novelUrl}/${chapterUrl}/`;
 
-    const result = await fetch(url);
-    const body = await result.text();
+    const body = await scraper(url);
 
     $ = cheerio.load(body);
 
@@ -158,9 +159,18 @@ const chapterScraper = async (req, res) => {
 const searchScraper = async (req, res) => {
     const searchTerm = req.query.s;
 
+    const userAgent = new UserAgent();
+
+    let headers = new fetch.Headers({
+        Accept: "application/json",
+        "Content-Type": "application/json",
+        "user-agent": userAgent,
+    });
+
     const result = await fetch(baseUrl, {
         method: "POST",
         body: { s: searchTerm },
+        headers: headers,
     });
     const body = await result.text();