From b5cbd514a29f474b175a4f72c151e96fb78301e7 Mon Sep 17 00:00:00 2001
From: Rajarshee Chatterjee <rajarshee.adm@gmail.com>
Date: Sun, 16 May 2021 20:51:24 +0530
Subject: [PATCH] Fix MTLNovel & LightNovelPub (#20)

* Test user-agent with fetch

* Use fetch.Headers

* Use puppeteer for MTLNovel & LightNovelPub

* Import scraper in LightNovelPub

* Update status
---
 README.md                                    |  2 +-
 sources.js                                   |  4 ++--
 src/en/lightnovelpub/LightNovelPubScraper.js | 11 +++++------
 src/en/mtlnovel/MTLNovelScraper.js           | 14 ++++++++++++--
 4 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index 47fe302..151dc0b 100644
--- a/README.md
+++ b/README.md
@@ -10,7 +10,7 @@ Sources for [LNReader](https://github.com/rajarsheechatterjee/lnreader)
 | ReadLightNovel  | :heavy_check_mark: |
 | FastNovel       |        :x:         |
 | ReadNovelFull   | :heavy_check_mark: |
-| MTLNovel        |        :x:         |
+| MTLNovel        | :heavy_check_mark: |
 | NovelHall       | :heavy_check_mark: |
 | WuxiaWorld      | :heavy_check_mark: |
 | NovelFull       | :heavy_check_mark: |
diff --git a/sources.js b/sources.js
index af7e851..59da9da 100644
--- a/sources.js
+++ b/sources.js
@@ -42,7 +42,7 @@ extensions = [
         sourceCover:
             "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/mtlnovel/icon.png?raw=true",
         sourceLanguage: "English",
-        status: 0,
+        status: 1,
     },
     {
         sourceId: 6,
@@ -132,7 +132,7 @@ extensions = [
         sourceCover:
             "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/lightnovelpub/icon.png?raw=true",
         sourceLanguage: "English",
-        status: 0,
+        status: 1,
     },
 ];
 
diff --git a/src/en/lightnovelpub/LightNovelPubScraper.js b/src/en/lightnovelpub/LightNovelPubScraper.js
index af32e82..c34944d 100644
--- a/src/en/lightnovelpub/LightNovelPubScraper.js
+++ b/src/en/lightnovelpub/LightNovelPubScraper.js
@@ -1,14 +1,14 @@
 const cheerio = require("cheerio");
 const fetch = require("node-fetch");
 const { htmlToText } = require("html-to-text");
+const { scraper } = require("../../helper");
 
 const baseUrl = "https://www.lightnovelpub.com/";
 
 const novelsScraper = async (req, res) => {
     let url = baseUrl + "browse/all/popular/all/1";
 
-    const result = await fetch(url);
-    const body = await result.text();
+    const body = await scraper(url);
 
     $ = cheerio.load(body);
 
@@ -47,8 +47,7 @@ const novelScraper = async (req, res) => {
     const novelUrl = req.params.novelUrl;
     const url = `${baseUrl}novel/${novelUrl}/`;
 
-    const result = await fetch(url);
-    const body = await result.text();
+    const body = await scraper(url);
 
     $ = cheerio.load(body);
 
@@ -121,8 +120,7 @@ const chapterScraper = async (req, res) => {
 
     const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`;
 
-    const result = await fetch(url);
-    const body = await result.text();
+    const body = await scraper(url);
 
     $ = cheerio.load(body);
 
@@ -169,6 +167,7 @@ const searchScraper = async (req, res) => {
 
     const result = await fetch(url);
     const body = await result.json();
+
     const data = body.resultview;
 
     $ = cheerio.load(data);
diff --git a/src/en/mtlnovel/MTLNovelScraper.js b/src/en/mtlnovel/MTLNovelScraper.js
index 6672fb9..52b3a32 100644
--- a/src/en/mtlnovel/MTLNovelScraper.js
+++ b/src/en/mtlnovel/MTLNovelScraper.js
@@ -1,5 +1,7 @@
 const cheerio = require("cheerio");
 const fetch = require("node-fetch");
+const UserAgent = require("user-agents");
+
 const { scraper } = require("../../helper");
 
 const baseUrl = "https://mtlnovel.com";
@@ -124,8 +126,7 @@ const chapterScraper = async (req, res) => {
 
     const url = `${baseUrl}/${novelUrl}/${chapterUrl}/`;
 
-    const result = await fetch(url);
-    const body = await result.text();
+    const body = await scraper(url);
 
     $ = cheerio.load(body);
 
@@ -158,9 +159,18 @@ const chapterScraper = async (req, res) => {
 const searchScraper = async (req, res) => {
     const searchTerm = req.query.s;
 
+    const userAgent = new UserAgent();
+
+    let headers = new fetch.Headers({
+        Accept: "application/json",
+        "Content-Type": "application/json",
+        "user-agent": userAgent,
+    });
+
     const result = await fetch(baseUrl, {
         method: "POST",
         body: { s: searchTerm },
+        headers: headers,
     });
     const body = await result.text();