Add user agent (#19)

Add user agent to try circumventing DDoS protection.

package-lock.json

@@ -575,11 +575,21 @@
       "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
       "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA="
     },
+    "detect-indent": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/detect-indent/-/detect-indent-6.0.0.tgz",
+      "integrity": "sha512-oSyFlqaTHCItVRGK5RmrmjB+CmaMOW7IaNA/kdxqhoa6d17j/5ce9O9eWXmV/KEdRwqpQA+Vqe8a8Bsybu4YnA=="
+    },
     "devtools-protocol": {
       "version": "0.0.869402",
       "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.869402.tgz",
       "integrity": "sha512-VvlVYY+VDJe639yHs5PHISzdWTLL3Aw8rO4cvUtwvoxFd6FHbE4OpHHcde52M6096uYYazAmd4l0o5VuFRO2WA=="
     },
+    "docopt": {
+      "version": "0.6.2",
+      "resolved": "https://registry.npmjs.org/docopt/-/docopt-0.6.2.tgz",
+      "integrity": "sha1-so6eIiDaXsSffqW7JKR3h0Be6xE="
+    },
     "dom-serializer": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.2.0.tgz",
@@ -613,6 +623,16 @@
         "domhandler": "^4.0.0"
       }
     },
+    "dot-json": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/dot-json/-/dot-json-1.2.2.tgz",
+      "integrity": "sha512-AKL+GsO4wSEU4LU+fAk/PqN4nQ6PB1vT3HpMiZous9xCzK5S0kh4DzfUY0EfU67jsIXLlu0ty71659N9Nmg+Tw==",
+      "requires": {
+        "detect-indent": "~6.0.0",
+        "docopt": "~0.6.2",
+        "underscore-keypath": "~0.0.22"
+      }
+    },
     "dot-prop": {
       "version": "5.3.0",
       "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.3.0.tgz",
@@ -1416,6 +1436,11 @@
         "p-locate": "^4.1.0"
       }
     },
+    "lodash.clonedeep": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
+      "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8="
+    },
     "lowercase-keys": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz",
@@ -2223,6 +2248,19 @@
         "debug": "^2.2.0"
       }
     },
+    "underscore": {
+      "version": "1.13.1",
+      "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.1.tgz",
+      "integrity": "sha512-hzSoAVtJF+3ZtiFX0VgfFPHEDRm7Y/QPjGyNo4TVdnDTdft3tr8hEkD25a1jC+TjTuE7tkHGKkhwCgs9dgBB2g=="
+    },
+    "underscore-keypath": {
+      "version": "0.0.22",
+      "resolved": "https://registry.npmjs.org/underscore-keypath/-/underscore-keypath-0.0.22.tgz",
+      "integrity": "sha1-SKUoOSu278QkvhyqVtpLX6zPJk0=",
+      "requires": {
+        "underscore": "*"
+      }
+    },
     "unique-string": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-2.0.0.tgz",
@@ -2275,6 +2313,15 @@
         "prepend-http": "^2.0.0"
       }
     },
+    "user-agents": {
+      "version": "1.0.654",
+      "resolved": "https://registry.npmjs.org/user-agents/-/user-agents-1.0.654.tgz",
+      "integrity": "sha512-auE1AmkOkjYbZvMFbakf3Je4dr/xEdjXCsJXOzz0GiUXKwScOt+/hp1s9gutRPgJnPFq4xCGM0LrfWaVWIGBAQ==",
+      "requires": {
+        "dot-json": "^1.2.2",
+        "lodash.clonedeep": "^4.5.0"
+      }
+    },
     "util-deprecate": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",

package.json

@@ -16,7 +16,8 @@
         "html-to-text": "^7.1.1",
         "node-fetch": "^2.6.1",
         "puppeteer": "^9.0.0",
-        "request": "^2.88.2"
+        "request": "^2.88.2",
+        "user-agents": "^1.0.654"
     },
     "devDependencies": {
         "nodemon": "^2.0.7"

src/helper.js

@@ -1,9 +1,12 @@
 const chromium = require("chrome-aws-lambda");
 const puppeteer = require("puppeteer");
+const UserAgent = require("user-agents");
+
+const userAgent = new UserAgent();
 
 const scraper = async (url) => {
     const browser = await puppeteer.launch({
-        args: [...chromium.args, "--hide-scrollbars", "--disable-web-security"],
+        args: [...chromium.args, "--hide-scrollbars", "--disable-web-security", "--user-agent=" + userAgent + ""],
         defaultViewport: chromium.defaultViewport,
         executablePath: await chromium.executablePath,
         headless: true,