diff --git a/package-lock.json b/package-lock.json index 2a46b85..c10ab65 100644 --- a/package-lock.json +++ b/package-lock.json @@ -575,11 +575,21 @@ "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" }, + "detect-indent": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/detect-indent/-/detect-indent-6.0.0.tgz", + "integrity": "sha512-oSyFlqaTHCItVRGK5RmrmjB+CmaMOW7IaNA/kdxqhoa6d17j/5ce9O9eWXmV/KEdRwqpQA+Vqe8a8Bsybu4YnA==" + }, "devtools-protocol": { "version": "0.0.869402", "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.869402.tgz", "integrity": "sha512-VvlVYY+VDJe639yHs5PHISzdWTLL3Aw8rO4cvUtwvoxFd6FHbE4OpHHcde52M6096uYYazAmd4l0o5VuFRO2WA==" }, + "docopt": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/docopt/-/docopt-0.6.2.tgz", + "integrity": "sha1-so6eIiDaXsSffqW7JKR3h0Be6xE=" + }, "dom-serializer": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.2.0.tgz", @@ -613,6 +623,16 @@ "domhandler": "^4.0.0" } }, + "dot-json": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/dot-json/-/dot-json-1.2.2.tgz", + "integrity": "sha512-AKL+GsO4wSEU4LU+fAk/PqN4nQ6PB1vT3HpMiZous9xCzK5S0kh4DzfUY0EfU67jsIXLlu0ty71659N9Nmg+Tw==", + "requires": { + "detect-indent": "~6.0.0", + "docopt": "~0.6.2", + "underscore-keypath": "~0.0.22" + } + }, "dot-prop": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.3.0.tgz", @@ -1416,6 +1436,11 @@ "p-locate": "^4.1.0" } }, + "lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=" + }, "lowercase-keys": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", @@ -2223,6 +2248,19 @@ "debug": "^2.2.0" } }, + "underscore": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.1.tgz", + "integrity": "sha512-hzSoAVtJF+3ZtiFX0VgfFPHEDRm7Y/QPjGyNo4TVdnDTdft3tr8hEkD25a1jC+TjTuE7tkHGKkhwCgs9dgBB2g==" + }, + "underscore-keypath": { + "version": "0.0.22", + "resolved": "https://registry.npmjs.org/underscore-keypath/-/underscore-keypath-0.0.22.tgz", + "integrity": "sha1-SKUoOSu278QkvhyqVtpLX6zPJk0=", + "requires": { + "underscore": "*" + } + }, "unique-string": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-2.0.0.tgz", @@ -2275,6 +2313,15 @@ "prepend-http": "^2.0.0" } }, + "user-agents": { + "version": "1.0.654", + "resolved": "https://registry.npmjs.org/user-agents/-/user-agents-1.0.654.tgz", + "integrity": "sha512-auE1AmkOkjYbZvMFbakf3Je4dr/xEdjXCsJXOzz0GiUXKwScOt+/hp1s9gutRPgJnPFq4xCGM0LrfWaVWIGBAQ==", + "requires": { + "dot-json": "^1.2.2", + "lodash.clonedeep": "^4.5.0" + } + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/package.json b/package.json index 8e5e36d..d220ecc 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "html-to-text": "^7.1.1", "node-fetch": "^2.6.1", "puppeteer": "^9.0.0", - "request": "^2.88.2" + "request": "^2.88.2", + "user-agents": "^1.0.654" }, "devDependencies": { "nodemon": "^2.0.7" diff --git a/src/helper.js b/src/helper.js index 05568cb..d569e3f 100644 --- a/src/helper.js +++ b/src/helper.js @@ -1,9 +1,12 @@ const chromium = require("chrome-aws-lambda"); const puppeteer = require("puppeteer"); +const UserAgent = require("user-agents"); + +const userAgent = new UserAgent(); const scraper = async (url) => { const browser = await puppeteer.launch({ - args: [...chromium.args, "--hide-scrollbars", "--disable-web-security"], + args: [...chromium.args, "--hide-scrollbars", "--disable-web-security", "--user-agent=" + userAgent + ""], defaultViewport: chromium.defaultViewport, executablePath: await chromium.executablePath, headless: true,