AWS Lambda Web Scraper

const request = require('axios');
const MongoClient = require('mongodb').MongoClient;
const crypto = require('crypto');

// local .env files are loaded into process.env
require('dotenv').config({silent: false});

// load the MongoDB connection string from the .env file
const mongo_host = process.env.MONGO

// database and collection name
databaseName = 'scraper'
collectionName = 'rawdata'

// set of headers from which we will randomly select
let headers_list = [
	// Firefox 77 Mac
	 {
		"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:77.0) Gecko/20100101 Firefox/77.0",
		"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
		"Accept-Language": "en-US,en;q=0.5",
		"Referer": "https://www.google.com/",
		"DNT": "1",
		"Connection": "keep-alive",
		"Upgrade-Insecure-Requests": "1"
	},
	// Firefox 77 Windows
	{
		"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0",
		"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
		"Accept-Language": "en-US,en;q=0.5",
		"Accept-Encoding": "gzip, deflate, br",
		"Referer": "https://www.google.com/",
		"DNT": "1",
		"Connection": "keep-alive",
		"Upgrade-Insecure-Requests": "1"
	},
	// Chrome 83 Mac
	{
		"Connection": "keep-alive",
		"DNT": "1",
		"Upgrade-Insecure-Requests": "1",
		"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36",
		"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
		"Sec-Fetch-Site": "none",
		"Sec-Fetch-Mode": "navigate",
		"Sec-Fetch-Dest": "document",
		"Referer": "https://www.google.com/",
		"Accept-Encoding": "gzip, deflate, br",
		"Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8"
	},
	// Chrome 83 Windows 
	{
		"Connection": "keep-alive",
		"Upgrade-Insecure-Requests": "1",
		"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36",
		"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
		"Sec-Fetch-Site": "same-origin",
		"Sec-Fetch-Mode": "navigate",
		"Sec-Fetch-User": "?1",
		"Sec-Fetch-Dest": "document",
		"Referer": "https://www.google.com/",
		"Accept-Encoding": "gzip, deflate, br",
		"Accept-Language": "en-US,en;q=0.9"
	}
]

function isValidURL(string) {
	var res = string.match(/(http(s)?:\/\/.)?(www\.)?[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0-9@:%_\+.~#?&//=]*)/g);
	return (res !== null)
};

module.exports.scrape = async event => {
	// start by parsing the body assuming a POST statement with a JSON body
	let body = JSON.parse(event.body)

	// url is required
	if (!('url' in body)) {
		return {ok: 0, msg: 'Missing URL'}
	}

	// check the url is valid
	if (!isValidURL(body.url)) {
		return {ok: 0, msg: 'Invalid URL'}
	}

	let url = body.url
	let host = new URL(url)

	// randomly select a header
	let headers = headers_list[Math.floor(Math.random() * headers_list.length)]
	// the request should look like it is originating from the host
	headers['Host'] = host.host
	// referer is from the same domain, referers from google.com are often
	// redirected, which we want to avoid
	headers['Referer'] = host.origin

	// set json headers if we expect the response to be in json
	if ('json' in body) {
		headers['Accept'] = 'application/json, text/javascript, */*; q=0.01'
	}

	// set XMLHttpRequest header, which helps when calling private APIs that 
	// would typically be loaded by AJAX calls
	if ('ajax' in body) {
		headers['X-Requested-With'] = 'XMLHttpRequest'
	}

	// send a GET request with our headers
	const response = await request({
		'url': url,
		'method': 'get',
		'headers': headers,
	});

	if (response.status == 200) {
		// create a data object containing the response body and headers
		let date = new Date()
		let data = {
			'url': url,
			'url_hash': crypto.createHash('md5').update(url).digest("hex"),
			'host': host.host,
			'data': response.data,
			'processed': false,
			'scraped_at': date,
			'scraped_year': date.getFullYear(),
			'scraped_month': date.getMonth() + 1,
			'scraped_day': date.getDate(),
			'response_headers': response.headers,
			'request_headers': headers
		}

		// create a connection to the MongoDB
		const client = await MongoClient.connect(mongo_host, {useUnifiedTopology: true});
		// select the database
		const db = client.db(databaseName);
		// insert data into collection in database
		let r = await db.collection(collectionName).insertOne(data);
		// close the connection to MongoDB
		client.close();

		if (r.insertedCount == 1) {
			// return the newly created ObjectID if a new document was successfully inserted
			return {
				ok: 1,
				url: url,
				insertedId: r.insertedId,
			};
		}
	} else {
		return {
			ok: 0,
			url: url,
			status: response.status,
			msg: 'Bad response status'
		};
	}

	return {
		ok: 0,
		url: url
	};
};