☰

Email Scraper

Scrape emails across multiple pages and save to CSV

// ==UserScript==
// @name         Email Scraper
// @namespace    http://tampermonkey.net/
// @version      0.2
// @description  Scrape emails across multiple pages and save to CSV
// @match        https://solicitors.lawsociety.org.uk/*
// @grant        GM_setValue
// @grant        GM_getValue
// @run-at       document-end
// @license MIT
// ==/UserScript==

// Install this in Tampermonkey by clicking [https://gist.githubusercontent.com/MarwanShehata/34acd0b3d2a4431cf8a93c9d3cdf2d41/raw/3050c898be3f8fc651eaeaca3e982cee1a84642f/Email-Scraper-gig.js] 
// or copying the code from [https://gist.github.com/MarwanShehata/34acd0b3d2a4431cf8a93c9d3cdf2d41].

// https://solicitors.lawsociety.org.uk/search/results?Pro=False&Page=1

(function() {
    'use strict';

    const DELAY_BETWEEN_PAGES = 6000;
    const MAX_PAGES = 80; // Upper limit to prevent infinite loops

    // Helper functions
    function getEmails() {
        const dataEmailsArray = document.querySelectorAll("a[data-email]");
        return [...dataEmailsArray].map(element => element.getAttribute('data-email'));
    }

    function getStoredEmails() {
        return GM_getValue('emails', []);
    }

    function setStoredEmails(emails) {
        GM_setValue('emails', emails);
    }

    function getCurrentPage() {
        return GM_getValue('currentPage', 1);
    }

    function setCurrentPage(page) {
        GM_setValue('currentPage', page);
    }

    function saveToCSV(emails) {
        const csvContent = "data:text/csv;charset=utf-8," + emails.map(email => `"${email}"`).join("\n");
        const encodedUri = encodeURI(csvContent);
        const link = document.createElement("a");
        link.setAttribute("href", encodedUri);
        link.setAttribute("download", "extracted_emails.csv");

        const button = document.createElement("div");
        button.innerHTML = "Download Extracted Emails";
        button.style.position = "fixed";
        button.style.top = "10px";
        button.style.left = "50%";
        button.style.transform = "translateX(-50%)";
        button.style.backgroundColor = "green";
        button.style.color = "white";
        button.style.padding = "10px 20px";
        button.style.zIndex = "9999";
        button.style.cursor = "pointer";

        button.addEventListener("click", () => {
            document.body.appendChild(link);
            link.click();
            document.body.removeChild(link);
            document.body.removeChild(button);
            // Reset storage after download
            GM_setValue('emails', []);
            GM_setValue('currentPage', 1);
        });

        document.body.appendChild(button);
        console.log(`Total emails extracted: ${emails.length}`);
    }

    // Main logic
    function scrapePage() {
        let currentPage = getCurrentPage();
        let allEmails = getStoredEmails();

        if (currentPage > MAX_PAGES) {
            console.log('Reached max pages, stopping.');
            saveToCSV(allEmails);
            return;
        }

        // Scrape current page
        const newEmails = getEmails();
        allEmails = [...new Set([...allEmails, ...newEmails])]; // Remove duplicates
        setStoredEmails(allEmails);
        console.log(`Page ${currentPage}: Extracted ${newEmails.length} emails, Total: ${allEmails.length}`);

        // Check for next page
        const nextPage = document.querySelector('li.next>a');
        if (nextPage && currentPage < MAX_PAGES) {
            setCurrentPage(currentPage + 1);
            setTimeout(() => nextPage.click(), DELAY_BETWEEN_PAGES);
        } else {
            console.log('No more pages or max reached, finishing.');
            saveToCSV(allEmails);
        }
    }

    // Start scraping
    scrapePage();
})();