Lead Generation with AI Web Scraping: Extract Contacts at Scale

from scrapegraph_py import Client
 
# Initialize the client with your API key
client = Client(api_key="your-api-key-here")
 
# SmartScraper request to extract company details
response = client.smartscraper(
    website_url="https://www.hubspot.com/company/about",
user_prompt = (
        "Extract company name, description, industry, headquarters location, employee
            count, founded year, and all contact information including email, phone, and
             social media links"
    )
)
 
print("Result:", response)
 
 

{
  "company_name": "HubSpot",
  "description": "HubSpot is a CRM platform that helps companies grow better",
  "industry": "Software / SaaS",
  "headquarters": "Cambridge, Massachusetts",
  "employee_count": "7,000+",
  "founded_year": "2006",
  "social_media": {
    "linkedin": "https://linkedin.com/company/hubspot",
    "twitter": "https://twitter.com/HubSpot"
  }
}

from scrapegraph_py import Client
from pydantic import BaseModel, Field
from typing import Optional, List
 
class SocialLinks(BaseModel):
    linkedin: Optional[str] = Field(description="LinkedIn company URL")
    twitter: Optional[str] = Field(description="Twitter/X profile URL")
    facebook: Optional[str] = Field(description="Facebook page URL")
 
class CompanyLead(BaseModel):
    company_name: str = Field(description="Official company name")
    website: Optional[str] = Field(description="Company website URL")
    description: str = Field(description="What the company does")
    industry: str = Field(description="Primary industry")
    headquarters: str = Field(description="HQ city and state/country")
    employee_count: Optional[str] = Field(description="Approximate employee count")
    founded_year: Optional[str] = Field(description="Year founded")
    contact_email: Optional[str] = Field(description="General contact email")
    phone: Optional[str] = Field(description="Main phone number")
    social_media: Optional[SocialLinks] = Field(description="Social media profiles")
 
client = Client(api_key="your-api-key-here")
 
response = client.smartscraper(
    website_url="https://www.hubspot.com/company/about",
    user_prompt="Extract complete company information for lead generation",
    output_schema=CompanyLead
)
 
lead = CompanyLead(**response["result"])
print(f"Found: {lead.company_name} in {lead.industry}")

from scrapegraph_py import Client
 
# Initialize the client
client = Client(api_key="your-api-key-here")
 
# SearchScraper request to find companies
response = client.searchscraper(
    user_prompt="Find B2B SaaS companies in New York with 50-200 employees, extract
        company name, website, and description",
    num_results=10
)
 
print("Result:", response)
 

from scrapegraph_py import Client
 
client = Client(api_key="your-api-key-here")
 
# Extract multiple businesses from a directory page
response = client.smartscraper(
    website_url="https://clutch.co/agencies/digital-marketing/new-york",
user_prompt = (
        "Extract all businesses listed: company name, phone number, address, website
            URL, rating, and number of reviews"
    )
)
 
print("Result:", response)
 
 

target_criteria = {
    "industries": ["SaaS", "Marketing Agency", "E-commerce"],
    "company_size": "10-500 employees",
    "locations": ["United States", "United Kingdom", "Canada"],
    "technologies": ["Salesforce", "HubSpot", "Shopify"]
}

data_sources = {
    "company_directories": [
        "https://www.crunchbase.com/",
        "https://www.g2.com/",
        "https://clutch.co/"
    ],
    "job_boards": [
        "https://www.indeed.com/",
        "https://www.linkedin.com/jobs/"
    ],
    "industry_lists": [
        "https://www.inc.com/inc5000",
        "https://www.forbes.com/lists/"
    ]
}

from scrapegraph_py import Client
 
client = Client(api_key="your-api-key-here")
 
def extract_company_leads(directory_url):
    # First, get list of companies
    companies = client.smartscraper(
        website_url=directory_url,
        user_prompt="Extract all company names and their profile URLs from this page"
    )
 
    leads = []
 
    # Then enrich each company
    for company in companies.get("companies", []):
        if company.get("profile_url"):
            details = client.smartscraper(
                website_url=company["profile_url"],
                user_prompt="""Extract:
                - Company name
                - Website URL
                - Description
                - Industry
                - Employee count
                - Headquarters location
                - Founded year
                - Key executives with their titles
                - Contact email
                - Phone number
                - Social media profiles
                """
            )
            leads.append(details)
 
    return leads

def find_decision_makers(company_website):
    # Look for team/about pages
    response = client.smartscraper(
        website_url=f"{company_website}/about",
        user_prompt="""Find all team members, especially:
        - CEO, Founder, Owner
        - VP of Sales, Sales Director
        - VP of Marketing, CMO
        - CTO, VP of Engineering
 
        Extract their names, titles, and any contact information or LinkedIn profiles"""
    )
 
    return response

import csv
 
def export_to_csv(leads, filename="leads.csv"):
    if not leads:
        return
 
    keys = leads[0].keys()
 
    with open(filename, 'w', newline='') as f:
        writer = csv.DictWriter(f, fieldnames=keys)
        writer.writeheader()
        writer.writerows(leads)
 
# Export for CRM import
export_to_csv(leads, "new_leads.csv")