AI Agent Tool: Give Your Agents Fast Web Access

from scrapegraph_py import Client
from langchain.agents import Tool
 
# Initialize ScrapeGraphAI client
sgai = Client(api_key="your-api-key-here")
 
# Define scraping tool for LangChain agent
def scrape_website(url_and_prompt: str) -> str:
    """Scrape a website and extract information based on prompt"""
    parts = url_and_prompt.split("|")
    url = parts[0].strip()
    prompt = parts[1].strip() if len(parts) > 1 else "Extract the main content"
 
    result = sgai.smartscraper(
        website_url=url,
        user_prompt=prompt
    )
    return str(result)
 
scraping_tool = Tool(
    name="web_scraper",
    func=scrape_website,
description = (
        "Scrape a website to get current information. Input format: 'URL | what to
            extract'"
    )
)
 
 

from scrapegraph_py import Client
 
# Initialize the client
client = Client(api_key="your-api-key-here")
 
# SmartScraper request - extract structured data
response = client.smartscraper(
    website_url="https://news.ycombinator.com",
    user_prompt="Extract the top 10 stories with titles, points, and comment counts"
)
 
print("Result:", response)

from scrapegraph_py import Client
 
# Initialize the client
client = Client(api_key="your-api-key-here")
 
# SearchScraper request - search and extract
response = client.searchscraper(
    user_prompt="Find the latest news about OpenAI with article titles and summaries",
    num_results=5
)
 
print("Result:", response)

from scrapegraph_py import Client
from pydantic import BaseModel, Field
from typing import List, Optional
 
class NewsStory(BaseModel):
    title: str = Field(description="Article headline")
    points: int = Field(description="Upvote count")
    comments: int = Field(description="Number of comments")
    url: Optional[str] = Field(description="Link to article")
    author: Optional[str] = Field(description="Submitter username")
 
class HackerNewsResponse(BaseModel):
    stories: List[NewsStory] = Field(description="Top stories")
    scraped_at: str = Field(description="Timestamp of scrape")
 
client = Client(api_key="your-api-key-here")
 
response = client.smartscraper(
    website_url="https://news.ycombinator.com",
    user_prompt="Extract the top 10 stories",
    output_schema=HackerNewsResponse
)
 
data = HackerNewsResponse(**response["result"])
for story in data.stories:
    print(f"{story.title} ({story.points} points)")

from langchain.agents import initialize_agent, AgentType
from langchain.llms import OpenAI
from scrapegraph_py import Client
 
sgai = Client(api_key="your-api-key-here")
 
# Define tools
tools = [
    Tool(
        name="smart_scraper",
        func=lambda x: sgai.smartscraper(
            website_url=x.split("|")[0],
            user_prompt=x.split("|")[1] if "|" in x else "Extract main content"
        ),
description = (
            "Extract structured data from any webpage. Format: 'url | what to extract'"
        )
    ),
    Tool(
        name="web_search",
        func=lambda x: sgai.searchscraper(user_prompt=x),
        description="Search the web and extract structured results"
    )
]
 
# Create agent
agent = initialize_agent(
    tools,
    OpenAI(temperature=0),
    agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION,
    verbose=True
)
 
# Now the agent can access the web!
result = agent.run("What is the current price of Bitcoin on Coinbase?")
 

from crewai import Agent, Task, Crew
from crewai_tools import tool
from scrapegraph_py import Client
 
sgai = Client(api_key="your-api-key-here")
 
@tool("Web Scraper")
def scrape_web(url: str, prompt: str) -> str:
    """Scrape a website and extract specific information"""
    result = sgai.smartscraper(website_url=url, user_prompt=prompt)
    return str(result)
 
# Create research agent with web access
researcher = Agent(
    role="Market Researcher",
    goal="Gather comprehensive market intelligence from the web",
    backstory="Expert at finding and analyzing online data",
    tools=[scrape_web],
    verbose=True
)
 
# Define research task
research_task = Task(
    description="Research current pricing for the top 5 CRM platforms",
    agent=researcher
)
 
# Run the crew
crew = Crew(agents=[researcher], tasks=[research_task])
result = crew.kickoff()

from scrapegraph_py import Client
 
class WebEnabledAgent:
    def __init__(self, api_key):
        self.sgai = Client(api_key=api_key)
        self.memory = []
 
    def scrape(self, url, prompt):
        """Extract data from a webpage"""
        return self.sgai.smartscraper(
            website_url=url,
            user_prompt=prompt
        )
 
    def search_and_scrape(self, query):
        """Search the web and extract structured results"""
        return self.sgai.searchscraper(user_prompt=query)
 
    def crawl_site(self, url, depth=2):
        """Crawl a website and extract data from multiple pages"""
        return self.sgai.crawl(
            website_url=url,
            max_depth=depth
        )
 
    def research(self, topic):
        """Autonomous research on a topic"""
        # Search for relevant sources
        search_results = self.search_and_scrape(
            f"Find authoritative sources about {topic}"
        )
 
        # Deep dive into top results
        detailed_info = []
        for result in search_results.get("results", [])[:3]:
            if result.get("url"):
                detail = self.scrape(
                    result["url"],
                    f"Extract detailed information about {topic}"
                )
                detailed_info.append(detail)
 
        return {
            "search_results": search_results,
            "detailed_analysis": detailed_info
        }

from functools import lru_cache
 
@lru_cache(maxsize=100)
def cached_scrape(url, prompt):
    return sgai.smartscraper(website_url=url, user_prompt=prompt)

# Good - specific
prompt = (
    "Extract product name, price in USD, availability status, and shipping estimate"
)
 
# Bad - vague
prompt = "Get product info"
 

def safe_scrape(url, prompt, retries=3):
    for attempt in range(retries):
        try:
            return sgai.smartscraper(website_url=url, user_prompt=prompt)
        except Exception as e:
            if attempt == retries - 1:
                return {"error": str(e)}
            time.sleep(2 ** attempt)  # Exponential backoff

import time
 
def batch_scrape(urls, prompt, delay=1):
    results = []
    for url in urls:
        result = sgai.smartscraper(website_url=url, user_prompt=prompt)
        results.append(result)
        time.sleep(delay)
    return results