Market Research with AI Web Scraping: Build Your Research Dashboard

from scrapegraph_py import Client
 
# Initialize the client with your API key
client = Client(api_key="your-api-key-here")
 
# SmartScraper request to extract reviews
response = client.smartscraper(
    website_url="https://www.amazon.com/product-reviews/B09V3KXJPB",
    user_prompt="""Extract all reviews including:
    - Reviewer name
    - Rating (stars)
    - Review title
    - Review text
    - Review date
    - Verified purchase status
    - Helpful votes count
    """
)
 
print("Result:", response)

{
  "reviews": [
    {
      "reviewer": "John D.",
      "rating": 5,
      "title": "Best noise cancellation ever",
      "text": "These are amazing headphones...",
      "date": "2024-12-15",
      "verified_purchase": true,
      "helpful_votes": 42
    }
  ]
}

from scrapegraph_py import Client
from pydantic import BaseModel, Field
from typing import List, Optional
 
class Review(BaseModel):
    reviewer: str = Field(description="Reviewer name or username")
    rating: int = Field(description="Star rating 1-5")
    title: str = Field(description="Review headline")
    text: str = Field(description="Full review content")
    date: str = Field(description="Review date")
    verified_purchase: bool = Field(description="Whether purchase was verified")
    helpful_votes: Optional[int] = Field(description="Number of helpful votes")
 
class ReviewsResponse(BaseModel):
    product_name: str = Field(description="Name of the reviewed product")
    average_rating: float = Field(description="Average star rating")
    total_reviews: int = Field(description="Total number of reviews")
    reviews: List[Review] = Field(description="List of individual reviews")
 
client = Client(api_key="your-api-key-here")
 
response = client.smartscraper(
    website_url="https://www.amazon.com/product-reviews/B09V3KXJPB",
    user_prompt="Extract product reviews with ratings and sentiment",
    output_schema=ReviewsResponse
)
 
data = ReviewsResponse(**response["result"])
print(f"Average: {data.average_rating}/5 from {data.total_reviews} reviews")

from scrapegraph_py import Client
 
# Initialize the client
client = Client(api_key="your-api-key-here")
 
# SearchScraper request to find market trends
response = client.searchscraper(
    user_prompt="Find recent discussions and articles about CRM software trends in 2025,
         extract key insights and sentiment",
    num_results=5
)
 
print("Result:", response)
 

from scrapegraph_py import Client
 
# Initialize the client
client = Client(api_key="your-api-key-here")
 
# Monitor what customers say about competitors
def analyze_competitor_reviews(competitor_products):
    all_reviews = []
 
    for product in competitor_products:
        reviews = client.smartscraper(
            website_url=product["review_url"],
            user_prompt="Extract all reviews with ratings, dates, and full review text"
        )
 
        all_reviews.append({
            "competitor": product["company"],
            "product": product["name"],
            "reviews": reviews
        })
 
    return all_reviews

# Extract mentions and sentiment from forums
response = client.smartscraper(
    website_url="https://www.reddit.com/r/technology/search?q=your+product",
    user_prompt="""Extract discussions including:
    - Post title
    - Post content
    - Upvotes
    - Number of comments
    - Top comments and their sentiment
    - Date posted
    """
)

# Track competitor product launches and updates
def monitor_competitor_news(competitors):
    intelligence = []
 
    for competitor in competitors:
        # Check their blog/news page
        news = client.smartscraper(
            website_url=f"{competitor['website']}/blog",
user_prompt = (
                "Extract recent blog posts: titles, dates, summaries, and any product
                    announcements"
            )
        )
 
        # Check their pricing page
        pricing = client.smartscraper(
            website_url=f"{competitor['website']}/pricing",
user_prompt = (
                "Extract all pricing tiers, features included in each tier, and any
                    promotional offers"
            )
        )
 
        intelligence.append({
            "competitor": competitor["name"],
            "news": news,
            "pricing": pricing
        })
 
    return intelligence
 
 

research_categories = {
    "customer_sentiment": {
        "sources": ["amazon_reviews", "g2_reviews", "trustpilot"],
        "metrics": ["average_rating", "review_volume", "sentiment_score"]
    },
    "competitive_pricing": {
        "sources": ["competitor_websites", "comparison_sites"],
        "metrics": ["price_points", "discount_frequency", "feature_comparison"]
    },
    "market_trends": {
        "sources": ["industry_blogs", "news_sites", "social_media"],
        "metrics": ["trending_topics", "mention_volume", "share_of_voice"]
    }
}

from datetime import datetime
import schedule
 
def daily_market_research():
    timestamp = datetime.now().isoformat()
 
    # Collect customer sentiment
    sentiment_data = collect_reviews(review_sources)
 
    # Monitor competitors
    competitor_data = monitor_competitors(competitor_list)
 
    # Track market trends
    trend_data = analyze_trends(trend_sources)
 
    # Store in database
    save_research_data({
        "timestamp": timestamp,
        "sentiment": sentiment_data,
        "competitors": competitor_data,
        "trends": trend_data
    })
 
# Run daily
schedule.every().day.at("06:00").do(daily_market_research)

def calculate_market_metrics(data):
    metrics = {}
 
    # Average rating across all review sources
    all_ratings = []
    for source in data["sentiment"]:
        ratings = [r["rating"] for r in source.get("reviews", [])]
        all_ratings.extend(ratings)
 
    metrics["average_rating"] = sum(all_ratings) / len(all_ratings) if all_ratings else
        0
 
    # Review volume trend
    metrics["review_count"] = len(all_ratings)
 
    # Competitive price position
    prices = [c["pricing"]["base_price"] for c in data["competitors"]]
    metrics["price_rank"] = sorted(prices).index(your_price) + 1
 
    return metrics