Market Research with AI Web Scraping: Build Your Research Dashboard

from scrapegraph_py import Client
 
# Initialize the client with your API key
client = Client(api_key="your-api-key-here")
 
# SmartScraper request to extract reviews
response = client.smartscraper(
    website_url="https://www.amazon.com/product-reviews/B09V3KXJPB",
    user_prompt="""Extract all reviews including:
    - Reviewer name
    - Rating (stars)
    - Review title
    - Review text
    - Review date
    - Verified purchase status
    - Helpful votes count
    """
)
 
print("Result:", response)

{
  "reviews": [
    {
      "reviewer": "John D.",
      "rating": 5,
      "title": "Best noise cancellation ever",
      "text": "These are amazing headphones...",
      "date": "2024-12-15",
      "verified_purchase": true,
      "helpful_votes": 42
    }
  ]
}

from scrapegraph_py import Client
 
# Initialize the client
client = Client(api_key="your-api-key-here")
 
# SearchScraper request to find market trends
response = client.searchscraper(
    user_prompt="Find recent discussions and articles about CRM software trends in 2025, extract key insights and sentiment",
    num_results=5
)
 
print("Result:", response)

from scrapegraph_py import Client
 
# Initialize the client
client = Client(api_key="your-api-key-here")
 
# Monitor what customers say about competitors
def analyze_competitor_reviews(competitor_products):
    all_reviews = []
    
    for product in competitor_products:
        reviews = client.smartscraper(
            website_url=product["review_url"],
            user_prompt="Extract all reviews with ratings, dates, and full review text"
        )
        
        all_reviews.append({
            "competitor": product["company"],
            "product": product["name"],
            "reviews": reviews
        })
    
    return all_reviews

# Extract mentions and sentiment from forums
response = client.smartscraper(
    website_url="https://www.reddit.com/r/technology/search?q=your+product",
    user_prompt="""Extract discussions including:
    - Post title
    - Post content
    - Upvotes
    - Number of comments
    - Top comments and their sentiment
    - Date posted
    """
)

# Track competitor product launches and updates
def monitor_competitor_news(competitors):
    intelligence = []
    
    for competitor in competitors:
        # Check their blog/news page
        news = client.smartscraper(
            website_url=f"{competitor['website']}/blog",
            user_prompt="Extract recent blog posts: titles, dates, summaries, and any product announcements"
        )
        
        # Check their pricing page
        pricing = client.smartscraper(
            website_url=f"{competitor['website']}/pricing",
            user_prompt="Extract all pricing tiers, features included in each tier, and any promotional offers"
        )
        
        intelligence.append({
            "competitor": competitor["name"],
            "news": news,
            "pricing": pricing
        })
    
    return intelligence

research_categories = {
    "customer_sentiment": {
        "sources": ["amazon_reviews", "g2_reviews", "trustpilot"],
        "metrics": ["average_rating", "review_volume", "sentiment_score"]
    },
    "competitive_pricing": {
        "sources": ["competitor_websites", "comparison_sites"],
        "metrics": ["price_points", "discount_frequency", "feature_comparison"]
    },
    "market_trends": {
        "sources": ["industry_blogs", "news_sites", "social_media"],
        "metrics": ["trending_topics", "mention_volume", "share_of_voice"]
    }
}

from datetime import datetime
import schedule
 
def daily_market_research():
    timestamp = datetime.now().isoformat()
    
    # Collect customer sentiment
    sentiment_data = collect_reviews(review_sources)
    
    # Monitor competitors
    competitor_data = monitor_competitors(competitor_list)
    
    # Track market trends
    trend_data = analyze_trends(trend_sources)
    
    # Store in database
    save_research_data({
        "timestamp": timestamp,
        "sentiment": sentiment_data,
        "competitors": competitor_data,
        "trends": trend_data
    })
 
# Run daily
schedule.every().day.at("06:00").do(daily_market_research)

def calculate_market_metrics(data):
    metrics = {}
    
    # Average rating across all review sources
    all_ratings = []
    for source in data["sentiment"]:
        ratings = [r["rating"] for r in source.get("reviews", [])]
        all_ratings.extend(ratings)
    
    metrics["average_rating"] = sum(all_ratings) / len(all_ratings) if all_ratings else 0
    
    # Review volume trend
    metrics["review_count"] = len(all_ratings)
    
    # Competitive price position
    prices = [c["pricing"]["base_price"] for c in data["competitors"]]
    metrics["price_rank"] = sorted(prices).index(your_price) + 1
    
    return metrics