ScrapeGraphAI Tutorial: Master AI-Powered Web Scraping

pip install scrapegraph_py

from scrapegraph_py import Client
from scrapegraph_py.logger import sgai_logger
 
# Configure logging
sgai_logger.set_logging(level="INFO")
 
# Initialize client with your API key
sgai_client = Client(api_key="your-api-key")
 
try:
    # Make SmartScraper request
    response = sgai_client.smartscraper(
        website_url="https://example.com",
        user_prompt="Extract webpage information"
    )
 
    # Process and print results
    print(f"Request ID: {response['request_id']}")
    print(f"Result: {response['result']}")
    if response.get('reference_urls'):
        print(f"Reference URLs: {response['reference_urls']}")
 
finally:
    # Always close the client
    sgai_client.close()

from scrapegraph_py import Client
from scrapegraph_py.logger import sgai_logger
 
# Configure logging
sgai_logger.set_logging(level="INFO")
 
# Initialize client
sgai_client = Client(api_key="your-api-key")
 
try:
    # Extract product data
    response = sgai_client.smartscraper(
        website_url="https://example.com/product",
        user_prompt="Extract product name, price, and description"
    )
 
    # Process results
    print(f"Request ID: {response['request_id']}")
    print(f"Result: {response['result']}")
 
finally:
    sgai_client.close()

{
  "product_name": "Example Product",
  "price": "$29.99",
  "description": "This is an example product description."
}

from scrapegraph_py import Client
from scrapegraph_py.logger import sgai_logger
 
# Configure logging
sgai_logger.set_logging(level="INFO")
 
# Initialize client
sgai_client = Client(api_key="your-api-key")
 
try:
    # Make SearchScraper request
    response = sgai_client.searchscraper(
        user_prompt="Extract webpage information"
    )
 
    # Process results
    print(f"Request ID: {response['request_id']}")
    print(f"Result: {response['result']}")
    if response.get('reference_urls'):
        print(f"Reference URLs: {response['reference_urls']}")
 
finally:
    sgai_client.close()

from scrapegraph_py import Client
from scrapegraph_py.logger import sgai_logger
 
# Configure logging
sgai_logger.set_logging(level="INFO")
 
# Initialize client
sgai_client = Client(api_key="your-api-key")
 
try:
    # Search for healthcare AI information
    response = sgai_client.searchscraper(
        user_prompt="What are the benefits of AI in healthcare?"
    )
 
    # Process and display results
    print(f"Request ID: {response['request_id']}")
    print(f"Result: {response['result']}")
 
finally:
    sgai_client.close()

{
  "summary": "AI in healthcare offers numerous benefits, including improved diagnostic accuracy, personalized treatment plans, and efficient data management.",
  "details": [
    {
      "benefit": "Improved Diagnostic Accuracy",
      "description": "AI algorithms can analyze medical images and data to assist in accurate diagnosis."
    },
    {
      "benefit": "Personalized Treatment Plans",
      "description": "AI helps in tailoring treatment plans based on individual patient data."
    },
    {
      "benefit": "Efficient Data Management",
      "description": "AI streamlines the management and analysis of large volumes of healthcare data."
    }
  ],
  "reference_urls": [
    "https://example.com/ai-healthcare-benefits",
    "https://example.com/ai-medical-data"
  ]
}

from scrapegraph_py import Client
from scrapegraph_py.logger import sgai_logger
 
# Configure logging
sgai_logger.set_logging(level="INFO")
 
# Initialize client
sgai_client = Client(api_key="your-api-key")
 
try:
    # Convert webpage to markdown
    response = sgai_client.markdownify(
        website_url="https://example.com"
    )
 
    # Process results
    print(f"Request ID: {response['request_id']}")
    print(f"Result: {response['result']}")
 
finally:
    sgai_client.close()

from scrapegraph_py import Client
from scrapegraph_py.logger import sgai_logger
 
# Configure logging
sgai_logger.set_logging(level="INFO")
 
# Initialize client
sgai_client = Client(api_key="your-api-key")
 
try:
    # Convert article to markdown
    response = sgai_client.markdownify(
        website_url="https://example.com/article"
    )
 
    # Save markdown to file
    with open("article.md", "w") as f:
        f.write(response['result'])
 
finally:
    sgai_client.close()

# Title of the Article
 
Introduction paragraph...
 
## Subheading
 
Content under the subheading...
 
- Bullet point 1
- Bullet point 2
 
> A relevant quote from the article.
 
Conclusion paragraph...