Scraping

from pydantic import BaseModel
from notte_sdk import NotteClient
 
class HackerNewsPost(BaseModel):
    title: str
    url: str
    points: int
    author: str
 
class HackerNewsFeed(BaseModel):
    posts: list[HackerNewsPost]
 
client = NotteClient()
 
result = client.scrape(
    url="https://news.ycombinator.com",
    response_format=HackerNewsFeed,
    instructions="Extract the top 5 posts from the front page"
)
 
for i, post in enumerate(result.data.posts, 1):
    print(f"{i}. {post.points} - {post.title}")

client = NotteClient()
 
# Returns markdown content
markdown = client.scrape("https://example.com")

with client.Session() as session:
    session.execute(type="goto", url="https://example.com/login")
    session.execute(type="fill", selector="input[name='email']", value="user@example.com")
    session.execute(type="fill", selector="input[name='password']", value="password")
    session.execute(type="click", selector="button[type='submit']")
 
    session.execute(type="goto", url="https://example.com/dashboard")
 
    content = session.scrape()

from pydantic import BaseModel
 
class Product(BaseModel):
    name: str
    price: float
    description: str
 
client = NotteClient()
product = client.scrape(
    "https://example.com/product", response_format=Product, instructions="Extract the product details"
)
 
print(f"Name: {product.name}, Price: {product.price}")

result = client.scrape(
    "https://example.com/article", instructions="Extract the article title, author, and publication date"
)

class Article(BaseModel):
    title: str
    url: str
    summary: str
 
class ArticleList(BaseModel):
    articles: list[Article]
 
articles = client.scrape(
    "https://news.example.com", response_format=ArticleList, instructions="Extract all articles from the homepage"
)

class Address(BaseModel):
    street: str
    city: str
    country: str
 
class Company(BaseModel):
    name: str
    description: str
    address: Address
    employee_count: int | None
 
company = client.scrape(
    "https://example.com/about", response_format=Company, instructions="Extract company information including address"
)

images = client.scrape("https://example.com/gallery", only_images=True)
 
for image in images:
    print(f"URL: {image.url}")
    print(f"Description: {image.description}")

# Only main content (excludes navbars, footers, sidebars) - Default
markdown = client.scrape(url, only_main_content=True)
 
# Include all page content
markdown = client.scrape(url, only_main_content=False)

markdown = client.scrape(url, scrape_links=True)   # Include links (default)
markdown = client.scrape(url, scrape_images=True)  # Include images (off by default)

with client.Session() as session:
    content = session.scrape(selector="article.main-content")
    content = session.scrape(selector="#product-details")

class ProductInfo(BaseModel):
    name: str
    price: float
    rating: float | None
    reviews_count: int | None
 
urls = ["https://store.example.com/product/1", "https://store.example.com/product/2"]
 
products: list[ProductInfo] = []
for url in urls:
    data = client.scrape(url, response_format=ProductInfo)
    products.append(data)

class ResearchPaper(BaseModel):
    title: str
    authors: list[str]
    abstract: str
    publication_date: str | None
    citations: int | None
 
result = client.scrape("https://papers.example.com/paper/123", response_format=ResearchPaper)

# Good
instructions = "Extract the product name, price in USD, and availability status"
 
# Vague
instructions = "Get product info"

class Product(BaseModel):
    name: str
    price: float
    discount_price: float | None = None
    rating: float | None = None

with client.Session() as session:
    # Scrape only the main article, not comments or sidebar
    content = session.scrape(selector="article.main")