Scraping E-Commerce Product Data for Business Intelligence

Business Uses for E-Commerce Data

E-commerce product data powers a wide range of business intelligence applications:

Data Sources and Access Methods

Major e-commerce platforms

Structured data on product pages

Many e-commerce sites include structured data (JSON-LD, Microdata, RDFa) in their HTML that is designed for search engines. This data is machine-readable and often includes:

Extracting structured data with Python

import requests
from bs4 import BeautifulSoup
import json

def extract_structured_data(url, headers=None):
    """Extract JSON-LD structured data from a product page."""
    if headers is None:
        headers = {
            'User-Agent': (
                'Mozilla/5.0 (compatible; '
                'DataResearchBot/1.0; '
                '+https://example.com/bot)'
            )
        }

    response = requests.get(url, headers=headers, timeout=30)
    response.raise_for_status()

    soup = BeautifulSoup(response.text, 'html.parser')
    structured_data = []

    # Find all JSON-LD blocks
    for script in soup.find_all(
        'script', type='application/ld+json'
    ):
        try:
            data = json.loads(script.string)
            structured_data.append(data)
        except (json.JSONDecodeError, TypeError):
            continue

    # Filter for product data
    products = []
    for data in structured_data:
        if isinstance(data, list):
            for item in data:
                if item.get('@type') == 'Product':
                    products.append(item)
        elif isinstance(data, dict):
            if data.get('@type') == 'Product':
                products.append(data)
            # Check @graph
            if '@graph' in data:
                for item in data['@graph']:
                    if item.get('@type') == 'Product':
                        products.append(item)

    return products


def parse_product(product_data):
    """Parse structured product data into a flat dictionary."""
    result = {
        'name': product_data.get('name', ''),
        'description': product_data.get('description', ''),
        'sku': product_data.get('sku', ''),
        'brand': '',
        'price': '',
        'currency': '',
        'availability': '',
        'rating': '',
        'review_count': '',
    }

    # Brand
    brand = product_data.get('brand', {})
    if isinstance(brand, dict):
        result['brand'] = brand.get('name', '')
    elif isinstance(brand, str):
        result['brand'] = brand

    # Offer
    offers = product_data.get('offers', {})
    if isinstance(offers, dict):
        result['price'] = offers.get(
            'price', offers.get('lowPrice', '')
        )
        result['currency'] = offers.get('priceCurrency', '')
        result['availability'] = offers.get('availability', '')
    elif isinstance(offers, list) and offers:
        result['price'] = offers[0].get('price', '')
        result['currency'] = offers[0].get(
            'priceCurrency', ''
        )
        result['availability'] = offers[0].get(
            'availability', ''
        )

    # Rating
    rating = product_data.get('aggregateRating', {})
    if isinstance(rating, dict):
        result['rating'] = rating.get('ratingValue', '')
        result['review_count'] = rating.get(
            'reviewCount', rating.get('ratingCount', '')
        )

    return result

Common Scraping Patterns

Sitemap-based discovery

Many e-commerce sites publish sitemaps that list all product URLs:

import requests
import xml.etree.ElementTree as ET

def get_product_urls_from_sitemap(sitemap_url):
    """Extract product URLs from a sitemap."""
    response = requests.get(sitemap_url, timeout=30)
    response.raise_for_status()

    root = ET.fromstring(response.content)
    # Handle namespace
    namespace = (
        '{http://www.sitemaps.org/schemas/sitemap/0.9}'
    )

    urls = []
    for url_element in root.findall(f'{namespace}url'):
        loc = url_element.find(f'{namespace}loc')
        if loc is not None and '/product' in loc.text:
            urls.append(loc.text)

    return urls

Pagination handling

def scrape_category_pages(base_url, max_pages=50):
    """Scrape products from paginated category pages."""
    all_products = []
    page = 1

    while page <= max_pages:
        url = f"{base_url}?page={page}"
        try:
            products = extract_structured_data(url)
            if not products:
                break  # No more products

            for product in products:
                parsed = parse_product(product)
                parsed['source_url'] = url
                parsed['page'] = page
                all_products.append(parsed)

            page += 1
            import time
            time.sleep(2)  # Polite delay

        except requests.RequestException:
            break

    return all_products

Anti-Bot Challenges

Legal Considerations

Best practices for legal compliance

Pricing Intelligence Applications

Price monitoring

Python: Simple price tracker

import csv
import os
from datetime import datetime

def track_price(product_id, product_name, price,
                currency, source, output_file):
    """Append a price observation to a tracking file."""
    file_exists = os.path.exists(output_file)

    with open(output_file, 'a', newline='') as f:
        writer = csv.writer(f)
        if not file_exists:
            writer.writerow([
                'timestamp', 'product_id', 'product_name',
                'price', 'currency', 'source'
            ])
        writer.writerow([
            datetime.now().isoformat(),
            product_id,
            product_name,
            price,
            currency,
            source
        ])

Extracting Seller Contact Information

E-commerce marketplace pages sometimes display seller contact information (email addresses, phone numbers, website links). When collecting seller data for B2B outreach purposes, save product or seller pages as HTML files and upload to Email Extractor to extract email addresses. This is useful for building outreach lists of marketplace sellers, brand owners or suppliers.

Related Guides

• Scraping Pricing Data
• Web Scraping for Market Research
• Scraping for Lead Generation
• Structured Data Extraction Guide
• Email Extraction for E-Commerce