commit 8bd33db6ddffc3d9f5bf4bad32fd768c4d44f018 Author: Steffen Date: Sat Nov 7 16:53:30 2026 +0100 Initial commit diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f5597bf --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +.venv +__pycache__ +*.pyc +*.pyo +*.pyd +.pytest_cache +.coverage +htmlcov/ +*.db +*.sqlite +*.sqlite3 +sample_*.json +*.log +.git +.gitignore +.env diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..420eab9 --- /dev/null +++ b/.env.example @@ -0,0 +1,18 @@ +# Database connection +DATABASE_URL=postgresql+psycopg://crawler_user:crawler_password@localhost:5432/oem_stock_db +DB_ECHO_SQL=false + +# Crawler Defaults +DEFAULT_MARKET=DE +DEFAULT_PAGE_SIZE=12 +DEFAULT_DELAY_SECONDS=1.0 +HTTP_TIMEOUT_SECONDS=25.0 +DEFAULT_PROXY= + +# Raw Payload Storage & Extraction +ENABLE_RAW_EXTRACTION=true +RAW_STORAGE_DIR=data/raw +RAW_COMPRESSION_LEVEL=6 + +# Logging +LOG_LEVEL=INFO diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e94707b --- /dev/null +++ b/.gitignore @@ -0,0 +1,72 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Virtual Environments & Package Managers +.venv/ +env/ +venv/ +ENV/ + +# Python Build & Packaging +build/ +dist/ +*.egg-info/ +*.egg + +# Testing, Coverage & Linters +.pytest_cache/ +.coverage +htmlcov/ +.mypy_cache/ +.ruff_cache/ + +# Playwright & Browser Artifacts +test-results/ +playwright-report/ +.playwright/ + +# Local Data & SQLite Storage +data/ +/data/ +tmp/ +*.sqlite +*.sqlite3 +*.db +*.db-journal +*.db-wal +*.db-shm + +# Local Logs, Exports & Payloads +*.log +*.json +*.json.gz +!pyproject.toml + +# Environment Variables (Keep example tracked) +.env +.env.* +.env.local +!.env.example + +# IDEs & Editors +.vscode/ +.idea/ +*.swp +*.swo +*.sublime-project +*.sublime-workspace + +# macOS & OS Metadata +.DS_Store +.AppleDouble +.LSOverride +._* +Icon? + +# Nextcloud Sync Artifacts +*conflict* +*conflicted copy* +.sync_*.db* +.sync/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2220b09 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +# syntax=docker/dockerfile:1 +FROM python:3.11-slim + +# Install system dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + libpq-dev \ + gcc \ + && rm -rf /var/lib/apt/lists/* + +# Install uv for fast Python package management +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ + +WORKDIR /app + +# Copy dependency definitions +COPY pyproject.toml /app/ + +# Install Python dependencies +RUN uv pip install --system --no-cache -r pyproject.toml + +# Copy source code and database migrations +COPY src/ /app/src/ +COPY schema.sql /app/ +COPY README.md /app/ + +# Install package in editable mode +RUN uv pip install --system --no-cache -e . + +ENV PYTHONPATH="/app/src" +ENV PYTHONUNBUFFERED=1 + +ENTRYPOINT ["oem-crawler"] +CMD ["--brand", "bmw", "--limit", "50"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..0f11c3d --- /dev/null +++ b/README.md @@ -0,0 +1,213 @@ +# OEM Vehicle Stock Crawler + +A production-ready, modular platform for crawling car manufacturer (OEM) inventory, tracking vehicle stock, specifications, optional equipment, and price changes over time. + +--- + +## πŸ“ Project Architecture + +``` +oem_crawler/ +β”œβ”€β”€ Dockerfile # Lean production container using uv +β”œβ”€β”€ docker-compose.yml # PostgreSQL 16 + crawler container orchestrator +β”œβ”€β”€ .env.example # Environment configuration template +β”œβ”€β”€ pyproject.toml # Dependencies and CLI entrypoints (`oem-crawler`, `oem-inspect`) +β”œβ”€β”€ schema.sql # PostgreSQL database DDL +β”‚ +β”œβ”€β”€ data/ # Development SQLite databases (gitignored) +β”‚ └── crawler_stock.db # Default local SQLite database file +β”‚ +β”œβ”€β”€ src/oem_crawler/ +β”‚ β”œβ”€β”€ config.py # Environment configuration (pydantic-settings) +β”‚ β”œβ”€β”€ cli.py # Main CLI application (`oem-crawler`) +β”‚ β”‚ +β”‚ β”œβ”€β”€ tools/ # Developer & stock inspection tools +β”‚ β”‚ β”œβ”€β”€ README.md # Tools documentation and sqlite3 recipes +β”‚ β”‚ └── view_stock.py # Terminal inventory inspector (`oem-inspect`) +β”‚ β”‚ +β”‚ β”œβ”€β”€ db/ # Database Layer +β”‚ β”‚ β”œβ”€β”€ models.py # SQLAlchemy 2.0 ORM mappings +β”‚ β”‚ β”œβ”€β”€ schemas.py # Pydantic v2 validation models & SHA-256 hashing +β”‚ β”‚ β”œβ”€β”€ session.py # Connection and session factory +β”‚ β”‚ └── pipeline.py # Ingestion pipeline, change-detection & delisting +β”‚ β”‚ +β”‚ └── crawlers/ # Modular OEM Crawlers +β”‚ β”œβ”€β”€ base.py # BaseCrawler interface & CrawlerRegistry +β”‚ β”œβ”€β”€ audi/ # Audi Implementation +β”‚ β”‚ β”œβ”€β”€ crawler.py # Stock Car Service (SCS) REST client +β”‚ β”‚ └── parser.py # PR option codes, EV specs & GPS coordinates +β”‚ β”œβ”€β”€ bmw/ # BMW Implementation +β”‚ β”‚ β”œβ”€β”€ crawler.py # REST API client & pagination +β”‚ β”‚ └── parser.py # OEM response normalization +β”‚ β”œβ”€β”€ mercedes/ # Mercedes-Benz Implementation +β”‚ β”‚ β”œβ”€β”€ crawler.py # OneSearch GraphQL + PLS + VPC client +β”‚ β”‚ └── parser.py # OneWeb normalization logic +β”‚ β”‚ └── query.py # Tested GraphQL query definitions +β”‚ └── porsche/ # Porsche Implementation +β”‚ β”œβ”€β”€ crawler.py # Finder Next.js RSC client & pagination +β”‚ └── parser.py # Option C, equipment & metadata normalization +β”‚ +└── tests/ # Automated Test Suite + β”œβ”€β”€ test_schema.py # Schema & hashing unit tests + β”œβ”€β”€ test_audi_parser.py # Audi parser & registry unit tests + β”œβ”€β”€ test_bmw_parser.py # BMW parser & registry unit tests + β”œβ”€β”€ test_mercedes_parser.py # Mercedes parser unit tests + β”œβ”€β”€ test_porsche_parser.py # Porsche parser unit tests + └── test_pipeline.py # Ingestion pipeline unit tests +``` + +--- + +## πŸ”Œ Adding a New Car Manufacturer Crawler + +Adding a new OEM crawler (e.g. Mercedes, Audi, Tesla, Porsche) takes 2 simple steps: + +1. Create a new folder under `src/oem_crawler/crawlers//` (e.g. `mercedes/`): +2. Implement your crawler inheriting from `BaseCrawler` and register it: + +```python +from oem_crawler.crawlers.base import BaseCrawler, CrawlerRegistry +from oem_crawler.db.schemas import VehicleIngestSchema + +@CrawlerRegistry.register("mercedes") +class MercedesStockCrawler(BaseCrawler): + @property + def brand_name(self) -> str: + return "Mercedes-Benz" + + def crawl(self, max_vehicles=None, page_size=20, search_filter=None): + # 1. Fetch from OEM API / site + # 2. Parse into VehicleIngestSchema + # 3. Yield parsed vehicle + ... +``` + +The CLI (`oem-crawler --brand mercedes`) and pipeline will automatically discover and support it. + +--- + +## 🐳 Running with Docker + +### 1. Start PostgreSQL Database & Run a Crawl +```bash +docker compose up --build +``` + +### 2. Run Ad-hoc Crawl inside Docker +```bash +# Fetch 100 vehicles for BMW +docker compose run --rm crawler --brand bmw --limit 100 +``` + +### 3. Query PostgreSQL Database inside Docker + +You can query the database directly in your terminal using `docker compose exec`: + +```bash +# View latest vehicles in stock +docker compose exec postgres psql -U crawler_user -d oem_stock_db -c \ + "SELECT brand, series, model_name, price_gross, exterior_color, location_city FROM vehicles LIMIT 10;" + +# View stored factory equipment & options for a vehicle +docker compose exec postgres psql -U crawler_user -d oem_stock_db -c \ + "SELECT option_code, name, category, is_standard, price_gross FROM vehicle_options LIMIT 15;" + +# View crawl run history and telemetry +docker compose exec postgres psql -U crawler_user -d oem_stock_db -c \ + "SELECT id, brand, status, items_discovered, items_inserted, duration_ms, started_at FROM crawl_runs;" + +# View price and status history snapshots +docker compose exec postgres psql -U crawler_user -d oem_stock_db -c \ + "SELECT vehicle_id, price_gross, monthly_rate, status, recorded_at FROM vehicle_price_history LIMIT 10;" +``` + +### 4. Connect with GUI Database Clients (DBeaver, TablePlus, pgAdmin) +- **Host**: `localhost` +- **Port**: `5432` +- **Database**: `oem_stock_db` +- **User**: `crawler_user` +- **Password**: `crawler_password` + +--- + +## πŸ’» Local Development (without Docker) + +### 1. Run Unit Tests +```bash +uv run pytest +``` + +### 2. Run Local Crawl & Persistent Ingestion + +> **Note**: The default local database is automatically stored in `data/crawler_stock.db` (gitignored). You do not need to specify `--db-url` unless targeting a custom location or PostgreSQL. + +```bash +# Crawl New BMW Vehicles (Neuwagensuche) +uv run oem-crawler --brand bmw --stock-type new --limit 50 + +# Crawl New Mercedes-Benz Vehicles (Neuwagensuche via OneSearch GraphQL + PLS/VPC) +uv run oem-crawler --brand mercedes --stock-type new --limit 50 + +# Crawl Used Mercedes-Benz Vehicles (Gebrauchtwagen via gebrauchtwagen.mercedes-benz.de) +uv run oem-crawler --brand mercedes --stock-type used --limit 50 + +# Crawl Both New & Used Vehicles +uv run oem-crawler --brand mercedes --stock-type all --limit 100 + +# Crawl Used BMW Vehicles (Gebrauchtwagen with mileage & first registration) +uv run oem-crawler --brand bmw --stock-type used --limit 50 + +# Crawl New Porsche Vehicles (Finder Neuwagen via Next.js RSC) +uv run oem-crawler --brand porsche --stock-type new --limit 50 + +# Crawl Used / Approved Porsche Vehicles (Finder Gebrauchtwagen via Next.js RSC) +uv run oem-crawler --brand porsche --stock-type used --limit 50 + +# Crawl Both New & Used Porsche Vehicles +uv run oem-crawler --brand porsche --stock-type all --limit 100 + +# Or export Porsche stock directly to local JSON +uv run oem-crawler --brand porsche --stock-type new --limit 20 --output-json porsche_new_stock.json + +# Crawl New Audi Vehicles (NeuwagenbΓΆrse via Stock Car Service scs.audi.de) +uv run oem-crawler --brand audi --stock-type new --limit 50 + +# Crawl Used Audi Vehicles (GebrauchtwagenbΓΆrse with mileage, dealer GPS & PR options) +uv run oem-crawler --brand audi --stock-type used --limit 50 + +# Crawl Both New & Used Audi Vehicles +uv run oem-crawler --brand audi --stock-type all --limit 100 + +# --- Full Scans & Multi-Brand Ingestion --- + +# πŸš€ Full Scan across ALL Brands (Audi, BMW, Mercedes-Benz, Porsche) +uv run oem-crawler --all-brands --full-scan --delay 0.3 + +# Full Scan across all brands for New Cars only +uv run oem-crawler --all-brands --stock-type new --full-scan --delay 0.3 + +# Scan a custom selection of brands in one command +uv run oem-crawler --brand bmw,audi,porsche --full-scan --delay 0.3 +``` + +### 3. Inspect Stored Inventory in Terminal (`oem-inspect`) + +The project includes an inspection and auditing utility located in `src/oem_crawler/tools/`: + +```bash +# View inventory summary and brand breakdown table +uv run oem-inspect --summary + +# View first 10 vehicles across all brands +uv run oem-inspect --limit 10 + +# Filter by brand +uv run oem-inspect --brand audi --limit 5 +uv run oem-inspect --brand porsche --limit 10 + +# Or run via Python module directly +uv run python -m oem_crawler.tools.view_stock --summary +``` + +For more options and direct SQLite query recipes, see the [Tools Documentation](src/oem_crawler/tools/README.md). + diff --git a/demo_crawl.py b/demo_crawl.py new file mode 100644 index 0000000..d8c5e23 --- /dev/null +++ b/demo_crawl.py @@ -0,0 +1,60 @@ +""" +Demo Script: Live BMW Stock Crawling and Database Ingestion Prototype +""" +from __future__ import annotations + +import logging +from sqlalchemy import select, func + +from oem_crawler.crawlers.bmw.crawler import BMWStockCrawler +from oem_crawler.db.models import Vehicle, VehicleOption, VehiclePriceHistory +from oem_crawler.db.pipeline import StockIngestionPipeline + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") + + +def main(): + print("=" * 70) + print("πŸš€ Initializing OEM Stock Crawler Prototype") + print("=" * 70) + + pipeline = StockIngestionPipeline() + crawler = BMWStockCrawler(market_country="DE") + + limit = 6 + print(f"\nπŸ“‘ Crawling first {limit} live vehicles from BMW Neuwagensuche REST API...") + stream = crawler.crawl(max_vehicles=limit, page_size=limit) + + crawl_run = pipeline.run_crawler_pipeline( + brand="BMW", + crawler_name="bmw-rest-crawler-v1", + vehicle_stream=stream, + market_country="DE", + config_snapshot={"max_vehicles": limit, "is_full_market_scan": False}, + ) + + print("\n" + "=" * 70) + print(f"βœ… Crawl Run Complete: {crawl_run.id}") + print(f" - Status: {crawl_run.status}") + print(f" - Duration: {crawl_run.duration_ms} ms") + print(f" - Discovered: {crawl_run.items_discovered}") + print(f" - Inserted: {crawl_run.items_inserted}") + print(f" - Updated: {crawl_run.items_updated}") + print(f" - Unmodified: {crawl_run.items_unmodified}") + print("=" * 70) + + with pipeline.session_factory() as session: + vehicles = session.scalars(select(Vehicle).limit(3)).all() + print("\nπŸ” Sample Stored Vehicles:") + for v in vehicles: + options_count = session.scalar( + select(func.count(VehicleOption.id)).where(VehicleOption.vehicle_id == v.id) + ) + print(f" β€’ [{v.brand}] {v.series} {v.model_name}") + print(f" Gross Price: {v.price_gross:,.2f} {v.currency} | Options: {options_count} items stored") + print(f" Dealer: {v.dealer_name or 'N/A'} ({v.location_city or 'N/A'})") + print("-" * 60) + + +if __name__ == "__main__": + main() diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..0729777 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,43 @@ +services: + postgres: + image: postgres:16-alpine + container_name: oem_postgres + environment: + POSTGRES_USER: crawler_user + POSTGRES_PASSWORD: crawler_password + POSTGRES_DB: oem_stock_db + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + - ./schema.sql:/docker-entrypoint-initdb.d/init_schema.sql:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U crawler_user -d oem_stock_db"] + interval: 5s + timeout: 5s + retries: 5 + restart: unless-stopped + + crawler: + build: + context: . + dockerfile: Dockerfile + container_name: oem_crawler + depends_on: + postgres: + condition: service_healthy + environment: + DATABASE_URL: postgresql+psycopg://crawler_user:crawler_password@postgres:5432/oem_stock_db + DEFAULT_MARKET: DE + LOG_LEVEL: INFO + DEFAULT_DELAY_SECONDS: 1.0 + ENABLE_RAW_EXTRACTION: "true" + RAW_STORAGE_DIR: "/data/raw" + volumes: + - ./data/raw:/data/raw + command: ["--brand", "bmw", "--limit", "50"] + restart: "no" + +volumes: + postgres_data: + driver: local diff --git a/human_backlog.md b/human_backlog.md new file mode 100644 index 0000000..b44ec68 --- /dev/null +++ b/human_backlog.md @@ -0,0 +1,3 @@ +- [x] store raw data +- [ ] b2b +- [ ] alerting diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b8f9baa --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[project] +name = "oem-crawler" +version = "0.1.0" +description = "Automotive OEM stock inventory crawler and data pipeline" +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "pydantic>=2.7.0", + "pydantic-settings>=2.2.0", + "sqlalchemy>=2.0.30", + "psycopg[binary]>=3.1.18", + "httpx[http2]>=0.27.0", + "playwright>=1.64.1", + "beautifulsoup4>=4.15.0", +] + +[project.scripts] +oem-crawler = "oem_crawler.cli:main" +oem-inspect = "oem_crawler.tools.view_stock:main" +oem-restore = "oem_crawler.tools.restore_from_raw:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[dependency-groups] +dev = [ + "pytest>=8.0.0", +] diff --git a/schema.sql b/schema.sql new file mode 100644 index 0000000..e2183e0 --- /dev/null +++ b/schema.sql @@ -0,0 +1,217 @@ +-- ============================================================================= +-- OEM Vehicle Stock Crawler - PostgreSQL Database Schema +-- ============================================================================= + +-- Enable UUID extension if not already available +CREATE EXTENSION IF NOT EXISTS "pgcrypto"; + +-- Function to automatically update 'updated_at' column +CREATE OR REPLACE FUNCTION update_updated_at_column() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ language 'plpgsql'; + +-- ============================================================================= +-- 1. Table: crawl_runs (Execution telemetry & audit log) +-- ============================================================================= +CREATE TABLE IF NOT EXISTS crawl_runs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + brand VARCHAR(50) NOT NULL, + market_country VARCHAR(2) NOT NULL DEFAULT 'DE', + crawler_name VARCHAR(100) NOT NULL, + crawler_version VARCHAR(20), + status VARCHAR(20) NOT NULL DEFAULT 'PENDING' + CHECK (status IN ('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', 'PARTIAL')), + started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + finished_at TIMESTAMPTZ, + duration_ms INTEGER, + items_discovered INTEGER NOT NULL DEFAULT 0, + items_inserted INTEGER NOT NULL DEFAULT 0, + items_updated INTEGER NOT NULL DEFAULT 0, + items_unmodified INTEGER NOT NULL DEFAULT 0, + items_delisted INTEGER NOT NULL DEFAULT 0, + items_failed INTEGER NOT NULL DEFAULT 0, + error_summary TEXT, + config_snapshot JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_crawl_runs_brand_status + ON crawl_runs(brand, status, started_at DESC); + +-- ============================================================================= +-- 2. Table: vehicles (Master Vehicle Stock Inventory) +-- ============================================================================= +CREATE TABLE IF NOT EXISTS vehicles ( + -- Identifiers + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + brand VARCHAR(50) NOT NULL, + oem_vehicle_id VARCHAR(100) NOT NULL, + vin VARCHAR(17), + + -- Model & Classification + series VARCHAR(100) NOT NULL, + model_name VARCHAR(255) NOT NULL, + trim_line VARCHAR(100), + body_type VARCHAR(50), + condition VARCHAR(30) NOT NULL DEFAULT 'new' + CHECK (condition IN ('new', 'used', 'demonstration', 'day_registration', 'certified_pre_owned')), + model_year SMALLINT, + first_registration_date DATE, + previous_owners_count SMALLINT, + + -- Powertrain & Technical Specifications + fuel_type VARCHAR(30) NOT NULL DEFAULT 'petrol' + CHECK (fuel_type IN ('bev', 'phev', 'petrol', 'diesel', 'mhev', 'fcev', 'cng', 'lpg', 'hybrid_other')), + transmission VARCHAR(30) + CHECK (transmission IN ('automatic', 'manual', 'direct_drive', 'dual_clutch', 'cvt', 'other')), + drivetrain VARCHAR(20) + CHECK (drivetrain IN ('awd', 'rwd', 'fwd', '4x4', 'other')), + power_kw INTEGER, + power_hp INTEGER, + battery_capacity NUMERIC(6,2), + battery_capacity_unit VARCHAR(10) + CHECK (battery_capacity_unit IS NULL OR battery_capacity_unit IN ('kwh', 'ah')), + battery_capacity_gross_kwh NUMERIC(6,2), + battery_capacity_net_kwh NUMERIC(6,2), + charging_power_ac_kw NUMERIC(5,2), + charging_power_dc_kw NUMERIC(5,2), + charging_time_dc_minutes SMALLINT, + electric_range_wltp_km INTEGER, + consumption_combined NUMERIC(5,2), + co2_emissions_g_km NUMERIC(5,1), + mileage_km INTEGER NOT NULL DEFAULT 0, + + -- Colors & Upholstery + exterior_color VARCHAR(100), + exterior_color_code VARCHAR(50), + exterior_color_generic VARCHAR(50), + interior_color VARCHAR(100), + interior_material VARCHAR(50), + + -- Pricing, Financing & Availability + currency VARCHAR(3) NOT NULL DEFAULT 'EUR', + base_price_gross NUMERIC(12,2), + options_price_gross NUMERIC(12,2), + price_gross NUMERIC(12,2) NOT NULL, + price_net NUMERIC(12,2), + msrp_gross NUMERIC(12,2), + discount_percentage NUMERIC(5,2), + vat_deductible BOOLEAN, + monthly_rate NUMERIC(10,2), + leasing_details JSONB NOT NULL DEFAULT '{}'::jsonb, + status VARCHAR(30) NOT NULL DEFAULT 'in_stock' + CHECK (status IN ('in_stock', 'in_transit', 'on_order', 'reserved', 'sold', 'delisted')), + availability_date DATE, + delivery_time_days INTEGER, + + -- Location, Dealer & Media + dealer_name VARCHAR(255), + dealer_id VARCHAR(100), + location_city VARCHAR(100), + location_postal_code VARCHAR(20), + location_country VARCHAR(2) NOT NULL DEFAULT 'DE', + latitude NUMERIC(9,6), + longitude NUMERIC(9,6), + vehicle_url TEXT, + image_urls JSONB NOT NULL DEFAULT '[]'::jsonb, + thumbnail_url TEXT, + + -- Timestamps, Audit & Change Tracking + first_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + delisted_at TIMESTAMPTZ, + last_crawl_run_id UUID REFERENCES crawl_runs(id) ON DELETE SET NULL, + content_hash VARCHAR(64) NOT NULL, + raw_payload JSONB NOT NULL DEFAULT '{}'::jsonb, + raw_payload_path VARCHAR(512), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + -- Compound uniqueness constraint to avoid duplicate vehicles per OEM per country + CONSTRAINT uq_vehicles_brand_market_oem_id UNIQUE (brand, location_country, oem_vehicle_id) +); + +-- Triggers +DROP TRIGGER IF EXISTS trg_vehicles_updated_at ON vehicles; +CREATE TRIGGER trg_vehicles_updated_at + BEFORE UPDATE ON vehicles + FOR EACH ROW + EXECUTE FUNCTION update_updated_at_column(); + +-- Indexes on vehicles +CREATE INDEX IF NOT EXISTS idx_vehicles_brand_series + ON vehicles(brand, series); +CREATE INDEX IF NOT EXISTS idx_vehicles_fuel_transmission + ON vehicles(fuel_type, transmission); +CREATE INDEX IF NOT EXISTS idx_vehicles_price_gross + ON vehicles(price_gross); +CREATE INDEX IF NOT EXISTS idx_vehicles_status_last_seen + ON vehicles(status, last_seen_at DESC); +CREATE INDEX IF NOT EXISTS idx_vehicles_vin + ON vehicles(vin) WHERE vin IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_vehicles_location + ON vehicles(location_country, location_city); + +-- ============================================================================= +-- 3. Table: vehicle_options (Normalized Equipment & Option Pricing) +-- ============================================================================= +CREATE TABLE IF NOT EXISTS vehicle_options ( + id BIGSERIAL PRIMARY KEY, + vehicle_id UUID NOT NULL REFERENCES vehicles(id) ON DELETE CASCADE, + option_code VARCHAR(50), + name VARCHAR(255) NOT NULL, + category VARCHAR(100), + price_gross NUMERIC(10,2), + price_net NUMERIC(10,2), + is_standard BOOLEAN NOT NULL DEFAULT FALSE, + description TEXT, + raw_data JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_vehicle_options_vehicle_id + ON vehicle_options(vehicle_id); +CREATE INDEX IF NOT EXISTS idx_vehicle_options_code + ON vehicle_options(option_code); +CREATE INDEX IF NOT EXISTS idx_vehicle_options_category + ON vehicle_options(category); + +-- ============================================================================= +-- 4. Table: vehicle_price_history (Historical Price & Status Snapshots) +-- ============================================================================= +CREATE TABLE IF NOT EXISTS vehicle_price_history ( + id BIGSERIAL PRIMARY KEY, + vehicle_id UUID NOT NULL REFERENCES vehicles(id) ON DELETE CASCADE, + crawl_run_id UUID NOT NULL REFERENCES crawl_runs(id) ON DELETE CASCADE, + price_gross NUMERIC(12,2) NOT NULL, + price_net NUMERIC(12,2), + monthly_rate NUMERIC(10,2), + status VARCHAR(30) NOT NULL, + recorded_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_price_history_vehicle_recorded + ON vehicle_price_history(vehicle_id, recorded_at DESC); +CREATE INDEX IF NOT EXISTS idx_price_history_crawl_run + ON vehicle_price_history(crawl_run_id); + +-- ============================================================================= +-- 5. Table: crawl_logs (Granular Diagnostic & Error Logging) +-- ============================================================================= +CREATE TABLE IF NOT EXISTS crawl_logs ( + id BIGSERIAL PRIMARY KEY, + crawl_run_id UUID NOT NULL REFERENCES crawl_runs(id) ON DELETE CASCADE, + vehicle_id UUID REFERENCES vehicles(id) ON DELETE SET NULL, + level VARCHAR(10) NOT NULL CHECK (level IN ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL')), + stage VARCHAR(50) NOT NULL CHECK (stage IN ('FETCH', 'PARSE', 'VALIDATE', 'PERSIST', 'DELIST_CHECK', 'GENERAL')), + message TEXT NOT NULL, + error_context JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_crawl_logs_run_level + ON crawl_logs(crawl_run_id, level, created_at DESC); diff --git a/scripts/migrate_raw_payloads.py b/scripts/migrate_raw_payloads.py new file mode 100755 index 0000000..3cc709c --- /dev/null +++ b/scripts/migrate_raw_payloads.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +""" +Migration script to extract raw OEM payloads from SQLite into external compressed .json.gz files, +update vehicles with raw_payload_path, clear in-database raw JSON, and compact the database via VACUUM. +""" +from __future__ import annotations + +import argparse +import gzip +import json +import logging +import os +import sqlite3 +import sys +import time +from pathlib import Path +from typing import Any, Dict, Optional + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + datefmt="%H:%M:%S", +) +logger = logging.getLogger("migrate_raw_payloads") + + +def format_bytes(num_bytes: int | float) -> str: + """Formats bytes into human-readable string (KB, MB, GB).""" + for unit in ["B", "KB", "MB", "GB", "TB"]: + if abs(num_bytes) < 1024.0: + return f"{num_bytes:.2f} {unit}" + num_bytes /= 1024.0 + return f"{num_bytes:.2f} PB" + + +def sanitize_filename(name: str) -> str: + """Sanitizes an OEM vehicle id for filesystem safety.""" + return "".join(c if c.isalnum() or c in ("-", "_") else "_" for c in str(name)) + + +def migrate_database( + db_path: Path, + raw_dir: Path, + compression_level: int = 6, + batch_size: int = 500, + dry_run: bool = False, + skip_vacuum: bool = False, +) -> None: + if not db_path.exists(): + logger.error(f"Database file not found: {db_path}") + sys.exit(1) + + initial_db_size = db_path.stat().st_size + logger.info(f"Target Database: {db_path} (Size: {format_bytes(initial_db_size)})") + logger.info(f"Storage Directory: {raw_dir.resolve()}") + logger.info(f"Dry Run Mode: {dry_run}") + + conn = sqlite3.connect(str(db_path)) + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + # 1. Ensure raw_payload_path column exists + cursor.execute("PRAGMA table_info(vehicles);") + columns = [row["name"] for row in cursor.fetchall()] + if "raw_payload_path" not in columns: + if dry_run: + logger.info("[Dry Run] Would execute: ALTER TABLE vehicles ADD COLUMN raw_payload_path VARCHAR(512);") + else: + logger.info("Adding 'raw_payload_path' column to vehicles table...") + cursor.execute("ALTER TABLE vehicles ADD COLUMN raw_payload_path VARCHAR(512);") + conn.commit() + + # 2. Count vehicles requiring migration + cursor.execute(""" + SELECT COUNT(*) as count + FROM vehicles + WHERE raw_payload IS NOT NULL AND length(raw_payload) > 2; + """) + total_vehicles = cursor.fetchone()["count"] + logger.info(f"Found {total_vehicles:,} vehicles with raw payload to extract.") + + cursor.execute(""" + SELECT COUNT(*) as count + FROM vehicle_options + WHERE raw_data IS NOT NULL AND raw_data != '{}'; + """) + total_options = cursor.fetchone()["count"] + logger.info(f"Found {total_options:,} vehicle options with redundant raw_data.") + + if total_vehicles == 0 and total_options == 0: + logger.info("No vehicles or options require migration.") + if not skip_vacuum and not dry_run: + logger.info("Running VACUUM to ensure database is compact...") + cursor.execute("VACUUM;") + conn.close() + return + + if dry_run: + logger.info("[Dry Run] Migration simulation complete. No changes written.") + conn.close() + return + + # 3. Process vehicles in batches + raw_dir.mkdir(parents=True, exist_ok=True) + migrated_count = 0 + total_uncompressed_bytes = 0 + total_compressed_bytes = 0 + start_time = time.time() + + cursor.execute(""" + SELECT id, brand, oem_vehicle_id, created_at, last_seen_at, raw_payload, raw_payload_path + FROM vehicles + WHERE raw_payload IS NOT NULL AND length(raw_payload) > 2; + """) + + updates_batch = [] + + while True: + rows = cursor.fetchmany(batch_size) + if not rows: + break + + for row in rows: + v_id = row["id"] + brand = (row["brand"] or "unknown").strip().lower() + oem_id = row["oem_vehicle_id"] or v_id + created_at = str(row["last_seen_at"] or row["created_at"] or "") + date_str = created_at[:10] if len(created_at) >= 10 else "archive" + + raw_str = row["raw_payload"] + if not raw_str or raw_str == "{}": + continue + + try: + payload = json.loads(raw_str) if isinstance(raw_str, str) else raw_str + except Exception as e: + logger.warning(f"Could not parse JSON for vehicle {v_id}: {e}") + continue + + # Compute relative path + safe_id = sanitize_filename(oem_id) + rel_path = f"{brand}/{date_str}/{safe_id}.json.gz" + dest_file = raw_dir / rel_path + dest_file.parent.mkdir(parents=True, exist_ok=True) + + # Compress and write + json_bytes = json.dumps(payload, ensure_ascii=False).encode("utf-8") + compressed_data = gzip.compress(json_bytes, compresslevel=compression_level) + + with open(dest_file, "wb") as f: + f.write(compressed_data) + + total_uncompressed_bytes += len(json_bytes) + total_compressed_bytes += len(compressed_data) + migrated_count += 1 + + updates_batch.append((rel_path, v_id)) + + # Commit DB updates for batch + if updates_batch: + update_cursor = conn.cursor() + update_cursor.executemany( + "UPDATE vehicles SET raw_payload_path = ?, raw_payload = '{}' WHERE id = ?;", + updates_batch, + ) + conn.commit() + updates_batch.clear() + + elapsed = time.time() - start_time + rate = migrated_count / elapsed if elapsed > 0 else 0 + logger.info( + f"Progress: {migrated_count:,}/{total_vehicles:,} vehicles ({migrated_count/total_vehicles*100:.1f}%) " + f"| Rate: {rate:.0f} veh/sec" + ) + + # 4. Clean vehicle_options raw_data + logger.info("Clearing redundant raw_data in vehicle_options...") + cursor.execute("UPDATE vehicle_options SET raw_data = '{}' WHERE raw_data != '{}';") + conn.commit() + + # 5. Compact the database with VACUUM + if not skip_vacuum: + logger.info("Running VACUUM to reclaim free space (this may take a few moments)...") + vac_start = time.time() + cursor.execute("VACUUM;") + logger.info(f"VACUUM completed in {time.time() - vac_start:.1f}s.") + + conn.close() + + final_db_size = db_path.stat().st_size + reclaimed_bytes = initial_db_size - final_db_size + + # Summary + logger.info("\n" + "=" * 60) + logger.info("MIGRATION COMPLETED SUCCESSFULLY") + logger.info("=" * 60) + logger.info(f"Vehicles Migrated: {migrated_count:,}") + logger.info(f"Options Cleaned: {total_options:,}") + logger.info(f"Uncompressed Payload Size:{format_bytes(total_uncompressed_bytes)}") + logger.info(f"Compressed File Size: {format_bytes(total_compressed_bytes)} " + f"({total_compressed_bytes / max(total_uncompressed_bytes, 1) * 100:.1f}%)") + logger.info(f"Original DB Size: {format_bytes(initial_db_size)}") + logger.info(f"New DB Size: {format_bytes(final_db_size)}") + logger.info(f"Space Reclaimed: {format_bytes(reclaimed_bytes)} " + f"({reclaimed_bytes / max(initial_db_size, 1) * 100:.1f}% reduction)") + logger.info("=" * 60) + + +def main(): + parser = argparse.ArgumentParser(description="Extract raw JSON payloads from SQLite to external compressed files") + parser.add_argument( + "--db-path", + type=Path, + default=Path("data/crawler_stock.db"), + help="Path to SQLite database file (default: data/crawler_stock.db)", + ) + parser.add_argument( + "--raw-dir", + type=Path, + default=Path("data/raw"), + help="Destination directory for compressed JSON files (default: data/raw)", + ) + parser.add_argument( + "--compression-level", + type=int, + default=6, + help="Gzip compression level (1-9, default: 6)", + ) + parser.add_argument( + "--batch-size", + type=int, + default=500, + help="Batch size for processing and DB updates (default: 500)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Inspect database without writing files or modifying database", + ) + parser.add_argument( + "--skip-vacuum", + action="store_true", + help="Skip database VACUUM at end of migration", + ) + args = parser.parse_args() + + migrate_database( + db_path=args.db_path, + raw_dir=args.raw_dir, + compression_level=args.compression_level, + batch_size=args.batch_size, + dry_run=args.dry_run, + skip_vacuum=args.skip_vacuum, + ) + + +if __name__ == "__main__": + main() + diff --git a/scripts/restore_from_raw.py b/scripts/restore_from_raw.py new file mode 100755 index 0000000..e0af92e --- /dev/null +++ b/scripts/restore_from_raw.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +""" +CLI launcher for the restore_from_raw tool. +""" +import sys +from pathlib import Path + +# Add src to sys.path +root_dir = Path(__file__).resolve().parent.parent +src_dir = root_dir / "src" +if str(src_dir) not in sys.path: + sys.path.insert(0, str(src_dir)) + +from oem_crawler.tools.restore_from_raw import main + +if __name__ == "__main__": + main() + diff --git a/src/oem_crawler/__init__.py b/src/oem_crawler/__init__.py new file mode 100644 index 0000000..f7519c3 --- /dev/null +++ b/src/oem_crawler/__init__.py @@ -0,0 +1,4 @@ +""" +OEM Vehicle Stock Crawler Package +""" +__version__ = "0.1.0" diff --git a/src/oem_crawler/cli.py b/src/oem_crawler/cli.py new file mode 100644 index 0000000..ba94db1 --- /dev/null +++ b/src/oem_crawler/cli.py @@ -0,0 +1,246 @@ +""" +OEM Stock Crawler Command-Line Interface (CLI). +Supports crawling individual brands or all registered brands in a single command. +""" +import argparse +import json +import logging +import sys +from typing import List + +from oem_crawler.config import settings +from oem_crawler.crawlers import CrawlerRegistry +from oem_crawler.db.pipeline import StockIngestionPipeline + +logging.basicConfig( + level=getattr(logging, settings.LOG_LEVEL.upper(), logging.INFO), + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s" +) +logger = logging.getLogger("oem_crawler") + + +def main(): + supported_aliases = CrawlerRegistry.supported_brands() + unique_brands = [brand for brand, _ in CrawlerRegistry.unique_brand_crawlers()] + + parser = argparse.ArgumentParser(description="OEM Vehicle Stock Crawler & Ingestion Pipeline") + parser.add_argument( + "--brand", + type=str, + default="all", + help=f"Car manufacturer to crawl (Supported: {', '.join(unique_brands)}, or 'all' to crawl all brands; comma-separated: 'bmw,audi')", + ) + parser.add_argument( + "--all-brands", + action="store_true", + help="Shortcut to run crawl across all registered OEM brands", + ) + parser.add_argument( + "--country", + type=str, + default=settings.DEFAULT_MARKET, + help=f"ISO market country code (default: {settings.DEFAULT_MARKET})", + ) + parser.add_argument( + "--stock-type", + type=str, + default="new", + choices=["new", "used", "all"], + help="Vehicle stock condition category: 'new', 'used', or 'all' (default: new)", + ) + parser.add_argument( + "--limit", + type=int, + default=50, + help="Maximum number of vehicles to fetch per brand (default: 50, use 0 or --full-scan for unlimited)", + ) + parser.add_argument( + "--full-scan", + action="store_true", + help="Perform a complete market scan without limit (enables automatic delisting detection in DB)", + ) + parser.add_argument( + "--page-size", + type=int, + default=settings.DEFAULT_PAGE_SIZE, + help=f"Page size for REST API queries (default: {settings.DEFAULT_PAGE_SIZE})", + ) + parser.add_argument( + "--delay", + type=float, + default=settings.DEFAULT_DELAY_SECONDS, + help=f"Polite delay between page requests in seconds (default: {settings.DEFAULT_DELAY_SECONDS}s)", + ) + parser.add_argument( + "--proxy", + type=str, + default=settings.DEFAULT_PROXY, + help="HTTP/SOCKS5 proxy URL (e.g. http://user:pass@proxy.com:8080)", + ) + parser.add_argument( + "--db-url", + type=str, + default=settings.DATABASE_URL, + help="SQLAlchemy Database URL (e.g. postgresql+psycopg://user:pass@localhost:5432/stock_db)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print/dump parsed vehicle JSON without saving to database", + ) + parser.add_argument( + "--output-json", + type=str, + default=None, + help="Save parsed vehicles directly to a local JSON file", + ) + parser.add_argument( + "--raw", + dest="enable_raw", + action="store_true", + default=None, + help="Force enable raw payload extraction and external compressed storage", + ) + parser.add_argument( + "--no-raw", + dest="enable_raw", + action="store_false", + default=None, + help="Force disable raw payload extraction and storage", + ) + + args = parser.parse_args() + + if args.enable_raw is not None: + settings.ENABLE_RAW_EXTRACTION = args.enable_raw + + # 1. Resolve brands to crawl + if args.all_brands or args.brand.strip().lower() in ("all", "*"): + brands_to_run = [brand for brand, _ in CrawlerRegistry.unique_brand_crawlers()] + else: + raw_brands = [b.strip().lower() for b in args.brand.split(",") if b.strip()] + for b in raw_brands: + if b not in supported_aliases: + parser.error(f"Unknown brand '{b}'. Supported brands: {', '.join(unique_brands)}, or 'all'") + # Deduplicate while preserving order and resolving aliases to unique crawler classes + seen_classes = set() + brands_to_run = [] + for b in raw_brands: + cls_ = CrawlerRegistry.get(b) + if cls_ not in seen_classes: + seen_classes.add(cls_) + brands_to_run.append(b) + + is_full_scan = args.full_scan or (args.limit is not None and args.limit <= 0) + max_vehicles = None if is_full_scan else args.limit + + scan_label = "FULL SCAN" if is_full_scan else f"limit={max_vehicles}" + brand_label = ", ".join(b.upper() for b in brands_to_run) + logger.info( + f"Target brands: [{brand_label}] | Market: {args.country} | Mode: {scan_label} | Stock: {args.stock_type} | Raw Extraction: {settings.ENABLE_RAW_EXTRACTION}" + ) + + # 2. Execution loop + pipeline = StockIngestionPipeline(db_url=args.db_url) if not (args.dry_run or args.output_json) else None + results_summary = [] + all_output_data = {} + + for brand in brands_to_run: + logger.info(f"\n{'='*70}\nStarting {brand.upper()} crawl ({scan_label}, stock_type: {args.stock_type})...\n{'='*70}") + crawler_cls = CrawlerRegistry.get(brand) + crawler = crawler_cls( + market_country=args.country, + proxy=args.proxy, + request_delay=args.delay, + timeout=settings.HTTP_TIMEOUT_SECONDS, + stock_type=args.stock_type, + ) + + stream = crawler.crawl(max_vehicles=max_vehicles, page_size=args.page_size) + + if args.dry_run or args.output_json: + vehicles_list = [v.model_dump(mode="json") for v in stream] + logger.info(f"[{brand.upper()}] Fetched and parsed {len(vehicles_list)} vehicles successfully.") + all_output_data[brand] = vehicles_list + results_summary.append({ + "brand": brand.upper(), + "status": "DRY_RUN", + "discovered": len(vehicles_list), + "inserted": 0, + "updated": 0, + "unmodified": 0, + "delisted": 0, + "failed": 0, + "duration_s": 0.0, + }) + continue + + try: + crawl_run = pipeline.run_crawler_pipeline( + brand=brand.upper(), + crawler_name=f"{brand.lower()}-rest-crawler", + vehicle_stream=stream, + market_country=args.country, + config_snapshot={ + "limit": max_vehicles, + "page_size": args.page_size, + "stock_type": args.stock_type, + "is_full_market_scan": is_full_scan, + } + ) + results_summary.append({ + "brand": brand.upper(), + "status": crawl_run.status, + "discovered": crawl_run.items_discovered, + "inserted": crawl_run.items_inserted, + "updated": crawl_run.items_updated, + "unmodified": crawl_run.items_unmodified, + "delisted": crawl_run.items_delisted, + "failed": crawl_run.items_failed, + "duration_s": round(crawl_run.duration_ms / 1000.0, 1), + }) + except Exception as e: + logger.error(f"[{brand.upper()}] Crawl run failed: {e}", exc_info=True) + results_summary.append({ + "brand": brand.upper(), + "status": "FAILED", + "discovered": crawl_run.items_discovered if crawl_run else 0, + "inserted": crawl_run.items_inserted if crawl_run else 0, + "updated": crawl_run.items_updated if crawl_run else 0, + "unmodified": crawl_run.items_unmodified if crawl_run else 0, + "delisted": crawl_run.items_delisted if crawl_run else 0, + "failed": 1, + "duration_s": round((crawl_run.duration_ms or 0) / 1000.0, 1) if crawl_run else 0.0, + }) + + # 3. Optional JSON Export + if args.output_json: + payload_to_write = all_output_data if len(brands_to_run) > 1 else list(all_output_data.values())[0] + with open(args.output_json, "w", encoding="utf-8") as f: + json.dump(payload_to_write, f, indent=2, ensure_ascii=False) + logger.info(f"Saved vehicle JSON dataset to {args.output_json}") + elif args.dry_run and all_output_data: + first_brand = list(all_output_data.keys())[0] + print(json.dumps(all_output_data[first_brand][:2], indent=2, ensure_ascii=False)) + + # 4. Final Aggregated Report + print("\n" + "=" * 95) + print("πŸ“Š MULTI-BRAND CRAWL SUMMARY REPORT") + print("=" * 95) + print(f"{'Brand':<15} {'Status':<12} {'Discovered':<12} {'Inserted':<10} {'Updated':<10} {'Unchanged':<10} {'Delisted':<10} {'Duration':<10}") + print("-" * 95) + total_disc = sum(r['discovered'] for r in results_summary) + total_ins = sum(r['inserted'] for r in results_summary) + total_upd = sum(r['updated'] for r in results_summary) + total_unm = sum(r['unmodified'] for r in results_summary) + total_del = sum(r['delisted'] for r in results_summary) + total_time = sum(r['duration_s'] for r in results_summary) + + for r in results_summary: + print(f"{r['brand']:<15} {r['status']:<12} {r['discovered']:<12} {r['inserted']:<10} {r['updated']:<10} {r['unmodified']:<10} {r['delisted']:<10} {r['duration_s']}s") + print("=" * 95) + print(f"Total Discovered: {total_disc:,} | Total Inserted: {total_ins:,} | Updated: {total_upd:,} | Delisted: {total_del:,} | Elapsed: {total_time:.1f}s\n") + + +if __name__ == "__main__": + main() diff --git a/src/oem_crawler/config.py b/src/oem_crawler/config.py new file mode 100644 index 0000000..52ad7e1 --- /dev/null +++ b/src/oem_crawler/config.py @@ -0,0 +1,43 @@ +""" +Configuration settings for OEM Crawler (loaded from environment or .env). +""" +from typing import Optional +from pydantic import Field, AliasChoices +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") + + # Database + DATABASE_URL: str = "sqlite:///data/crawler_stock.db" + DB_ECHO_SQL: bool = False + + # Crawler Settings + DEFAULT_MARKET: str = "DE" + DEFAULT_PAGE_SIZE: int = 12 + DEFAULT_DELAY_SECONDS: float = 0.8 + HTTP_TIMEOUT_SECONDS: float = 25.0 + DEFAULT_PROXY: Optional[str] = None + + # Raw Payload Storage & Extraction + ENABLE_RAW_EXTRACTION: bool = Field( + default=True, + validation_alias=AliasChoices("ENABLE_RAW_EXTRACTION", "SAVE_RAW_PAYLOAD", "EXTRACT_RAW_DATA"), + description="Global switch to enable/disable extraction and storage of raw OEM JSON payloads.", + ) + RAW_STORAGE_DIR: str = Field( + default="data/raw", + validation_alias=AliasChoices("RAW_STORAGE_DIR", "RAW_DATA_DIR", "RAW_PAYLOAD_DIR"), + description="Directory where compressed raw payloads are stored (relative for dev, mounted host dir for docker).", + ) + RAW_COMPRESSION_LEVEL: int = Field( + default=6, + description="Gzip compression level from 1 (fastest) to 9 (best compression).", + ) + + # Logging + LOG_LEVEL: str = "INFO" + + +settings = Settings() diff --git a/src/oem_crawler/crawlers/__init__.py b/src/oem_crawler/crawlers/__init__.py new file mode 100644 index 0000000..2be023a --- /dev/null +++ b/src/oem_crawler/crawlers/__init__.py @@ -0,0 +1,11 @@ +""" +Crawler module exports and automatic registry initialization. +""" +from oem_crawler.crawlers.base import BaseCrawler, CrawlerRegistry +# Import all crawler subpackages to trigger @CrawlerRegistry.register +import oem_crawler.crawlers.audi # noqa: F401 +import oem_crawler.crawlers.bmw # noqa: F401 +import oem_crawler.crawlers.mercedes # noqa: F401 +import oem_crawler.crawlers.porsche # noqa: F401 + +__all__ = ["BaseCrawler", "CrawlerRegistry"] diff --git a/src/oem_crawler/crawlers/audi/__init__.py b/src/oem_crawler/crawlers/audi/__init__.py new file mode 100644 index 0000000..44a2e2d --- /dev/null +++ b/src/oem_crawler/crawlers/audi/__init__.py @@ -0,0 +1,7 @@ +""" +Audi Crawler Package Exports. +""" +from oem_crawler.crawlers.audi.crawler import AudiStockCrawler +from oem_crawler.crawlers.audi.parser import parse_audi_vehicle + +__all__ = ["AudiStockCrawler", "parse_audi_vehicle"] diff --git a/src/oem_crawler/crawlers/audi/crawler.py b/src/oem_crawler/crawlers/audi/crawler.py new file mode 100644 index 0000000..d50c53e --- /dev/null +++ b/src/oem_crawler/crawlers/audi/crawler.py @@ -0,0 +1,263 @@ +""" +Audi Stock Car Service (SCS) Crawler Implementation. +Interacts with https://scs.audi.de REST API endpoints. +Supports both new and used vehicles from the unified SCS backend. +""" +from __future__ import annotations + +import logging +import time +from typing import Any, Dict, Generator, List, Optional, Tuple + +import httpx + +from oem_crawler.crawlers.audi.parser import parse_audi_vehicle +from oem_crawler.crawlers.base import BaseCrawler, CrawlerRegistry +from oem_crawler.db.schemas import VehicleIngestSchema + +logger = logging.getLogger(__name__) + + +@CrawlerRegistry.register("audi") +class AudiStockCrawler(BaseCrawler): + BASE_URL = "https://scs.audi.de" + DEFAULT_TOKEN = "FJ54W6H" + DEFAULT_USER_AGENT = ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36" + ) + + def __init__( + self, + market_country: str = "DE", + proxy: Optional[str] = None, + request_delay: float = 0.5, + timeout: float = 25.0, + stock_type: str = "new", + fetch_details: bool = True, + sort: str = "dateOffer:desc", + token: Optional[str] = None, + **kwargs: Any, + ): + super().__init__( + market_country=market_country, + proxy=proxy, + request_delay=request_delay, + timeout=timeout, + **kwargs, + ) + self.stock_type = stock_type.lower() + self.fetch_details = fetch_details + self.sort = sort + self.token = token or self.DEFAULT_TOKEN + + self.headers = { + "token": self.token, + "User-Agent": self.DEFAULT_USER_AGENT, + "Accept": "application/json", + "Accept-Language": "de-DE,de;q=0.9,en;q=0.8", + } + + @property + def brand_name(self) -> str: + return "AUDI" + + def get_market_code(self, condition: str) -> str: + """ + Maps stock condition to Audi SCS market code. + New cars: 'de' + Used cars: 'deuc' + """ + if condition.lower() == "used": + return f"{self.market_country.lower()}uc" + return self.market_country.lower() + + def fetch_search_page( + self, + client: httpx.Client, + market_code: str, + page: int = 0, + size: int = 50, + search_filter: Optional[Dict[str, Any]] = None, + max_retries: int = 3, + ) -> Tuple[int, List[Dict[str, Any]]]: + """ + Fetches one page of search results from Audi Stock Car Service. + Returns (totalCount, vehicleBasic list). + """ + lang = self.market_country.lower() + url = f"{self.BASE_URL}/api/v2/search/filter/{market_code}/{lang}" + params: Dict[str, Any] = { + "from": page * size, + "size": size, + "sort": self.sort, + } + if search_filter: + params.update(search_filter) + + for attempt in range(1, max_retries + 1): + try: + time.sleep(self.request_delay) + resp = client.get(url, params=params, headers=self.headers) + + if resp.status_code == 429: + wait_s = 2.0 * attempt + logger.warning(f"Audi rate limit (HTTP 429). Retrying in {wait_s}s...") + time.sleep(wait_s) + continue + + resp.raise_for_status() + data = resp.json() + total_count = data.get("totalCount", 0) + items = data.get("vehicleBasic", []) + return total_count, items + + except httpx.HTTPError as e: + logger.warning(f"Attempt {attempt}/{max_retries} failed for Audi market {market_code} page {page}: {e}") + if attempt == max_retries: + raise + time.sleep(attempt * 1.5) + + return 0, [] + + def fetch_detail( + self, + client: httpx.Client, + market_code: str, + vehicle_id: str, + max_retries: int = 2, + ) -> Optional[Dict[str, Any]]: + """ + Fetches full vehicle detail (features, PR equipment, techData) for a vehicle. + """ + lang = self.market_country.lower() + url = f"{self.BASE_URL}/api/v1/vehicles/vehicle/{market_code}/{lang}/{vehicle_id}" + + for attempt in range(1, max_retries + 1): + try: + time.sleep(self.request_delay) + resp = client.get(url, headers=self.headers) + if resp.status_code == 200: + return resp.json() + elif resp.status_code == 404: + return None + elif resp.status_code == 429: + time.sleep(2.0 * attempt) + continue + resp.raise_for_status() + except Exception as e: + logger.warning(f"Attempt {attempt}/{max_retries} failed for Audi detail {vehicle_id}: {e}") + if attempt == max_retries: + return None + time.sleep(1.0) + + return None + + def _crawl_condition( + self, + client: httpx.Client, + condition: str, + max_vehicles: Optional[int], + page_size: int, + search_filter: Optional[Dict[str, Any]], + ) -> Generator[VehicleIngestSchema, None, None]: + """ + Streams vehicles for a specific condition ('new' or 'used'). + """ + market_code = self.get_market_code(condition) + count = 0 + current_page = 0 + total_count = None + seen_page_ids: set[str] = set() + + logger.info( + f"Starting Audi {condition.upper()} stock crawl (Market: {market_code}, Lang: {self.market_country.lower()})..." + ) + + while True: + total_items, items = self.fetch_search_page( + client=client, + market_code=market_code, + page=current_page, + size=page_size, + search_filter=search_filter, + ) + + if total_count is None: + total_count = total_items + total_pages = (total_count + page_size - 1) // page_size if total_count else 0 + logger.info(f"Audi {condition} reports {total_count} vehicles available across ~{total_pages} pages.") + + if not items: + logger.info(f"No more items returned on page {current_page}. Finishing {condition} crawl.") + break + + # Fail-safe against infinite pagination loop + page_ids = {str(item.get("id")) for item in items if item.get("id")} + new_ids = page_ids - seen_page_ids + if not new_ids and items: + logger.warning(f"Audi {condition} page {current_page} returned duplicate IDs. Terminating pagination.") + break + seen_page_ids.update(page_ids) + + logger.info(f"Audi {condition}: processing page {current_page} ({len(items)} items, yielded so far: {count}).") + + for item in items: + vid = item.get("id") + detail_data = None + if self.fetch_details and vid: + detail_data = self.fetch_detail(client, market_code, vid) + + try: + vehicle = parse_audi_vehicle( + basic_item=item, + detail_data=detail_data, + market_country=self.market_country, + requested_stock_type=condition, + ) + yield vehicle + count += 1 + except Exception as e: + logger.error(f"Failed to parse Audi vehicle {item.get('carId')}: {e}", exc_info=True) + + if max_vehicles and count >= max_vehicles: + logger.info(f"Reached limit of {max_vehicles} {condition} vehicles.") + return + + current_page += 1 + if total_count is not None and (current_page * page_size) >= total_count: + break + + def crawl( + self, + max_vehicles: Optional[int] = None, + page_size: int = 50, + search_filter: Optional[Dict[str, Any]] = None, + ) -> Generator[VehicleIngestSchema, None, None]: + """ + Main crawling generator yielding normalized VehicleIngestSchema objects. + Supports stock_type in ('new', 'used', 'all'). + """ + page_size = max(1, min(page_size, 100)) + client_kwargs: Dict[str, Any] = { + "timeout": self.timeout, + "http2": False, + } + if self.proxy: + client_kwargs["proxy"] = self.proxy + + with httpx.Client(**client_kwargs) as client: + if self.stock_type == "all": + half_limit = (max_vehicles // 2) if max_vehicles else None + new_count = 0 + for veh in self._crawl_condition(client, "new", half_limit, page_size, search_filter): + yield veh + new_count += 1 + + rem_limit = (max_vehicles - new_count) if max_vehicles else None + for veh in self._crawl_condition(client, "used", rem_limit, page_size, search_filter): + yield veh + elif self.stock_type == "used": + yield from self._crawl_condition(client, "used", max_vehicles, page_size, search_filter) + else: + yield from self._crawl_condition(client, "new", max_vehicles, page_size, search_filter) diff --git a/src/oem_crawler/crawlers/audi/parser.py b/src/oem_crawler/crawlers/audi/parser.py new file mode 100644 index 0000000..078b242 --- /dev/null +++ b/src/oem_crawler/crawlers/audi/parser.py @@ -0,0 +1,512 @@ +""" +Parser and normalization logic for Audi Stock Car Service (SCS) responses. +Handles search vehicleBasic payloads and detail payloads. +""" +from __future__ import annotations + +import logging +import re +from datetime import date, datetime +from decimal import Decimal +from typing import Any, Dict, List, Optional + +from oem_crawler.config import settings +from oem_crawler.db.schemas import ( + BatteryCapacityUnit, + DrivetrainType, + FuelType, + OptionSchema, + TransmissionType, + VehicleCondition, + VehicleIngestSchema, + VehicleStatus, +) + +logger = logging.getLogger(__name__) + + +def safe_decimal(val: Any) -> Optional[Decimal]: + """Safely extracts a Decimal from numeric, string, or dict values.""" + if val is None: + return None + if isinstance(val, dict): + val = val.get("value") or val.get("amount") or val.get("rawValue") + if val is None: + return None + try: + raw_s = str(val).strip().replace("€", "").replace("\xa0", "").strip() + if "," in raw_s and "." in raw_s: + # German number formatting like 21.480,00 + val_str = raw_s.replace(".", "").replace(",", ".") + return Decimal(val_str) + elif "," in raw_s: + return Decimal(raw_s.replace(",", ".")) + elif "." in raw_s: + return Decimal(raw_s) + if not raw_s: + return None + return Decimal(raw_s) + except Exception: + try: + return Decimal(str(val).strip()) + except Exception: + return None + + +def safe_int(val: Any) -> Optional[int]: + """Safely extracts an integer.""" + if val is None: + return None + if isinstance(val, dict): + val = val.get("value") + try: + val_str = str(val).split(".")[0].strip() + return int(val_str) + except Exception: + return None + + +def parse_german_date(val: Any) -> Optional[date]: + """Parses German date string DD.MM.YYYY or timestamp/ISO.""" + if not val: + return None + if isinstance(val, (int, float)): + # Millisecond timestamp e.g. 1319068800000 + try: + if val > 1e11: + val = val / 1000.0 + return datetime.fromtimestamp(val).date() + except Exception: + return None + if isinstance(val, str): + val = val.strip() + # DD.MM.YYYY + if "." in val: + try: + parts = val.split(".") + if len(parts) == 3: + return date(int(parts[2]), int(parts[1]), int(parts[0])) + except Exception: + pass + # ISO YYYY-MM-DD + if "-" in val: + try: + clean = val.split("T")[0].strip() + return datetime.strptime(clean, "%Y-%m-%d").date() + except Exception: + pass + return None + + +def map_audi_fuel_type(fuel_raw: Any) -> FuelType: + """Maps Audi fuel code / description to standardized FuelType enum.""" + code = "" + desc = "" + if isinstance(fuel_raw, dict): + code = str(fuel_raw.get("code") or "").upper() + desc = str(fuel_raw.get("description") or "").upper() + elif isinstance(fuel_raw, str): + code = fuel_raw.upper() + desc = fuel_raw.upper() + + combined = f"{code} {desc}" + if "HYBRID" in combined or code == "H": + if "MILD" in combined or "MHEV" in combined: + return FuelType.MHEV + return FuelType.PHEV + if "MILD" in combined or "MHEV" in combined: + return FuelType.MHEV + if code == "E" or "ELEKTRO" in combined or "ELECTRIC" in combined: + return FuelType.BEV + if code == "D" or "DIESEL" in combined or "TDI" in combined: + return FuelType.DIESEL + if code == "B" or "BENZIN" in combined or "TFSI" in combined or "PETROL" in combined: + return FuelType.PETROL + if "CNG" in combined or "ERDGAS" in combined or "G-TRON" in combined: + return FuelType.CNG + if "LPG" in combined or "AUTOGAS" in combined: + return FuelType.LPG + return FuelType.PETROL + + +def map_audi_transmission(trans_raw: Any) -> Optional[TransmissionType]: + """Maps Audi transmission code / description to TransmissionType.""" + desc = "" + if isinstance(trans_raw, dict): + desc = str(trans_raw.get("description") or trans_raw.get("code") or "").upper() + elif isinstance(trans_raw, str): + desc = trans_raw.upper() + + if "S TRONIC" in desc or "TIPTRONIC" in desc or "AUTOMATIK" in desc or "AUTOMATIC" in desc or "DIRECT_DRIVE" in desc: + return TransmissionType.AUTOMATIC + if "HANDSCHALTUNG" in desc or "MANUELL" in desc or "SCHALT" in desc or "MANUAL" in desc: + return TransmissionType.MANUAL + return None + + +def map_audi_drivetrain(drive_raw: Any) -> Optional[DrivetrainType]: + """Maps Audi drivetrain description/code to DrivetrainType.""" + desc = "" + if isinstance(drive_raw, dict): + desc = str(drive_raw.get("description") or drive_raw.get("code") or "").upper() + elif isinstance(drive_raw, str): + desc = drive_raw.upper() + + if "QUATTRO" in desc or "ALLRAD" in desc or "AWD" in desc or "4X4" in desc: + return DrivetrainType.AWD + if "FRONT" in desc or "FWD" in desc: + return DrivetrainType.FWD + if "HECK" in desc or "RWD" in desc or "REAR" in desc: + return DrivetrainType.RWD + return None + + +def map_audi_series(carline_raw: Any, model_desc: Optional[str] = None) -> str: + """Extracts standardized Audi model series (e.g. A3, A4, Q4 e-tron, e-tron GT).""" + carline = "" + if isinstance(carline_raw, dict): + carline = str(carline_raw.get("description") or carline_raw.get("code") or "").strip() + elif isinstance(carline_raw, str): + carline = carline_raw.strip() + + if carline: + return carline + + desc = (model_desc or "").strip() + # Check known Audi series names + known_series = [ + "e-tron GT", "RS e-tron GT", "Q4 e-tron", "Q6 e-tron", "Q8 e-tron", "e-tron", + "RS3", "RS4", "RS5", "RS6", "RS7", "RSQ3", "RSQ8", + "S1", "S3", "S4", "S5", "S6", "S7", "S8", "SQ2", "SQ5", "SQ7", "SQ8", + "A1", "A3", "A4", "A5", "A6", "A7", "A8", + "Q2", "Q3", "Q5", "Q7", "Q8", "TT", "R8" + ] + for s in known_series: + pattern = r"\b" + re.escape(s) + r"\b" + if re.search(pattern, desc, re.I): + return s + + return "Audi" + + +def parse_power(power_display: Optional[str]) -> tuple[Optional[int], Optional[int]]: + """ + Parses 'powerDisplay' string, e.g. '110 kW (150 PS)' or '70 kW (95 PS)'. + Returns (power_kw, power_hp). + """ + if not power_display: + return None, None + kw = None + hp = None + kw_match = re.search(r"(\d+)\s*kW", power_display, re.I) + if kw_match: + try: + kw = int(kw_match.group(1)) + except Exception: + pass + + hp_match = re.search(r"(\d+)\s*PS", power_display, re.I) + if hp_match: + try: + hp = int(hp_match.group(1)) + except Exception: + pass + elif kw: + hp = int(round(kw * 1.35962)) + + return kw, hp + + +def parse_audi_vehicle( + basic_item: Dict[str, Any], + detail_data: Optional[Dict[str, Any]] = None, + market_country: str = "DE", + requested_stock_type: str = "new", + include_raw: Optional[bool] = None, +) -> VehicleIngestSchema: + """ + Parses an Audi vehicle from search vehicleBasic object and optional detail payload. + """ + if include_raw is None: + include_raw = settings.ENABLE_RAW_EXTRACTION + vehicle_id = str(basic_item.get("id") or "").strip() + car_id = str(basic_item.get("carId") or basic_item.get("decodedCarId") or vehicle_id).strip() + + model_info = basic_item.get("model") or {} + model_name = model_info.get("description") or "Audi" + model_code = model_info.get("code") + + series_info = basic_item.get("symbolicCarlineGroup") or basic_item.get("symbolicCarline") + series = map_audi_series(series_info, model_name) + + body_type_info = basic_item.get("bodyType") or {} + body_type = body_type_info.get("description") or None + + # Condition determination + used_dict = basic_item.get("used") or {} + is_used_market = requested_stock_type.lower() == "used" or bool(used_dict.get("mileage") or used_dict.get("initialRegistration")) + + if is_used_market: + # Check if certified pre-owned (Audi Gebrauchtwagen :plus) + warranty_plus = basic_item.get("warrantyPlus", False) + condition = VehicleCondition.CERTIFIED_PRE_OWNED if warranty_plus else VehicleCondition.USED + else: + condition = VehicleCondition.NEW + + # Powertrain & Specs + fuel_info = basic_item.get("fuel") or {} + fuel_type = map_audi_fuel_type(fuel_info) + gear_info = basic_item.get("gearType") or basic_item.get("gearBox") or {} + transmission = map_audi_transmission(gear_info) + drive_info = basic_item.get("driveType") or {} + drivetrain = map_audi_drivetrain(drive_info) + + power_kw, power_hp = parse_power(basic_item.get("powerDisplay")) + model_year = safe_int(basic_item.get("modelYear")) + + # Used vehicle specifics + mileage_val = safe_int(used_dict.get("mileage")) + mileage_km = mileage_val if mileage_val is not None else 0 + + first_reg = parse_german_date(used_dict.get("initialRegistration") or used_dict.get("initialRegistrationDate")) + prev_owners = safe_int(used_dict.get("numPreviousOwners")) + + # Pricing & Rates + typed_prices = basic_item.get("typedPrices") or [] + price_gross = None + monthly_rate = None + currency = "EUR" + + for tp in typed_prices: + p_type = tp.get("type", "").lower() + amt = safe_decimal(tp.get("amount")) + if tp.get("currencyCode"): + currency = tp["currencyCode"] + + if p_type in ("retail", "salesprice", "standardprice"): + if amt and (price_gross is None or amt > price_gross): + price_gross = amt + elif p_type in ("financerate", "rate", "leasingrate"): + if amt and (monthly_rate is None or amt < monthly_rate): + monthly_rate = amt + + # Fallback to any positive amount if retail was not tagged + if price_gross is None: + amounts = [safe_decimal(tp.get("amount")) for tp in typed_prices if safe_decimal(tp.get("amount"))] + if amounts: + price_gross = max(amounts) + + if price_gross is None: + price_gross = Decimal("0.00") + + # Dealer / GPS Geolocation + dealer_info = basic_item.get("dealer") or {} + dealer_name = dealer_info.get("name") + dealer_id = str(dealer_info.get("id") or "").strip() or None + postal_code = dealer_info.get("zipCode") + city = dealer_info.get("city") + + geo_loc = dealer_info.get("geoLocation") or {} + lat_val = safe_decimal(geo_loc.get("lat")) + lon_val = safe_decimal(geo_loc.get("lon")) + + # Images + image_urls: List[str] = [] + # Collect tile and picture URLs + tiles = basic_item.get("tilesPictures") or [] + for tile in tiles: + if isinstance(tile, str) and tile.startswith("http") and tile not in image_urls: + image_urls.append(tile) + + pics = basic_item.get("pictures") or [] + for pic in pics: + if isinstance(pic, str) and pic.startswith("http") and pic not in image_urls: + image_urls.append(pic) + + used_pics = used_dict.get("pictureUrls") or [] + for upic in used_pics: + if isinstance(upic, str) and upic.startswith("http") and upic not in image_urls: + image_urls.append(upic) + + thumbnail_url = image_urls[0] if image_urls else None + + # Vehicle Web URL + portal = "gebrauchtwagenboerse" if is_used_market else "neuwagenboerse" + vehicle_url = f"https://www.audi.de/de/{portal}/fahrzeugdetails/?vehicleid={vehicle_id}" + + # Colors + ext_color_info = basic_item.get("extColor") or {} + ext_color = ext_color_info.get("description") if isinstance(ext_color_info, dict) else str(ext_color_info or "") or None + ext_color_code = ext_color_info.get("code") if isinstance(ext_color_info, dict) else None + + int_color_info = basic_item.get("intColor") or {} + int_color = int_color_info.get("description") if isinstance(int_color_info, dict) else str(int_color_info or "") or None + + # Technical Specs & Options (Enriched via detail payload if present) + battery_capacity = None + battery_capacity_unit = None + battery_capacity_gross_kwh = None + battery_capacity_net_kwh = None + charging_power_ac_kw = None + charging_power_dc_kw = None + charging_time_dc_minutes = None + electric_range_wltp_km = None + disp_cc = None + options: List[OptionSchema] = [] + + raw_payload: Dict[str, Any] = {"vehicleBasic": basic_item} if include_raw else {} + + if detail_data: + if include_raw: + raw_payload["detail"] = detail_data + detail_obj = detail_data.get("vehicle", {}).get("detail", {}) if "vehicle" in detail_data else detail_data + + # 1. Technical specifications from techData + tech_data = detail_obj.get("techData", {}) + fields = tech_data.get("fields", []) + for f in fields: + f_key = (f.get("key") or "").lower() + f_name = (f.get("name") or "").strip() + f_val = str(f.get("value") or "").strip() + + # Displacement + if f_key == "displacement" or "hubraum" in f_name.lower(): + disp_match = re.search(r"(\d+[\.\d+]*)", f_val.replace(".", "")) + if disp_match: + disp_cc = safe_int(disp_match.group(1)) + + # Battery Capacity Net + elif "netto-batterie-kapazit" in f_name.lower() or "netto-energieinhalt" in f_name.lower(): + cap_m = re.search(r"([0-9]+[,\.][0-9]+|[0-9]+)\s*kwh", f_val, re.I) + if cap_m: + val_dec = safe_decimal(cap_m.group(1)) + if val_dec: + battery_capacity_net_kwh = val_dec + battery_capacity = val_dec + battery_capacity_unit = BatteryCapacityUnit.KWH + + # Battery Capacity Gross + elif "brutto-batterie-kapazit" in f_name.lower() or "brutto-energieinhalt" in f_name.lower(): + cap_m = re.search(r"([0-9]+[,\.][0-9]+|[0-9]+)\s*kwh", f_val, re.I) + if cap_m: + val_dec = safe_decimal(cap_m.group(1)) + if val_dec: + battery_capacity_gross_kwh = val_dec + + # DC Charging Time (10-80%) + elif "ladedauer" in f_name.lower() and "dc" in f_name.lower(): + time_m = re.search(r"(\d+)\s*min", f_val, re.I) + if time_m: + charging_time_dc_minutes = safe_int(time_m.group(1)) + + # Charging Power AC / HPC (DC) + elif "ladedauer" not in f_name.lower() and ("max. ladeleistung" in f_name.lower() or "ladeleistung ac / hpc" in f_name.lower()): + # e.g. "11 kW / 160 kW" or "22 kW / 270 kW" + rates = re.findall(r"([0-9]+[,\.][0-9]+|[0-9]+)\s*kw", f_val, re.I) + if len(rates) >= 2: + charging_power_ac_kw = safe_decimal(rates[0]) + charging_power_dc_kw = safe_decimal(rates[1]) + elif len(rates) == 1: + charging_power_dc_kw = safe_decimal(rates[0]) + + # Electric WLTP Range + elif "elektrische reichweite" in f_name.lower() or ("reichweite" in f_name.lower() and "wltp" in f_name.lower()): + range_m = re.search(r"(\d+)\s*km", f_val, re.I) + if range_m: + electric_range_wltp_km = safe_int(range_m.group(1)) + + # 2. Options and PR Equipment from detail.features + features = detail_obj.get("features", []) + for feat in features: + f_key = feat.get("key") or feat.get("prno") + f_name = feat.get("name") + if not f_name and feat.get("texts"): + for t in feat["texts"]: + if t.get("text"): + f_name = t["text"] + break + f_name = f_name or f_key or "Ausstattung" + group_name = feat.get("groupName") or feat.get("parentGroupName") + kind = (feat.get("kind") or "").lower() + is_standard = kind == "serie" + + options.append( + OptionSchema( + option_code=f_key, + name=f_name, + category=group_name, + is_standard=is_standard, + raw_data={"prno": feat.get("prno"), "kind": feat.get("kind")} if include_raw else {}, + ) + ) + + # If pure EV, assign standard fallback charging power if not already provided + if fuel_type == FuelType.BEV: + if not charging_power_ac_kw: + charging_power_ac_kw = Decimal("11.00") + if not charging_power_dc_kw: + if "e-tron gt" in model_name.lower(): + charging_power_dc_kw = Decimal("270.00") + elif "q6" in model_name.lower(): + charging_power_dc_kw = Decimal("270.00") + elif "q8" in model_name.lower(): + charging_power_dc_kw = Decimal("170.00") + elif "q4" in model_name.lower(): + charging_power_dc_kw = Decimal("160.00") + + trim_info = basic_item.get("trimline") + trim_line = None + if isinstance(trim_info, dict): + trim_line = trim_info.get("description") or trim_info.get("code") + elif isinstance(trim_info, str): + trim_line = trim_info + if not trim_line: + trim_line = model_code + + return VehicleIngestSchema( + brand="AUDI", + oem_vehicle_id=car_id, + vin=None, + series=series, + model_name=model_name, + trim_line=trim_line, + body_type=body_type, + condition=condition, + model_year=model_year, + first_registration_date=first_reg, + previous_owners_count=prev_owners, + fuel_type=fuel_type, + transmission=transmission, + drivetrain=drivetrain, + power_kw=power_kw, + power_hp=power_hp, + engine_displacement_cc=disp_cc, + battery_capacity=battery_capacity, + battery_capacity_unit=battery_capacity_unit, + battery_capacity_gross_kwh=battery_capacity_gross_kwh, + battery_capacity_net_kwh=battery_capacity_net_kwh, + charging_power_ac_kw=charging_power_ac_kw, + charging_power_dc_kw=charging_power_dc_kw, + charging_time_dc_minutes=charging_time_dc_minutes, + electric_range_wltp_km=electric_range_wltp_km, + mileage_km=mileage_km, + exterior_color=ext_color, + exterior_color_code=ext_color_code, + interior_color=int_color, + currency=currency, + price_gross=price_gross, + monthly_rate=monthly_rate, + status=VehicleStatus.IN_STOCK, + dealer_name=dealer_name, + dealer_id=dealer_id, + location_city=city, + location_postal_code=postal_code, + location_country=market_country, + latitude=lat_val, + longitude=lon_val, + vehicle_url=vehicle_url, + image_urls=image_urls, + thumbnail_url=thumbnail_url, + options=options, + raw_payload=raw_payload, + ) diff --git a/src/oem_crawler/crawlers/base.py b/src/oem_crawler/crawlers/base.py new file mode 100644 index 0000000..9588e1d --- /dev/null +++ b/src/oem_crawler/crawlers/base.py @@ -0,0 +1,86 @@ +""" +Base Crawler Abstract Interface and Crawler Registry. +Allows adding new OEM crawlers (Mercedes, Audi, Tesla, VW, etc.) modularly. +""" +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any, Callable, Dict, Generator, Optional, Type + +from oem_crawler.db.schemas import VehicleIngestSchema + + +class BaseCrawler(ABC): + """Abstract base class for OEM car stock crawlers.""" + + def __init__( + self, + market_country: str = "DE", + proxy: Optional[str] = None, + request_delay: float = 1.0, + timeout: float = 25.0, + **kwargs: Any, + ): + self.market_country = market_country.upper() + self.proxy = proxy + self.request_delay = request_delay + self.timeout = timeout + self.extra_kwargs = kwargs + + @property + @abstractmethod + def brand_name(self) -> str: + """Returns the standardized brand name (e.g. 'BMW', 'MERCEDES', 'AUDI').""" + pass + + @abstractmethod + def crawl( + self, + max_vehicles: Optional[int] = None, + page_size: int = 12, + search_filter: Optional[Dict[str, Any]] = None, + ) -> Generator[VehicleIngestSchema, None, None]: + """ + Main crawling generator yielding normalized VehicleIngestSchema objects. + """ + pass + + +class CrawlerRegistry: + """Registry to register and dynamically instantiate OEM crawlers.""" + + _registry: Dict[str, Type[BaseCrawler]] = {} + + @classmethod + def register(cls, brand: str) -> Callable[[Type[BaseCrawler]], Type[BaseCrawler]]: + """Decorator to register a crawler class for a brand.""" + def decorator(subclass: Type[BaseCrawler]) -> Type[BaseCrawler]: + key = brand.strip().lower() + cls._registry[key] = subclass + return subclass + return decorator + + @classmethod + def get(cls, brand: str) -> Type[BaseCrawler]: + """Retrieves crawler class for a given brand name.""" + key = brand.strip().lower() + if key not in cls._registry: + supported = ", ".join(cls.supported_brands()) + raise ValueError(f"No crawler registered for brand '{brand}'. Supported brands: {supported}") + return cls._registry[key] + + @classmethod + def supported_brands(cls) -> list[str]: + """Lists all registered OEM brands.""" + return sorted(list(cls._registry.keys())) + + @classmethod + def unique_brand_crawlers(cls) -> list[tuple[str, Type[BaseCrawler]]]: + """Returns unique (brand_key, crawler_cls) pairs avoiding alias duplicates.""" + seen_classes = set() + unique = [] + for brand, crawler_cls in sorted(cls._registry.items()): + if crawler_cls not in seen_classes: + seen_classes.add(crawler_cls) + unique.append((brand, crawler_cls)) + return unique diff --git a/src/oem_crawler/crawlers/bmw/__init__.py b/src/oem_crawler/crawlers/bmw/__init__.py new file mode 100644 index 0000000..41669cd --- /dev/null +++ b/src/oem_crawler/crawlers/bmw/__init__.py @@ -0,0 +1,3 @@ +from oem_crawler.crawlers.bmw.crawler import BMWStockCrawler + +__all__ = ["BMWStockCrawler"] diff --git a/src/oem_crawler/crawlers/bmw/crawler.py b/src/oem_crawler/crawlers/bmw/crawler.py new file mode 100644 index 0000000..7f4de1c --- /dev/null +++ b/src/oem_crawler/crawlers/bmw/crawler.py @@ -0,0 +1,178 @@ +""" +BMW Stock Crawler Implementation. +Queries BMW AWS STOLO REST API with pagination, retry, and rate-limit backoff. +""" +from __future__ import annotations + +import logging +import time +from typing import Any, Dict, Generator, Optional + +import httpx + +from oem_crawler.crawlers.base import BaseCrawler, CrawlerRegistry +from oem_crawler.crawlers.bmw.parser import parse_bmw_vehicle +from oem_crawler.db.schemas import VehicleIngestSchema + +logger = logging.getLogger(__name__) + + +@CrawlerRegistry.register("bmw") +class BMWStockCrawler(BaseCrawler): + ENDPOINTS = { + "new": "https://stolo-data-service.prod.stolo.eu-central-1.aws.bmw.cloud/vehiclesearch/search/de-de/neuwagensuche", + "used": "https://stolo-data-service.prod.stolo.eu-central-1.aws.bmw.cloud/vehiclesearch/search/de-de/gebrauchtwagen", + } + DEFAULT_API_KEY = "7f665f5b3cb8fe8e83293052367c575f666078b87570d5557f02f248ec98" + DEFAULT_HASH = "44c98fad4834174d354bb294444a3f52af15e9b206f41a0e58c461050d00c8ff" + + def __init__( + self, + market_country: str = "DE", + proxy: Optional[str] = None, + request_delay: float = 1.0, + timeout: float = 25.0, + api_key: Optional[str] = None, + stock_type: str = "new", + **kwargs: Any, + ): + super().__init__( + market_country=market_country, + proxy=proxy, + request_delay=request_delay, + timeout=timeout, + **kwargs, + ) + self.api_key = api_key or self.DEFAULT_API_KEY + self.stock_type = stock_type.lower() + self.headers = { + "Accept": "application/json, text/plain, */*", + "Accept-Language": "de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7", + "Content-Type": "application/json", + "Origin": "https://www.bmw.de", + "Referer": "https://www.bmw.de/", + "Sec-Ch-Ua": '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"', + "Sec-Ch-Ua-Mobile": "?0", + "Sec-Ch-Ua-Platform": '"macOS"', + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "cross-site", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", + "X-Api-Key": self.api_key, + } + + @property + def brand_name(self) -> str: + return "BMW" + + def fetch_page( + self, + start_index: int = 0, + max_results: int = 12, + search_filter: Optional[Dict[str, Any]] = None, + stock_type: str = "new", + max_retries: int = 4, + ) -> Dict[str, Any]: + """ + Fetches a single page of vehicle stock from BMW's REST search API with exponential backoff on 429s. + Note: BMW limits maxResults to a maximum of 12 per request. + """ + max_results = min(max_results, 12) + endpoint = self.ENDPOINTS.get(stock_type, self.ENDPOINTS["new"]) + params = { + "maxResults": max_results, + "startIndex": start_index, + "brand": "BMW", + "context": "results-page", + } + payload = { + "searchContext": [search_filter or {}], + "resultsContext": { + "sort": [{"by": "SF_OFFER_INSTALLMENT", "order": "ASC"}] + }, + } + + with httpx.Client(timeout=self.timeout, proxy=self.proxy) as client: + for attempt in range(max_retries + 1): + response = client.post( + endpoint, + params=params, + headers=self.headers, + json=payload, + ) + if response.status_code == 429: + wait_time = 2.0 * (attempt + 1) + logger.warning( + f"BMW rate limit (HTTP 429). Retrying in {wait_time:.1f}s (attempt {attempt+1}/{max_retries})..." + ) + time.sleep(wait_time) + continue + + if response.status_code in (400, 403) and start_index >= 1000: + logger.info( + f"BMW search API reached maximum pagination window at offset {start_index} (HTTP {response.status_code}). Ending category." + ) + return {"hits": [], "metadata": {"totalCount": start_index}} + + response.raise_for_status() + return response.json() + + response.raise_for_status() + return response.json() + + def crawl( + self, + max_vehicles: Optional[int] = None, + page_size: int = 12, + search_filter: Optional[Dict[str, Any]] = None, + ) -> Generator[VehicleIngestSchema, None, None]: + """ + Iterates over pages, fetches raw vehicle data, parses and yields normalized VehicleIngestSchema objects. + Supports stock_type: 'new', 'used', or 'all'. + """ + stock_types = ["new", "used"] if self.stock_type == "all" else [self.stock_type] + page_size = min(page_size, 12) + total_yielded = 0 + + for current_type in stock_types: + start_index = 0 + total_count = None + logger.info(f"Starting BMW stock crawl for category: {current_type.upper()}...") + + while True: + logger.info(f"Fetching BMW ({current_type}) stock: offset={start_index}, pageSize={page_size}...") + data = self.fetch_page( + start_index=start_index, + max_results=page_size, + search_filter=search_filter, + stock_type=current_type, + ) + + if total_count is None: + total_count = data.get("metadata", {}).get("totalCount", 0) + logger.info(f"BMW reports {total_count} total {current_type} vehicles in stock.") + + hits = data.get("hits", []) + if not hits: + break + + for hit in hits: + raw_veh = hit.get("vehicle", {}) + if not raw_veh: + continue + + try: + parsed = parse_bmw_vehicle(raw_veh, market_country=self.market_country, stock_type=current_type) + yield parsed + total_yielded += 1 + if max_vehicles and total_yielded >= max_vehicles: + return + except Exception as e: + doc_id = raw_veh.get("documentId", "UNKNOWN") + logger.error(f"Failed to parse vehicle {doc_id}: {e}", exc_info=True) + + start_index += len(hits) + if start_index >= total_count: + break + + time.sleep(self.request_delay) diff --git a/src/oem_crawler/crawlers/bmw/parser.py b/src/oem_crawler/crawlers/bmw/parser.py new file mode 100644 index 0000000..60a333e --- /dev/null +++ b/src/oem_crawler/crawlers/bmw/parser.py @@ -0,0 +1,292 @@ +""" +Parser and normalization logic for BMW OEM Neuwagensuche responses. +""" +from __future__ import annotations + +import logging +from decimal import Decimal +from typing import Any, Dict, List, Optional + +from oem_crawler.config import settings +from oem_crawler.db.schemas import ( + DrivetrainType, + FuelType, + OptionSchema, + TransmissionType, + VehicleCondition, + VehicleIngestSchema, + VehicleStatus, +) + +logger = logging.getLogger(__name__) + + +def safe_decimal(val: Any) -> Optional[Decimal]: + """Extracts a Decimal from numeric, string, or nested dict value safely.""" + if val is None: + return None + if isinstance(val, dict): + val = val.get("value") + if val is None: + return None + try: + val_str = str(val).strip() + if not val_str: + return None + return Decimal(val_str) + except Exception: + return None + + +def map_fuel_type(raw_fuel: str, is_48v: bool) -> FuelType: + raw_fuel = (raw_fuel or "").upper() + if "ELECTRIC" in raw_fuel or raw_fuel == "BEV": + return FuelType.BEV + if "PLUGIN" in raw_fuel or "PHEV" in raw_fuel or raw_fuel == "HYBRID": + return FuelType.PHEV + if is_48v or "MILD" in raw_fuel: + return FuelType.MHEV + if "DIESEL" in raw_fuel: + return FuelType.DIESEL + if "GASOLINE" in raw_fuel or "BENZIN" in raw_fuel or "PETROL" in raw_fuel: + return FuelType.PETROL + return FuelType.PETROL + + +def map_transmission(raw_trans: str) -> Optional[TransmissionType]: + raw_trans = (raw_trans or "").upper() + if "AUTOMATIC" in raw_trans or "AUT" in raw_trans: + return TransmissionType.AUTOMATIC + if "MANUAL" in raw_trans: + return TransmissionType.MANUAL + if "DIRECT" in raw_trans: + return TransmissionType.DIRECT_DRIVE + return None + + +def map_drivetrain(raw_drive: str) -> Optional[DrivetrainType]: + raw_drive = (raw_drive or "").upper() + if "XDRIVE" in raw_drive or "AWD" in raw_drive or "4X4" in raw_drive: + return DrivetrainType.AWD + if "RWD" in raw_drive: + return DrivetrainType.RWD + if "SDRIVE" in raw_drive or "FWD" in raw_drive: + return DrivetrainType.FWD + return None + + +def parse_bmw_vehicle( + raw: Dict[str, Any], + market_country: str = "DE", + stock_type: str = "new", + include_raw: Optional[bool] = None, +) -> VehicleIngestSchema: + """ + Transforms raw OEM BMW JSON into the normalized VehicleIngestSchema for both New and Used vehicles. + """ + if include_raw is None: + include_raw = settings.ENABLE_RAW_EXTRACTION + spec = raw.get("vehicleSpecification", {}) + model_and_opt = spec.get("modelAndOption", {}) + model_info = model_and_opt.get("model", {}) + model_range = model_and_opt.get("modelRange", {}) + tech_data = spec.get("technicalAndEmission", {}).get("technicalData", {}) + pricing = raw.get("pricing", {}) + raw_price = raw.get("price", {}) + ordering = raw.get("ordering", {}) + retail = ordering.get("retailData", {}) + outlet_addr = retail.get("locationOutletAddress", {}) + life_cycle = raw.get("vehicleLifeCycle", {}) + + # Identifiers + document_id = str(raw.get("documentId")) + vin = raw.get("vin") or raw.get("internal", {}).get("vin") + + # Model & Classification + series_name = ( + model_range.get("description", {}).get("de_DE") + or model_range.get("description", {}).get("default_DE") + or model_and_opt.get("series", {}).get("name") + or "BMW" + ) + model_name = ( + model_info.get("modelName") + or model_info.get("modelDescription", {}).get("de_DE") + or series_name + ) + trim_line = model_and_opt.get("marketingModelRange") + body_type = model_and_opt.get("bodyType") + + # Condition & Lifecycle (New vs Used) + if stock_type.lower() == "used" or life_cycle.get("mileage") or "LMP" in document_id: + condition = VehicleCondition.USED + else: + condition = VehicleCondition.NEW + + # Mileage + mileage_km = 0 + if life_cycle and isinstance(life_cycle.get("mileage"), dict): + mileage_km = int(life_cycle["mileage"].get("km", 0)) + + # Previous owners + previous_owners_count = life_cycle.get("numberOfPreviousOwners") + + # Registration & Production Date + first_registration_date = None + raw_reg_date = ( + retail.get("officialRegistrationDocDate") + or raw_price.get("priceRequestSpecification", {}).get("productionDate") + ) + if raw_reg_date and isinstance(raw_reg_date, str): + try: + from datetime import date + first_registration_date = date.fromisoformat(raw_reg_date.split("T")[0]) + except Exception: + pass + + vat_deductible = life_cycle.get("taxDeductible") + + # Powertrain mapping + fuel_type = map_fuel_type(model_and_opt.get("baseFuelType", ""), model_and_opt.get("is48Volt", False)) + transmission = map_transmission(model_and_opt.get("transmission", "")) + drivetrain = map_drivetrain(model_and_opt.get("marketingDriveType", "")) + + power_kw = None + power_hp = None + if "power" in tech_data and isinstance(tech_data["power"], dict): + power_kw = tech_data["power"].get("value") + if "powerInHp" in tech_data and isinstance(tech_data["powerInHp"], dict): + power_hp = tech_data["powerInHp"].get("value") + + # EV & Battery + electric_range_km = None + if "electricRange" in tech_data and isinstance(tech_data["electricRange"], dict): + electric_range_km = tech_data["electricRange"].get("value") + + # Colors & Upholstery + color_info = model_and_opt.get("color", {}) + exterior_color = color_info.get("name", {}).get("de_DE") or color_info.get("name", {}).get("default_DE") + exterior_color_code = color_info.get("code") + + # If exterior_color is empty, fallback to searching equipments for paint (codes like P0xxx or category Außenfarben) + if not exterior_color: + for code, eq in model_and_opt.get("equipments", {}).items(): + if isinstance(eq, dict) and (code.startswith("P0") or "Außenfarbe" in str(eq.get("marketingText", {})) or "Farbe" in str(eq.get("translatedSalesGroups", {}))): + exterior_color = eq.get("name", {}).get("de_DE") or eq.get("name", {}).get("default_DE") + exterior_color_code = code + break + + # Pricing + price_gross = safe_decimal(pricing.get("price", {}).get("value")) + if price_gross is None: + price_gross = safe_decimal(pricing.get("grossPrice")) or safe_decimal(raw_price.get("grossSalesPrice")) or safe_decimal(pricing.get("price")) + if price_gross is None: + raise ValueError(f"Vehicle {document_id} missing mandatory pricing.price.value") + + # Monthly installment + monthly_rate = None + sf_params = ( + pricing.get("monthlyInstallment", {}) + .get("selectedSfCalculation", {}) + .get("financialProductParameters", {}) + ) + if sf_params and "totalInstallment" in sf_params: + monthly_rate = safe_decimal(sf_params["totalInstallment"].get("amount")) + if monthly_rate is None: + monthly_rate = safe_decimal(pricing.get("monthlyInstallment", {}).get("value")) or safe_decimal(pricing.get("monthlyInstallment")) + + # MSRP / Original list price + msrp_gross = safe_decimal(raw_price.get("grossListPrice")) or safe_decimal(pricing.get("originalPrice", {}).get("value")) or safe_decimal(pricing.get("originalPrice")) + options_price_gross = safe_decimal(raw_price.get("equipmentsTotalListPriceGross")) + + discount_pct = None + if msrp_gross and msrp_gross > 0 and price_gross < msrp_gross: + discount_pct = Decimal(str(round(((msrp_gross - price_gross) / msrp_gross) * 100, 2))) + + # Dealer & Location + dealer_name = retail.get("locationOutletName") + dealer_id = retail.get("locationOutletId") or retail.get("localDealerNumber") + city = outlet_addr.get("town") + postal_code = outlet_addr.get("postalCode") + country = outlet_addr.get("countryCode") or market_country + + # Media & URLs + vehicle_url = f"https://www.bmw.de/de-de/sl/neuwagensuche/details/{document_id}" + image_urls = [] + thumbnail_url = None + + # Parse normalized options + options: List[OptionSchema] = [] + equipments = model_and_opt.get("equipments", {}) + for opt_code, opt_data in equipments.items(): + if not isinstance(opt_data, dict): + continue + + opt_name = ( + opt_data.get("name", {}).get("de_DE") + or opt_data.get("name", {}).get("default_DE") + or opt_code + ) + opt_cat = ( + opt_data.get("marketingText", {}).get("category", {}).get("de_DE") + or (opt_data.get("translatedSalesGroups", {}).get("de_DE") or [""])[0] + or None + ) + opt_desc = ( + opt_data.get("marketingText", {}).get("longText", {}).get("de_DE") + or opt_data.get("marketingText", {}).get("shortText", {}).get("de_DE") + ) + is_std = bool(opt_data.get("standard", False)) + + img_dict = opt_data.get("images", {}) + if "default" in img_dict: + image_urls.append(img_dict["default"]) + if not thumbnail_url: + thumbnail_url = img_dict["default"] + + options.append( + OptionSchema( + option_code=opt_code, + name=opt_name, + category=opt_cat, + is_standard=is_std, + description=opt_desc, + raw_data=opt_data if include_raw else {}, + ) + ) + + return VehicleIngestSchema( + brand="BMW", + oem_vehicle_id=document_id, + vin=vin, + series=series_name, + model_name=model_name, + trim_line=trim_line, + body_type=body_type, + condition=VehicleCondition.NEW, + fuel_type=fuel_type, + transmission=transmission, + drivetrain=drivetrain, + power_kw=power_kw, + power_hp=power_hp, + electric_range_wltp_km=electric_range_km, + mileage_km=0, + exterior_color=exterior_color, + exterior_color_code=exterior_color_code, + currency="EUR", + price_gross=price_gross, + msrp_gross=msrp_gross, + discount_percentage=discount_pct, + monthly_rate=monthly_rate, + status=VehicleStatus.IN_STOCK, + dealer_name=dealer_name, + dealer_id=dealer_id, + location_city=city, + location_postal_code=postal_code, + location_country=country, + vehicle_url=vehicle_url, + image_urls=image_urls[:10], + thumbnail_url=thumbnail_url, + options=options, + raw_payload=raw if include_raw else {}, + ) diff --git a/src/oem_crawler/crawlers/mercedes/__init__.py b/src/oem_crawler/crawlers/mercedes/__init__.py new file mode 100644 index 0000000..80f5640 --- /dev/null +++ b/src/oem_crawler/crawlers/mercedes/__init__.py @@ -0,0 +1,3 @@ +from oem_crawler.crawlers.mercedes.crawler import MercedesStockCrawler + +__all__ = ["MercedesStockCrawler"] diff --git a/src/oem_crawler/crawlers/mercedes/crawler.py b/src/oem_crawler/crawlers/mercedes/crawler.py new file mode 100644 index 0000000..49b8c0b --- /dev/null +++ b/src/oem_crawler/crawlers/mercedes/crawler.py @@ -0,0 +1,448 @@ +""" +Mercedes-Benz Stock Crawler Implementation. +Queries Mercedes-Benz OneSearch GraphQL and PLS/VPC microservices. +""" +from __future__ import annotations + +import logging +import time +import re +from typing import Any, Dict, Generator, Optional + +from bs4 import BeautifulSoup +import httpx + +from oem_crawler.crawlers.base import BaseCrawler, CrawlerRegistry +from oem_crawler.crawlers.mercedes.parser import ( + parse_mercedes_used_vehicle, + parse_mercedes_vehicle, +) +from oem_crawler.crawlers.mercedes.query import GRAPHQL_QUERY +from oem_crawler.db.schemas import VehicleIngestSchema + +logger = logging.getLogger(__name__) + + +@CrawlerRegistry.register("mercedes-benz") +@CrawlerRegistry.register("mercedes") +class MercedesStockCrawler(BaseCrawler): + GRAPHQL_ENDPOINT = "https://eu.api.oneweb.mercedes-benz.com/commerce/onesearch/graphql" + PRICE_ENDPOINT = "https://eu.api.oneweb.mercedes-benz.com/commerce/pls/public/v6/ncos/markets/{market}/prices/{code}" + VPC_ENDPOINT = "https://eu.api.oneweb.mercedes-benz.com/commerce/vpc/v2/markets/{market}/products/{code}" + USED_BASE_URL = "https://gebrauchtwagen.mercedes-benz.de" + + DEFAULT_API_KEY = "d1dcd3a9-25fd-4896-b041-4d35cfdbb482" + DEFAULT_CUSTOMER_GROUP_ID = "5e65a044-ac5a-47ea-ad26-b8948d4e450a" + + def __init__( + self, + market_country: str = "DE", + proxy: Optional[str] = None, + request_delay: float = 0.5, + timeout: float = 25.0, + api_key: Optional[str] = None, + stock_type: str = "new", + context_type: str = "B2C", + fetch_pricing: bool = True, + fetch_details: bool = True, + **kwargs: Any, + ): + super().__init__( + market_country=market_country, + proxy=proxy, + request_delay=request_delay, + timeout=timeout, + **kwargs, + ) + self.api_key = api_key or self.DEFAULT_API_KEY + self.stock_type = stock_type.lower() + self.context_type = context_type.upper() + self.fetch_pricing = fetch_pricing + self.fetch_details = fetch_details + + self.headers = { + "Accept": "application/json", + "Content-Type": "application/json", + "Origin": f"https://www.mercedes-benz.{self.market_country.lower()}", + "Referer": f"https://www.mercedes-benz.{self.market_country.lower()}/", + "Sec-Ch-Ua": '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"', + "Sec-Ch-Ua-Mobile": "?0", + "Sec-Ch-Ua-Platform": '"macOS"', + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "cross-site", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", + "x-api-key": self.api_key, + } + + @property + def brand_name(self) -> str: + return "MERCEDES-BENZ" + + def fetch_page( + self, + client: httpx.Client, + page: int = 0, + limit: int = 50, + search_filter: Optional[Dict[str, Any]] = None, + max_retries: int = 3, + ) -> Dict[str, Any]: + """ + Fetches a page of search results from Mercedes OneSearch GraphQL API. + Accepts up to limit=100. + """ + limit = max(1, min(limit, 100)) + is_used = self.stock_type == "used" + profile_id = f"{self.market_country.upper()}-USED_VEHICLES" if is_used else f"{self.market_country.upper()}-NEW_VEHICLES" + + variables: Dict[str, Any] = { + "profileId": profile_id, + "vehicleCategory": "PASSENGER-CARS", + "brand": ["Mercedes-Benz"], + "contextType": self.context_type, + "language": "de", + "limit": limit, + "page": page, + "sortingType": "name-asc", + "enableColorAggregation": False, + "isUcos": is_used, + "dealerId": [""], + } + if search_filter: + variables.update(search_filter) + + payload = { + "query": GRAPHQL_QUERY, + "variables": variables, + } + + for attempt in range(max_retries + 1): + try: + response = client.post(self.GRAPHQL_ENDPOINT, headers=self.headers, json=payload) + if response.status_code == 429: + wait_time = 2.0 * (attempt + 1) + logger.warning(f"Mercedes rate limit (HTTP 429). Retrying in {wait_time:.1f}s (attempt {attempt+1}/{max_retries})...") + time.sleep(wait_time) + continue + + response.raise_for_status() + data = response.json() + if "errors" in data and not data.get("data"): + raise ValueError(f"GraphQL errors: {data['errors']}") + return data + except httpx.HTTPError as e: + if attempt == max_retries: + raise + logger.warning(f"Network error while fetching page {page}: {e}. Retrying...") + time.sleep(1.5 * (attempt + 1)) + + raise RuntimeError(f"Failed to fetch Mercedes page {page} after {max_retries} retries") + + def fetch_price(self, client: httpx.Client, code: str, max_retries: int = 2) -> Optional[Dict[str, Any]]: + """Fetches pricing and financial terms from the PLS endpoint.""" + url = self.PRICE_ENDPOINT.format(market=self.market_country.upper(), code=code) + params = { + "customerType": self.context_type, + "customerGroupId": self.DEFAULT_CUSTOMER_GROUP_ID, + "language": "de", + } + pls_headers = { + **self.headers, + "Consumer-Id": "VEHICLE_TILE", + } + + for attempt in range(max_retries + 1): + try: + res = client.get(url, params=params, headers=pls_headers) + if res.status_code == 200: + return res.json() + elif res.status_code == 404: + return None + elif res.status_code == 429: + time.sleep(1.5 * (attempt + 1)) + continue + except Exception as e: + if attempt == max_retries: + logger.debug(f"Could not fetch price for vehicle {code}: {e}") + return None + time.sleep(1.0) + return None + + def fetch_vpc(self, client: httpx.Client, code: str, max_retries: int = 2) -> Optional[Dict[str, Any]]: + """Fetches detailed vehicle options, equipment, and colors from VPC endpoint.""" + url = self.VPC_ENDPOINT.format(market=self.market_country.upper(), code=code) + params = { + "language": "de", + "legacy": "true", + } + + for attempt in range(max_retries + 1): + try: + res = client.get(url, params=params, headers=self.headers) + if res.status_code == 200: + return res.json() + elif res.status_code == 404: + return None + elif res.status_code == 429: + time.sleep(1.5 * (attempt + 1)) + continue + except Exception as e: + if attempt == max_retries: + logger.debug(f"Could not fetch VPC configuration for vehicle {code}: {e}") + return None + time.sleep(1.0) + return None + + def fetch_used_page( + self, + client: httpx.Client, + page: int = 1, + sorting: str = "acdade", + filters: Optional[Dict[str, Any]] = None, + max_retries: int = 3, + ) -> str: + """Fetches a page of used vehicles HTML from gebrauchtwagen.mercedes-benz.de.""" + url = f"{self.USED_BASE_URL}/vehicles" + params: Dict[str, Any] = { + "page": page, + "sorting": sorting, + "view": "box", + } + if filters: + params.update(filters) + + used_headers = { + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "de-DE,de;q=0.9,en;q=0.8", + } + + for attempt in range(max_retries + 1): + try: + res = client.get(url, params=params, headers=used_headers) + if res.status_code == 429: + wait_time = 2.0 * (attempt + 1) + logger.warning(f"Mercedes used rate limit (HTTP 429). Retrying in {wait_time:.1f}s...") + time.sleep(wait_time) + continue + res.raise_for_status() + return res.text + except httpx.HTTPError as e: + if attempt == max_retries: + raise + logger.warning(f"Error fetching used cars page {page}: {e}. Retrying...") + time.sleep(1.5 * (attempt + 1)) + + raise RuntimeError(f"Failed to fetch Mercedes used cars page {page} after {max_retries} retries") + + def fetch_used_detail( + self, + client: httpx.Client, + vehicle_id: str, + max_retries: int = 2, + ) -> Optional[str]: + """Fetches detailed vehicle HTML page for a used car.""" + url = f"{self.USED_BASE_URL}/vehicle" + params = {"vehicle": vehicle_id, "referrer": "vehicles"} + used_headers = { + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "de-DE,de;q=0.9,en;q=0.8", + } + + for attempt in range(max_retries + 1): + try: + res = client.get(url, params=params, headers=used_headers) + if res.status_code == 200: + return res.text + elif res.status_code == 404: + return None + elif res.status_code == 429: + time.sleep(1.5 * (attempt + 1)) + continue + except Exception as e: + if attempt == max_retries: + logger.debug(f"Could not fetch used vehicle detail for {vehicle_id}: {e}") + return None + time.sleep(1.0) + return None + + def crawl_used( + self, + client: httpx.Client, + max_vehicles: Optional[int] = None, + page_size: int = 12, + search_filter: Optional[Dict[str, Any]] = None, + ) -> Generator[VehicleIngestSchema, None, None]: + """ + Crawls Mercedes-Benz used vehicles catalog from gebrauchtwagen.mercedes-benz.de. + """ + logger.info(f"Starting Mercedes-Benz used stock crawl (Market: {self.market_country})...") + current_page = 1 + total_yielded = 0 + filters = dict(search_filter or {}) + sorting = filters.pop("sorting", "acdade") + + while True: + logger.info(f"Fetching Mercedes used stock page {current_page} (sorting={sorting})...") + html = self.fetch_used_page( + client=client, + page=current_page, + sorting=sorting, + filters=filters, + ) + soup = BeautifulSoup(html, "html.parser") + cards = soup.select(".list-vehicle-card") + if not cards: + logger.info(f"No more vehicle cards found on page {current_page}. Used stock crawl complete.") + break + + for card in cards: + raw_id = card.get("id", "") + veh_id = re.sub(r"^resultListItem", "", raw_id).strip() + if not veh_id: + link = card.select_one("a[href*='vehicle=']") + if link: + m = re.search(r"vehicle=(\d+)", link.get("href", "")) + if m: + veh_id = m.group(1) + + detail_html = None + if self.fetch_details and veh_id: + detail_html = self.fetch_used_detail(client, veh_id) + if self.request_delay > 0: + time.sleep(self.request_delay) + + try: + parsed = parse_mercedes_used_vehicle( + card=card, + detail_html=detail_html, + market_country=self.market_country, + ) + yield parsed + total_yielded += 1 + + if max_vehicles and total_yielded >= max_vehicles: + logger.info(f"Reached max limit of {max_vehicles} used vehicles. Done.") + return + except Exception as e: + logger.error(f"Failed to parse Mercedes used vehicle {veh_id}: {e}", exc_info=True) + + current_page += 1 + if self.request_delay > 0: + time.sleep(self.request_delay) + + def crawl_new( + self, + client: httpx.Client, + max_vehicles: Optional[int] = None, + page_size: int = 50, + search_filter: Optional[Dict[str, Any]] = None, + ) -> Generator[VehicleIngestSchema, None, None]: + """ + Crawls Mercedes-Benz new vehicles stock via OneSearch GraphQL API. + """ + page_size = max(1, min(page_size, 100)) + total_yielded = 0 + current_page = 0 + total_results: Optional[int] = None + total_pages: Optional[int] = None + + logger.info(f"Starting Mercedes-Benz new stock crawl (Market: {self.market_country}, Context: {self.context_type})...") + + while True: + logger.info(f"Fetching Mercedes new stock page {current_page} (limit={page_size})...") + page_data = self.fetch_page( + client=client, + page=current_page, + limit=page_size, + search_filter=search_filter, + ) + + search_res = page_data.get("data", {}).get("search", {}) + nav = search_res.get("navigation", {}) + if total_results is None: + total_results = nav.get("totalResults", 0) + total_pages = nav.get("totalPages", 1) + logger.info(f"Mercedes reports {total_results} total new vehicles ({total_pages} pages).") + + results = search_res.get("results", []) + if not results: + break + + for raw_veh in results: + code = raw_veh.get("identification", {}).get("code") + if not code: + continue + + raw_price = None + if self.fetch_pricing: + raw_price = self.fetch_price(client, code) + + raw_vpc = None + if self.fetch_details: + raw_vpc = self.fetch_vpc(client, code) + + try: + parsed = parse_mercedes_vehicle( + raw_search=raw_veh, + raw_price=raw_price, + raw_vpc=raw_vpc, + market_country=self.market_country, + stock_type=self.stock_type, + ) + yield parsed + total_yielded += 1 + + if max_vehicles and total_yielded >= max_vehicles: + logger.info(f"Reached max limit of {max_vehicles} new vehicles. Done.") + return + except Exception as e: + logger.error(f"Failed to parse Mercedes vehicle {code}: {e}", exc_info=True) + + current_page += 1 + if total_pages is not None and current_page >= total_pages: + break + + if self.request_delay > 0: + time.sleep(self.request_delay) + + def crawl( + self, + max_vehicles: Optional[int] = None, + page_size: int = 50, + search_filter: Optional[Dict[str, Any]] = None, + ) -> Generator[VehicleIngestSchema, None, None]: + """ + Crawls Mercedes-Benz stock (new, used, or all), yields normalized VehicleIngestSchema objects. + """ + with httpx.Client(http2=False, timeout=self.timeout, proxy=self.proxy) as client: + if self.stock_type == "used": + yield from self.crawl_used( + client=client, + max_vehicles=max_vehicles, + page_size=page_size, + search_filter=search_filter, + ) + elif self.stock_type == "all": + limit_each = (max_vehicles // 2) if max_vehicles else None + yield from self.crawl_new( + client=client, + max_vehicles=limit_each, + page_size=page_size, + search_filter=search_filter, + ) + yield from self.crawl_used( + client=client, + max_vehicles=limit_each, + page_size=page_size, + search_filter=search_filter, + ) + else: + yield from self.crawl_new( + client=client, + max_vehicles=max_vehicles, + page_size=page_size, + search_filter=search_filter, + ) + diff --git a/src/oem_crawler/crawlers/mercedes/parser.py b/src/oem_crawler/crawlers/mercedes/parser.py new file mode 100644 index 0000000..2801f7f --- /dev/null +++ b/src/oem_crawler/crawlers/mercedes/parser.py @@ -0,0 +1,1008 @@ +""" +Parser and normalization logic for Mercedes-Benz OneWeb responses. +""" +from __future__ import annotations + +import logging +import re +from datetime import date, datetime +from decimal import Decimal +from typing import Any, Dict, List, Optional + +from bs4 import BeautifulSoup, Tag + +from oem_crawler.config import settings +from oem_crawler.db.schemas import ( + BatteryCapacityUnit, + DrivetrainType, + FuelType, + OptionSchema, + TransmissionType, + VehicleCondition, + VehicleIngestSchema, + VehicleStatus, +) + +logger = logging.getLogger(__name__) + + +def safe_decimal(val: Any) -> Optional[Decimal]: + """Safely extracts a Decimal from numeric, string, or dict values.""" + if val is None: + return None + if isinstance(val, dict): + val = val.get("value") or val.get("amount") or val.get("rawValue") + if val is None: + return None + try: + val_str = str(val).replace(",", ".").replace("€", "").replace("\xa0", "").strip() + if not val_str: + return None + return Decimal(val_str) + except Exception: + return None + + +def safe_int(val: Any) -> Optional[int]: + """Safely extracts an integer.""" + if val is None: + return None + if isinstance(val, dict): + val = val.get("value") + try: + val_str = str(val).split(".")[0].strip() + return int(val_str) + except Exception: + return None + + +def parse_date(val: Any) -> Optional[date]: + """Parses date string in ISO formats.""" + if not val or not isinstance(val, str): + return None + try: + clean = val.split("T")[0].strip() + return datetime.strptime(clean, "%Y-%m-%d").date() + except Exception: + return None + + +def map_mercedes_fuel_type(fuel_val: Optional[str], emission_attrs: List[Dict[str, Any]]) -> FuelType: + raw = (fuel_val or "").upper() + + # Check emission attributes for mild hybrid indicators + for attr in emission_attrs: + attr_id = str(attr.get("id", "")).upper() + if "BI_MILD" in attr_id or "MILD" in attr_id: + return FuelType.MHEV + + if "ELECTRIC" in raw or raw == "BEV": + return FuelType.BEV + if "PLUGIN" in raw or "PHEV" in raw or "HYBRID" in raw: + return FuelType.PHEV + if "DIESEL" in raw: + return FuelType.DIESEL + if "GASOLINE" in raw or "BENZIN" in raw or "PETROL" in raw: + return FuelType.PETROL + if "CNG" in raw: + return FuelType.CNG + if "LPG" in raw: + return FuelType.LPG + return FuelType.PETROL + + +def map_mercedes_transmission(trans_val: Optional[str]) -> Optional[TransmissionType]: + raw = (trans_val or "").upper() + if "AUTOMATIC" in raw or "AUT" in raw or "7G" in raw or "9G" in raw or "DCT" in raw: + return TransmissionType.AUTOMATIC + if "MANUAL" in raw or "SCHALT" in raw: + return TransmissionType.MANUAL + if "DIRECT" in raw: + return TransmissionType.DIRECT_DRIVE + return None + + +def map_mercedes_drivetrain(name_or_motor: str, type_class: Optional[str] = None) -> Optional[DrivetrainType]: + upper_name = name_or_motor.upper() + if "4MATIC" in upper_name or "AWD" in upper_name or "4X4" in upper_name: + return DrivetrainType.AWD + # A-Class, B-Class, CLA, GLA are typically FWD by default unless 4MATIC + if type_class and any(tc in type_class.upper() for tc in ["W177", "W247", "C118", "H247"]): + return DrivetrainType.FWD + # C-Class, E-Class, S-Class are RWD by default unless 4MATIC + if type_class and any(tc in type_class.upper() for tc in ["W206", "W213", "W214", "W223", "V295", "V297"]): + return DrivetrainType.RWD + return None + + +def extract_city_postal_code(address_str: Optional[str]) -> tuple[Optional[str], Optional[str]]: + """Extracts (city, postal_code) from address strings like 'Mercedesstraße 120, 70372 Stuttgart'.""" + if not address_str: + return None, None + # Match standard German pattern: comma, optional street, 5-digit postal code, city name + match = re.search(r",\s*(\d{5})\s+([A-Za-zΓ€ΓΆΓΌΓ„Γ–ΓœΓŸ\s\-]+)$", address_str.strip()) + if match: + return match.group(2).strip(), match.group(1).strip() + return None, None + + +def parse_mercedes_vehicle( + raw_search: Dict[str, Any], + raw_price: Optional[Dict[str, Any]] = None, + raw_vpc: Optional[Dict[str, Any]] = None, + market_country: str = "DE", + stock_type: str = "new", + include_raw: Optional[bool] = None, +) -> VehicleIngestSchema: + """ + Transforms raw Mercedes OneSearch, PLS price, and VPC configuration responses into + the normalized VehicleIngestSchema. + """ + if include_raw is None: + include_raw = settings.ENABLE_RAW_EXTRACTION + ident = raw_search.get("identification", {}) + code = ident.get("code") + if not code: + raise ValueError("Missing vehicle code in Mercedes search payload") + + vehicle_model = raw_search.get("vehicleModel", {}) + tech_info = raw_search.get("technicalInformation", {}) + engine = tech_info.get("engine", {}) + power = engine.get("power", {}) + dealer = raw_search.get("dealer", {}) + emission_data = raw_search.get("emissionAndConsumption", {}) + emission_attrs = emission_data.get("attributes", []) + characteristics = raw_search.get("characteristics", {}) + + # Identification + oem_vehicle_id = str(code) + vin = ident.get("vin") # May be None for factory stock + + # Model & Classification + brand = vehicle_model.get("brand", {}).get("formattedValue") or "Mercedes-Benz" + series = vehicle_model.get("vehicleClass", {}).get("formattedValue") or "Mercedes-Benz" + model_name = vehicle_model.get("name") or vehicle_model.get("motorization") or series + body_type = vehicle_model.get("bodyType", {}).get("formattedValue") + + # Trim Line (from highlights like 'Progressive', 'AMG Line') + trim_line = None + highlights = characteristics.get("highlights", []) + if highlights and isinstance(highlights, list): + labels = [h.get("label") for h in highlights if h.get("label")] + if labels: + trim_line = labels[0] + + # Model Year + model_year_code = vehicle_model.get("modelYearCode") + model_year = None + if vehicle_model.get("modelYear"): + model_year = safe_int(vehicle_model.get("modelYear")) + elif model_year_code: + # Mercedes code 806 corresponds to 2026, 805 to 2025, etc. + try: + code_num = int(str(model_year_code)[:2]) + model_year = 1946 + code_num # 80 -> 2026 + except Exception: + pass + + # Condition & Lifecycle + condition = VehicleCondition.NEW if stock_type.lower() == "new" else VehicleCondition.USED + first_registration_date = None + mileage_km = 0 + used_data = raw_search.get("usedVehicleData") + if used_data and isinstance(used_data, dict): + condition = VehicleCondition.USED + first_registration_date = parse_date(used_data.get("firstRegistrationDate")) + mileage_km = safe_int(used_data.get("mileage", {}).get("value")) or 0 + + # Powertrain & Technical Specs + raw_fuel = engine.get("fuelType", {}).get("value") + fuel_type = map_mercedes_fuel_type(raw_fuel, emission_attrs) + transmission = map_mercedes_transmission(tech_info.get("transmission", {}).get("value")) + drivetrain = map_mercedes_drivetrain(model_name, vehicle_model.get("typeClass")) + + # Power: combinedKw -> combustionKw -> electricKw + power_kw = ( + safe_int(power.get("combinedKw", {}).get("value")) + or safe_int(power.get("combustionKw", {}).get("value")) + or safe_int(power.get("electricKw", {}).get("value")) + ) + power_hp = ( + safe_int(power.get("combinedHp", {}).get("value")) + or safe_int(power.get("combustionHp", {}).get("value")) + or safe_int(power.get("electricHp", {}).get("value")) + ) + + # Emissions, Consumption & Range + consumption_combined = None + co2_emissions_g_km = None + electric_range_wltp_km = None + + for attr in emission_attrs: + attr_id = str(attr.get("id", "")).upper() + if attr_id in ["90W-BI_MILD", "90W", "237W"] or "ENERGIEVERBRAUCH" in str(attr.get("label", "")).upper(): + if consumption_combined is None: + consumption_combined = safe_decimal(attr.get("value")) + elif attr_id == "85W" or "CO2" in str(attr.get("label", "")).upper(): + if co2_emissions_g_km is None: + co2_emissions_g_km = safe_decimal(attr.get("value")) + elif attr_id == "193W" or "REICHWEITE" in str(attr.get("label", "")).upper(): + if electric_range_wltp_km is None: + electric_range_wltp_km = safe_int(attr.get("value")) + + # Battery & Charging Telemetry (from VPC if available) + battery_capacity = None + battery_capacity_unit = None + battery_capacity_net_kwh = None + battery_capacity_gross_kwh = None + charging_power_ac_kw = None + charging_power_dc_kw = None + charging_time_dc_minutes = None + + if raw_vpc and isinstance(raw_vpc, dict): + vpc_tech = raw_vpc.get("technicalInformation", {}) + vpc_engine = vpc_tech.get("engine", {}) + b_cap = vpc_engine.get("batteryCapacity") + if b_cap and isinstance(b_cap, dict): + b_val = safe_decimal(b_cap.get("value")) + b_unit_str = str(b_cap.get("unit", "")).strip().lower() + if b_val: + battery_capacity = b_val + if "ah" in b_unit_str: + battery_capacity_unit = BatteryCapacityUnit.AH + elif "kwh" in b_unit_str: + battery_capacity_unit = BatteryCapacityUnit.KWH + battery_capacity_net_kwh = b_val + + charging_power_ac_kw = safe_decimal(vpc_engine.get("chargingCapacityAC", {}).get("value")) + charging_power_dc_kw = safe_decimal(vpc_engine.get("chargingCapacityDC", {}).get("value")) + + charge_times = vpc_tech.get("chargeTime", {}) + charging_time_dc_minutes = safe_int(charge_times.get("chargeTimeDCHigh", {}).get("value")) + + # Colors & Materials (from VPC if available) + exterior_color = None + exterior_color_code = None + exterior_color_generic = None + interior_color = None + interior_material = None + + if raw_vpc and isinstance(raw_vpc, dict): + vpc_color = raw_vpc.get("color", {}) + if vpc_color: + exterior_color = vpc_color.get("text", {}).get("de") + exterior_color_code = vpc_color.get("id") + exterior_color_generic = vpc_color.get("group", {}).get("de") + + vpc_upholstery = raw_vpc.get("upholstery", {}) + if vpc_upholstery: + interior_color = vpc_upholstery.get("group", {}).get("de") + interior_material = vpc_upholstery.get("polish", {}).get("de") + + # Pricing & Financing (from PLS) + currency = "EUR" + price_gross = Decimal("0.00") + price_net = None + msrp_gross = None + base_price_gross = None + options_price_gross = None + discount_percentage = None + vat_deductible = None + monthly_rate = None + leasing_details = {} + + if raw_price and isinstance(raw_price, dict): + currency = raw_price.get("currency") or "EUR" + purch_amount = raw_price.get("purchasePrice", {}).get("amount") + if purch_amount: + price_gross = safe_decimal(purch_amount) or Decimal("0.00") + + # MSRP / List price + finance_info = raw_price.get("financeLeasingInfo", {}) + leasing_details = finance_info + veh_prices = finance_info.get("vehicle", {}).get("prices", []) + for p_item in veh_prices: + if p_item.get("id") == "grossListPrice": + msrp_gross = safe_decimal(p_item.get("rawValue")) + elif p_item.get("id") == "baseListPrice": + base_price_gross = safe_decimal(p_item.get("rawValue")) + + # Base price fallback from priceBreakDown + if not base_price_gross: + for b_item in raw_price.get("priceBreakDown", []): + if b_item.get("type") == "BASE_PRICE": + base_price_gross = safe_decimal(b_item.get("amount")) + break + + if msrp_gross and base_price_gross and msrp_gross > base_price_gross: + options_price_gross = msrp_gross - base_price_gross + + if price_gross and price_gross > 0: + price_net = round(price_gross / Decimal("1.19"), 2) + + if msrp_gross and price_gross and msrp_gross > 0: + discount_percentage = round((msrp_gross - price_gross) / msrp_gross * Decimal("100.0"), 2) + + vat_deductible = raw_price.get("taxProfile") == "GROSS" + + # Status & Availability + stock_val = raw_search.get("stock", {}).get("stockType", "").upper() + status = VehicleStatus.IN_STOCK + if "RESERVED" in stock_val: + status = VehicleStatus.RESERVED + elif "SOLD" in stock_val: + status = VehicleStatus.SOLD + + availability_date = parse_date(raw_search.get("estimatedArrivalDate")) + delivery_time_days = None + if availability_date: + today = date.today() + if availability_date >= today: + delivery_time_days = (availability_date - today).days + + # Location & Dealer + dealer_name = dealer.get("nameLocalLanguage") + dealer_id = dealer.get("outletId") + address_str = dealer.get("addressLocalLanguage") + location_city, location_postal_code = extract_city_postal_code(address_str) + + # Media & Links + vehicle_url = f"https://www.mercedes-benz.de/passengercars/buy/new-car/product.html/{code}" + thumbnail_url = raw_search.get("images", {}).get("default") + image_urls = [thumbnail_url] if thumbnail_url else [] + + # Options & Equipment + options: List[OptionSchema] = [] + if raw_vpc and isinstance(raw_vpc.get("components"), dict): + for comp_id, comp in raw_vpc["components"].items(): + name = comp.get("name", {}).get("de") or comp.get("text", {}).get("de") + if not name: + continue + options.append( + OptionSchema( + option_code=comp.get("code") or comp_id, + name=name, + category=comp.get("codeType"), + is_standard=bool(comp.get("standard", False)), + description=comp.get("description", {}).get("de"), + raw_data=comp, + ) + ) + else: + # Fallback to search highlights + for h in highlights: + lbl = h.get("label") + if lbl: + options.append(OptionSchema(name=lbl, is_standard=True, raw_data=h)) + + # Raw Payload for Audit & Telemetry + raw_payload = { + "search": raw_search, + "price": raw_price or {}, + "vpc": raw_vpc or {}, + } if include_raw else {} + + return VehicleIngestSchema( + brand=brand, + oem_vehicle_id=oem_vehicle_id, + vin=vin, + series=series, + model_name=model_name, + trim_line=trim_line, + body_type=body_type, + condition=condition, + model_year=model_year, + first_registration_date=first_registration_date, + previous_owners_count=0 if condition == VehicleCondition.NEW else None, + fuel_type=fuel_type, + transmission=transmission, + drivetrain=drivetrain, + power_kw=power_kw, + power_hp=power_hp, + battery_capacity=battery_capacity, + battery_capacity_unit=battery_capacity_unit, + battery_capacity_net_kwh=battery_capacity_net_kwh, + battery_capacity_gross_kwh=battery_capacity_gross_kwh, + charging_power_ac_kw=charging_power_ac_kw, + charging_power_dc_kw=charging_power_dc_kw, + charging_time_dc_minutes=charging_time_dc_minutes, + electric_range_wltp_km=electric_range_wltp_km, + consumption_combined=consumption_combined, + co2_emissions_g_km=co2_emissions_g_km, + mileage_km=mileage_km, + exterior_color=exterior_color, + exterior_color_code=exterior_color_code, + exterior_color_generic=exterior_color_generic, + interior_color=interior_color, + interior_material=interior_material, + currency=currency, + base_price_gross=base_price_gross, + options_price_gross=options_price_gross, + price_gross=price_gross, + price_net=price_net, + msrp_gross=msrp_gross, + discount_percentage=discount_percentage, + vat_deductible=vat_deductible, + monthly_rate=monthly_rate, + leasing_details=leasing_details, + status=status, + availability_date=availability_date, + delivery_time_days=delivery_time_days, + dealer_name=dealer_name, + dealer_id=dealer_id, + location_city=location_city, + location_postal_code=location_postal_code, + location_country=market_country, + vehicle_url=vehicle_url, + image_urls=image_urls, + thumbnail_url=thumbnail_url, + options=options, + raw_payload=raw_payload, + ) + + +# ============================================================================= +# Used Vehicles Parser (gebrauchtwagen.mercedes-benz.de) +# ============================================================================= + +def map_mercedes_used_fuel_type(fuel_val: Optional[str]) -> FuelType: + """Maps German fuel type strings from used car platform to FuelType enum.""" + raw = (fuel_val or "").strip().lower() + if "elektro" in raw or "electric" in raw or "bev" in raw: + return FuelType.BEV + if "plug-in" in raw or "phev" in raw or "hybrid" in raw: + return FuelType.PHEV + if "diesel" in raw: + return FuelType.DIESEL + if "benzin" in raw or "petrol" in raw or "gasoline" in raw: + return FuelType.PETROL + if "cng" in raw: + return FuelType.CNG + if "lpg" in raw: + return FuelType.LPG + return FuelType.PETROL + + +def map_mercedes_used_series(model_name: str) -> str: + """Maps Mercedes-Benz used model names to series classes.""" + cleaned = model_name.strip() + prefixes = [ + "EQA", "EQB", "EQE", "EQS", "EQV", + "A-Klasse", "B-Klasse", "C-Klasse", "E-Klasse", "S-Klasse", "G-Klasse", "V-Klasse", "T-Klasse", + "CLA", "CLE", "CLS", "GLA", "GLB", "GLC", "GLE", "GLS", "SL", "SLC", "SLK", + "AMG GT", "GT", "Citan", "Vito", "Sprinter", "Maybach", "smart" + ] + for p in prefixes: + if cleaned.startswith(p): + return p + + m = re.match(r"^([ABCEGSV])\s*\d+", cleaned) + if m: + return f"{m.group(1)}-Klasse" + + parts = cleaned.split() + return parts[0] if parts else "Mercedes-Benz" + + +def extract_mercedes_used_options(detail_soup: BeautifulSoup) -> List[OptionSchema]: + """Extracts equipment options and codes from the used vehicle detail page.""" + options: List[OptionSchema] = [] + seen = set() + + for el in detail_soup.select(".eq-search-container"): + eq_item = el.select_one(".eq-item") + if not eq_item: + continue + + code_span = eq_item.select_one("span") + code_str: Optional[str] = None + if code_span: + raw_codes = code_span.get_text(strip=True).replace("|", "").replace("\xa0", "").strip() + if raw_codes: + code_str = raw_codes + code_span.extract() + + opt_name = eq_item.get_text(" ", strip=True) + if not opt_name or (code_str, opt_name) in seen: + continue + seen.add((code_str, opt_name)) + + options.append( + OptionSchema( + option_code=code_str, + name=opt_name, + is_standard=False, + ) + ) + return options + + +def parse_mercedes_used_vehicle( + card: Any, + detail_html: Optional[str] = None, + market_country: str = "DE", + include_raw: Optional[bool] = None, +) -> VehicleIngestSchema: + """ + Parses a used car HTML card and optional detail page from gebrauchtwagen.mercedes-benz.de + into a validated VehicleIngestSchema. + """ + if include_raw is None: + include_raw = settings.ENABLE_RAW_EXTRACTION + if isinstance(card, str): + card_soup = BeautifulSoup(card, "html.parser").select_one(".list-vehicle-card") or BeautifulSoup(card, "html.parser") + else: + card_soup = card + + # 1. OEM Vehicle ID + raw_card_id = card_soup.get("id", "") + card_id = re.sub(r"^resultListItem", "", raw_card_id).strip() + if not card_id: + link_el = card_soup.select_one("a[href*='vehicle=']") + if link_el: + href = link_el.get("href", "") + m_id = re.search(r"vehicle=(\d+)", href) + if m_id: + card_id = m_id.group(1) + if not card_id and detail_html: + m_id = re.search(r"vehicle=(\d+)", detail_html) + if m_id: + card_id = m_id.group(1) + + # 2. Commission Number / Fahrzeugnummer + sub_el = card_soup.select_one(".vehicle_title_subheadline") + sub_text = sub_el.get_text(strip=True) if sub_el else "" + comm_no = re.sub(r"Fahrzeugnummer\s*", "", sub_text).strip() or None + + # 3. Model Name & Series + title_el = card_soup.select_one(".vehicle_title_headline") + manuf_el = card_soup.select_one(".manufacturer") + manuf = manuf_el.get_text(strip=True) if manuf_el else "Mercedes-Benz" + full_title = title_el.get_text(" ", strip=True) if title_el else "" + model_name = full_title.replace(manuf, "").strip() if full_title else "Mercedes-Benz" + model_name = re.sub(r"\s+", " ", model_name).strip() + + series = map_mercedes_used_series(model_name) + + # Trim Line detection + trim_line = None + for candidate in ["Avantgarde", "AMG Line", "Exclusive", "Progressive", "Style", "Edition 1", "Night-Paket"]: + if candidate.lower() in model_name.lower(): + trim_line = candidate + break + + # 4. Card Attributes + card_attrs: Dict[str, str] = {} + for li in card_soup.select(".vc-vehicle-attribute-row"): + use_tag = li.select_one("use") + href = (use_tag.get("xlink:href") or use_tag.get("href") or "") if use_tag else "" + text_el = li.select_one(".vc-vehicle-attribute-text") or li.select_one("p") + val = text_el.get_text(strip=True) if text_el else "" + if href and val: + card_attrs[href.replace("#", "")] = val + + body_type = card_attrs.get("vehicle-box-body-type") + reg_str = card_attrs.get("vehicle-box-first-registration", "") + power_str = card_attrs.get("vehicle-box-power", "") + mileage_str = card_attrs.get("vehicle-box-mileage", "") + fuel_str = card_attrs.get("vehicle-box-engine-ice") or card_attrs.get("vehicle-box-engine-hybrid") or card_attrs.get("vehicle-box-engine-electric") or "" + + # Parse power + power_kw, power_hp = None, None + m_p = re.search(r"(\d+)\s*kW\s*\((\d+)\s*PS\)", power_str.replace("\xa0", " ")) + if m_p: + power_kw = int(m_p.group(1)) + power_hp = int(m_p.group(2)) + + # Parse mileage + mileage_km = 0 + m_m = re.search(r"([0-9\.]+)\s*km", mileage_str.replace("\xa0", " ")) + if m_m: + mileage_km = int(m_m.group(1).replace(".", "")) + + # Parse first registration date + first_registration_date = None + if reg_str: + try: + first_registration_date = datetime.strptime(reg_str, "%d.%m.%Y").date() + except Exception: + pass + + # 5. Pricing & Financing + price_el = card_soup.select_one(".vehicle_price_headline") + price_text = price_el.get_text(strip=True) if price_el else "" + m_price = re.search(r"([0-9\.]+)\s*€", price_text.replace("\xa0", " ")) + price_gross = Decimal(m_price.group(1).replace(".", "")) if m_price else Decimal("0") + + vat_deductible = "MwSt." in price_text or "ausweisbar" in card_soup.get_text() + price_net = None + if vat_deductible and price_gross > 0: + price_net = round(price_gross / Decimal("1.19"), 2) + + monthly_rate = None + fin_el = card_soup.select_one(".financing-box .financing-data") + if fin_el: + fin_text = fin_el.get_text(strip=True).replace("\xa0", " ") + m_fin = re.search(r"([0-9\.,]+)\s*€", fin_text) + if m_fin: + monthly_rate = Decimal(m_fin.group(1).replace(".", "").replace(",", ".")) + + # 6. Dealer & Location + dealer_name, location_postal_code, location_city = None, None, None + loc_item = card_soup.select_one(".result-box-location-item") + if loc_item: + spans = loc_item.select("span.vc-vehicle-attribute-text") + if spans: + dealer_name = spans[0].get_text(strip=True) + if len(spans) > 1: + loc_str = spans[1].get_text(strip=True) + m_loc = re.match(r"^(\d{5})\s+(.+)$", loc_str) + if m_loc: + location_postal_code = m_loc.group(1) + location_city = m_loc.group(2) + else: + location_city = loc_str + + # 7. Card Images + image_urls: List[str] = [] + for img in card_soup.select("img[data-src], img.v-i-g-main-image"): + src = img.get("data-src") or img.get("src") or "" + if "img.autodo.eu" in src and src not in image_urls: + image_urls.append(src) + + # 8. Detail Page Enrichment (if available) + basic_data: Dict[str, str] = {} + bento_data: Dict[str, str] = {} + options: List[OptionSchema] = [] + model_year = None + engine_displacement_cc = None + exterior_color = None + exterior_color_generic = None + interior_color = None + interior_material = None + battery_capacity = None + battery_capacity_unit = None + battery_capacity_net_kwh = None + charging_power_ac_kw = None + charging_power_dc_kw = None + charging_time_dc_minutes = None + electric_range_wltp_km = None + dealer_latitude = None + dealer_longitude = None + condition = VehicleCondition.USED + + if detail_html: + detail_soup = BeautifulSoup(detail_html, "html.parser") + + # Basic Data rows + for item in detail_soup.select(".basic-data"): + lbl_el = item.select_one(".data-label") + val_el = item.select_one(".data-value") + if lbl_el and val_el: + lbl = lbl_el.get_text(strip=True) + val = val_el.get_text(" ", strip=True) + basic_data[lbl] = val + + # Bento grid (EV data) + for item in detail_soup.select(".bento-2-item"): + t_el = item.select_one(".bento-2-title") + v_el = item.select_one(".bento-2-value") + if t_el and v_el: + bento_data[t_el.get_text(strip=True)] = v_el.get_text(strip=True) + + if not comm_no: + comm_no = basic_data.get("Fahrzeugnummer") + + # Body type fallback + if not body_type: + for k, v in basic_data.items(): + if "van" in k.lower() or "limousine" in k.lower() or "kombi" in k.lower() or "suv" in k.lower(): + body_type = v + break + + # Reg date fallback + if not first_registration_date and "Erstzulassung" in basic_data: + try: + first_registration_date = datetime.strptime(basic_data["Erstzulassung"], "%d.%m.%Y").date() + except Exception: + pass + + # Model year + my_str = basic_data.get("Modeljahr") or basic_data.get("Modelljahr") + if my_str and my_str.isdigit(): + model_year = int(my_str) + + # Mileage fallback + if mileage_km == 0 and "Laufleistung" in basic_data: + m_det_m = re.search(r"([0-9\.]+)\s*km", basic_data["Laufleistung"].replace("\xa0", " ")) + if m_det_m: + mileage_km = int(m_det_m.group(1).replace(".", "")) + + # Power fallback + if power_kw is None and "Leistung" in basic_data: + m_det_p = re.search(r"(\d+)\s*kW\s*\((\d+)\s*PS\)", basic_data["Leistung"].replace("\xa0", " ")) + if m_det_p: + power_kw = int(m_det_p.group(1)) + power_hp = int(m_det_p.group(2)) + + # Displacement + if "Hubraum" in basic_data: + m_disp = re.search(r"([0-9\.]+)\s*ccm", basic_data["Hubraum"].replace("\xa0", " ")) + if m_disp: + engine_displacement_cc = int(m_disp.group(1).replace(".", "")) + + # Transmission + trans_str = basic_data.get("Getriebe", "") + if trans_str: + transmission = map_mercedes_transmission(trans_str) + else: + transmission = TransmissionType.AUTOMATIC + + # Fuel string fallback + if not fuel_str and "Kraftstoffart" in basic_data: + fuel_str = basic_data["Kraftstoffart"] + + # Colors & Upholstery + for k, v in basic_data.items(): + k_lower = k.lower() + if "außenfarbe" in k_lower: + exterior_color = v + m_gen = re.search(r"\(([^)]+)\)", k) + if m_gen: + exterior_color_generic = m_gen.group(1).strip() + elif "polster" in k_lower: + interior_material = v + elif "interieur farbton" in k_lower: + interior_color = v + + # Certified Pre-Owned (Junge Sterne) + garantie = basic_data.get("Garantie", "") + if "junge sterne" in garantie.lower() or "junge sterne" in detail_html.lower() or "junge sterne" in card_soup.get_text().lower(): + condition = VehicleCondition.CERTIFIED_PRE_OWNED + + # Battery capacity & range + for k, v in bento_data.items(): + k_lower = k.lower() + if "energie" in k_lower: + m_bat = re.search(r"(\d+(?:[\.,]\d+)?)\s*kWh", v, re.I) + if m_bat: + battery_capacity = Decimal(m_bat.group(1).replace(",", ".")) + battery_capacity_unit = BatteryCapacityUnit.KWH + battery_capacity_net_kwh = battery_capacity + elif "reichweite" in k_lower: + m_r = re.search(r"(\d+)\s*km", v) + if m_r: + electric_range_wltp_km = int(m_r.group(1)) + + if not battery_capacity: + m_bat = re.search(r"Batterie\s+(\d+(?:[\.,]\d+)?)\s*kWh", detail_html, re.I) + if m_bat: + battery_capacity = Decimal(m_bat.group(1).replace(",", ".")) + battery_capacity_unit = BatteryCapacityUnit.KWH + battery_capacity_net_kwh = battery_capacity + + # Charging power specs + m_ac = ( + re.search(r"AC(?:\s*Laden)?\s*(\d+(?:[\.,]\d+)?)\s*kW", detail_html, re.I) + or re.search(r"(\d+(?:[\.,]\d+)?)\s*kW\s*(?:Ladeleistung\s*)?(?:und\s*)?(?:Wechselstrom\s*)?(?:\()?\s*AC", detail_html, re.I) + ) + if m_ac: + charging_power_ac_kw = Decimal(m_ac.group(1).replace(",", ".")) + + m_dc = ( + re.search(r"DC(?:\s*Laden)?\s*(\d+(?:[\.,]\d+)?)\s*kW", detail_html, re.I) + or re.search(r"(\d+(?:[\.,]\d+)?)\s*kW\s*(?:Ladeleistung\s*)?(?:und\s*)?(?:Gleichstrom\s*)?(?:\()?\s*DC", detail_html, re.I) + or re.search(r"Ladeleistung\s*von\s*(\d+(?:[\.,]\d+)?)\s*kW[^\.]*?DC", detail_html, re.I) + ) + if m_dc: + charging_power_dc_kw = Decimal(m_dc.group(1).replace(",", ".")) + + # Leaflet GPS coordinates (Germany boundary: lat 47-55, lon 5-16) + m_coord = re.search(r"setCenterCoordinate\([\x27\"]([0-9\.]+)[\x27\"],\s*[\x27\"]([0-9\.]+)[\x27\"]\)", detail_html) + if m_coord: + dealer_latitude = Decimal(m_coord.group(1)) + dealer_longitude = Decimal(m_coord.group(2)) + else: + coords = re.findall(r"([0-9]{1,2}\.[0-9]{4,}),\s*([0-9]{1,2}\.[0-9]{4,})", detail_html) + for lat_str, lon_str in coords: + lat, lon = float(lat_str), float(lon_str) + if 47.0 <= lat <= 55.0 and 5.0 <= lon <= 16.0: + dealer_latitude = Decimal(str(lat)) + dealer_longitude = Decimal(str(lon)) + break + + # Gallery images + for img in detail_soup.select("img"): + src = img.get("data-src") or img.get("src") or "" + if "img.autodo.eu" in src and src not in image_urls: + image_urls.append(src) + + # Options + options = extract_mercedes_used_options(detail_soup) + else: + transmission = TransmissionType.AUTOMATIC + + # Final Fuel type & drivetrain + fuel_type = map_mercedes_used_fuel_type(fuel_str) + drivetrain = DrivetrainType.AWD if "4matic" in model_name.lower() or "allrad" in str(card_attrs).lower() else None + + # Vehicle URL & thumbnail + vehicle_url = f"https://gebrauchtwagen.mercedes-benz.de/vehicle?vehicle={card_id}" if card_id else None + thumbnail_url = image_urls[0] if image_urls else None + + raw_payload = { + "card_id": card_id, + "commission_number": comm_no, + "card_attributes": card_attrs, + "basic_data": basic_data, + "bento_data": bento_data, + } if include_raw else {} + + return VehicleIngestSchema( + brand="MERCEDES-BENZ", + oem_vehicle_id=card_id or comm_no or "unknown", + vin=None, + commission_number=comm_no, + series=series, + model_name=model_name, + trim_line=trim_line, + body_type=body_type, + condition=condition, + model_year=model_year, + first_registration_date=first_registration_date, + previous_owners_count=None, + fuel_type=fuel_type, + transmission=transmission, + drivetrain=drivetrain, + power_kw=power_kw, + power_hp=power_hp, + engine_displacement_cc=engine_displacement_cc, + battery_capacity=battery_capacity, + battery_capacity_unit=battery_capacity_unit, + battery_capacity_gross_kwh=None, + battery_capacity_net_kwh=battery_capacity_net_kwh, + charging_power_ac_kw=charging_power_ac_kw, + charging_power_dc_kw=charging_power_dc_kw, + charging_time_dc_minutes=charging_time_dc_minutes, + electric_range_wltp_km=electric_range_wltp_km, + consumption_combined=None, + co2_emissions_g_km=None, + mileage_km=mileage_km, + exterior_color=exterior_color, + exterior_color_code=None, + exterior_color_generic=exterior_color_generic, + interior_color=interior_color, + interior_material=interior_material, + currency="EUR", + base_price_gross=None, + options_price_gross=None, + price_gross=price_gross, + price_net=price_net, + msrp_gross=None, + discount_percentage=None, + vat_deductible=vat_deductible, + monthly_rate=monthly_rate, + leasing_details={}, + status=VehicleStatus.IN_STOCK, + availability_date=None, + delivery_time_days=None, + dealer_name=dealer_name, + dealer_id=None, + location_city=location_city, + location_postal_code=location_postal_code, + location_country=market_country, + latitude=dealer_latitude, + longitude=dealer_longitude, + vehicle_url=vehicle_url, + image_urls=image_urls, + thumbnail_url=thumbnail_url, + options=options, + raw_payload=raw_payload, + ) + + +def parse_mercedes_used_dict( + data: Dict[str, Any], + market_country: str = "DE", + include_raw: Optional[bool] = None, +) -> VehicleIngestSchema: + """ + Parses a previously extracted Mercedes used vehicle payload dictionary into a VehicleIngestSchema. + """ + if include_raw is None: + include_raw = settings.ENABLE_RAW_EXTRACTION + + card_id = data.get("card_id") + comm_no = data.get("commission_number") + card_attrs = data.get("card_attributes", {}) + basic_data = data.get("basic_data", {}) + bento_data = data.get("bento_data", {}) + + model_name = basic_data.get("Mercedes-Benz") or "Mercedes-Benz" + series = map_mercedes_used_series(model_name) + trim_line = None + for candidate in ["Avantgarde", "AMG Line", "Exclusive", "Progressive", "Style", "Edition 1", "Night-Paket"]: + if candidate.lower() in model_name.lower(): + trim_line = candidate + break + + body_type = card_attrs.get("vehicle-box-body-type") or basic_data.get("Van") + power_str = card_attrs.get("vehicle-box-power", "") or basic_data.get("Leistung", "") + power_kw, power_hp = None, None + m_p = re.search(r"(\d+)\s*kW\s*\((\d+)\s*PS\)", power_str.replace("\xa0", " ")) + if m_p: + power_kw = int(m_p.group(1)) + power_hp = int(m_p.group(2)) + + mileage_str = card_attrs.get("vehicle-box-mileage", "") or basic_data.get("Laufleistung", "") + mileage_km = 0 + m_m = re.search(r"([0-9\.]+)\s*km", mileage_str.replace("\xa0", " ")) + if m_m: + mileage_km = int(m_m.group(1).replace(".", "")) + + first_registration_date = None + reg_str = card_attrs.get("vehicle-box-first-registration", "") or basic_data.get("Erstzulassung", "") + if reg_str: + try: + first_registration_date = datetime.strptime(reg_str, "%d.%m.%Y").date() + except Exception: + pass + + fuel_str = card_attrs.get("vehicle-box-engine-ice") or card_attrs.get("vehicle-box-engine-bev") or basic_data.get("Kraftstoffart", "") + fuel_type = map_mercedes_used_fuel_type(fuel_str) + drivetrain = DrivetrainType.AWD if "4matic" in model_name.lower() or "allrad" in str(card_attrs).lower() else None + + exterior_color = None + exterior_color_generic = None + interior_color = basic_data.get("Interieur Farbton") + interior_material = basic_data.get("Polster (Leder)") or basic_data.get("Polster") + + for k, v in basic_data.items(): + if "außenfarbe" in k.lower(): + exterior_color = v + m_gen = re.search(r"\(([^)]+)\)", k) + if m_gen: + exterior_color_generic = m_gen.group(1).strip() + + price_gross = safe_decimal(data.get("price_gross")) or Decimal(0) + price_net = safe_decimal(data.get("price_net")) + monthly_rate = safe_decimal(data.get("monthly_rate")) + dealer_name = card_attrs.get("vehicle-box-location") + vehicle_url = f"https://gebrauchtwagen.mercedes-benz.de/vehicle?vehicle={card_id}" if card_id else None + + return VehicleIngestSchema( + brand="MERCEDES-BENZ", + oem_vehicle_id=card_id or comm_no or "unknown", + vin=None, + commission_number=comm_no, + series=series, + model_name=model_name, + trim_line=trim_line, + body_type=body_type, + condition=VehicleCondition.USED, + model_year=safe_int(basic_data.get("Modeljahr")), + first_registration_date=first_registration_date, + previous_owners_count=None, + fuel_type=fuel_type, + transmission=TransmissionType.AUTOMATIC, + drivetrain=drivetrain, + power_kw=power_kw, + power_hp=power_hp, + engine_displacement_cc=safe_int(re.sub(r"[^\d]", "", basic_data.get("Hubraum", ""))) if basic_data.get("Hubraum") else None, + mileage_km=mileage_km, + exterior_color=exterior_color, + exterior_color_generic=exterior_color_generic, + interior_color=interior_color, + interior_material=interior_material, + currency="EUR", + price_gross=price_gross, + price_net=price_net, + monthly_rate=monthly_rate, + status=VehicleStatus.IN_STOCK, + dealer_name=dealer_name, + location_country=market_country, + vehicle_url=vehicle_url, + options=[], + raw_payload=data if include_raw else {}, + ) + diff --git a/src/oem_crawler/crawlers/mercedes/query.py b/src/oem_crawler/crawlers/mercedes/query.py new file mode 100644 index 0000000..325a7c3 --- /dev/null +++ b/src/oem_crawler/crawlers/mercedes/query.py @@ -0,0 +1 @@ +GRAPHQL_QUERY = '\n query GetSearchResults(\n $contextType: ContextType! = B2C\n $enableColorAggregation: Boolean = false\n $isUcos: Boolean = false\n $language: String\n $limit: Int! = 10\n $page: Int! = 0\n $productCode: String\n $profileId: String!\n $sortingType: String! = "price-asc"\n $variantId: String\n $vehicleCategory: String!\n $brand: [String!]\n$modelIdentifier: [VehicleClass!]\n$motorization: [String!]\n$bodyType: [BodyType!]\n$price: ValueRange\n$monthlyRate: ValueRange\n$fuelType: [FuelTypeHarmonized!]\n$electricRange: ValueRange\n$chargeTimeDCHigh: ValueRange\n$energyContent: ValueRange\n$driveType: [TypeOfPropulsion!]\n$enginePowerKW: IntRange\n$stockType: [StockItemState!]\n$upholstery: [String!]\n$upholsteryPolish: [String!]\n$packages: [String!]\n$lines: [String!]\n$equipment: [String!]\n$colorPolish: [String!]\n$color: [String!]\n$campaigns: [String!]\n$gearbox: [TransmissionCategory!]\n$seats: [Int!]\n$payload: IntRange\n$maximumWeight: IntRange\n$loadspaceVolume: IntRange\n$loadspaceWidth: IntRange\n$loadspaceHeight: IntRange\n$vehicleHeight: IntRange\n $dealerId: [String!]\n ) {\n search(\n contextType: $contextType\n language: $language\n limit: $limit\n page: $page\n productCode: $productCode\n profileId: $profileId\n sortingType: $sortingType\n variantId: $variantId\n vehicleCategory: $vehicleCategory\n brand: $brand\nmodelIdentifier: $modelIdentifier\nmotorization: $motorization\nbodyType: $bodyType\nprice: $price\nmonthlyRate: $monthlyRate\nfuelType: $fuelType\nelectricRange: $electricRange\nchargeTimeDCHigh: $chargeTimeDCHigh\nenergyContent: $energyContent\ndriveType: $driveType\nenginePowerKW: $enginePowerKW\nstockType: $stockType\nupholstery: $upholstery\nupholsteryPolish: $upholsteryPolish\npackages: $packages\nlines: $lines\nequipment: $equipment\ncolorPolish: $colorPolish\ncolor: $color\ncampaigns: $campaigns\ngearbox: $gearbox\nseats: $seats\npayload: $payload\nmaximumWeight: $maximumWeight\nloadspaceVolume: $loadspaceVolume\nloadspaceWidth: $loadspaceWidth\nloadspaceHeight: $loadspaceHeight\nvehicleHeight: $vehicleHeight\n dealerId: $dealerId\n ) {\n facets {\n \n brand {\n \n ... on FormattedValueFacet {\n values {\n label\n formattedValue\n value\n count\n tags\n rgbValue\n }\n facetType\n }\n }\n \n\n modelIdentifier {\n \n ... on FormattedValueFacet {\n values {\n label\n formattedValue\n value\n count\n tags\n rgbValue\n }\n facetType\n }\n }\n \n\n motorization {\n \n ... on SimpleCountFacet {\n values {\n value\n count\n }\n facetType\n }\n \n }\n \n\n bodyType {\n \n ... on FormattedValueFacet {\n values {\n label\n formattedValue\n value\n count\n tags\n rgbValue\n }\n facetType\n }\n }\n \n\n price {\n \n ... on RangeFacet {\n values {\n min\n max\n count\n }\n facetType\n }\n \n }\n \n\n monthlyRate {\n \n ... on RangeFacet {\n values {\n min\n max\n count\n }\n facetType\n }\n \n }\n \n\n fuelType {\n \n ... on FormattedValueFacet {\n values {\n label\n formattedValue\n value\n count\n tags\n rgbValue\n }\n facetType\n }\n }\n \n\n electricRange {\n \n ... on RangeFacet {\n values {\n min\n max\n count\n }\n facetType\n }\n \n }\n \n\n chargeTimeDCHigh {\n \n ... on RangeFacet {\n values {\n min\n max\n count\n }\n facetType\n }\n \n }\n \n\n energyContent {\n \n ... on RangeFacet {\n values {\n min\n max\n count\n }\n facetType\n }\n \n }\n \n\n driveType {\n \n ... on FormattedValueFacet {\n values {\n label\n formattedValue\n value\n count\n tags\n rgbValue\n }\n facetType\n }\n }\n \n\n enginePowerKW {\n \n ... on RangeFacet {\n values {\n min\n max\n count\n }\n facetType\n }\n \n }\n \n\n stockType {\n \n ... on SimpleCountFacet {\n values {\n value\n count\n }\n facetType\n }\n \n }\n \n\n upholstery {\n \n ... on FormattedValueFacet {\n values {\n label\n formattedValue\n value\n count\n tags\n rgbValue\n }\n facetType\n }\n }\n \n\n upholsteryPolish {\n \n ... on FormattedValueFacet {\n values {\n label\n formattedValue\n value\n count\n tags\n rgbValue\n }\n facetType\n }\n }\n \n\n packages {\n \n ... on FormattedValueFacet {\n values {\n label\n formattedValue\n value\n count\n tags\n rgbValue\n }\n facetType\n }\n }\n \n\n lines {\n \n ... on FormattedValueFacet {\n values {\n label\n formattedValue\n value\n count\n tags\n rgbValue\n }\n facetType\n }\n }\n \n\n equipment {\n \n ... on FormattedValueFacet {\n values {\n label\n formattedValue\n value\n count\n tags\n rgbValue\n }\n facetType\n }\n }\n \n\n colorPolish {\n \n ... on FormattedValueFacet {\n values {\n label\n formattedValue\n value\n count\n tags\n rgbValue\n }\n facetType\n }\n }\n \n\n color {\n \n ... on FormattedValueFacet {\n values {\n label\n formattedValue\n value\n count\n tags\n rgbValue\n }\n facetType\n }\n }\n \n\n campaigns {\n \n ... on FormattedValueFacet {\n values {\n label\n formattedValue\n value\n count\n tags\n rgbValue\n }\n facetType\n }\n }\n \n\n gearbox {\n \n ... on FormattedValueFacet {\n values {\n label\n formattedValue\n value\n count\n tags\n rgbValue\n }\n facetType\n }\n }\n \n\n seats {\n \n ... on NumberCountFacet {\n values {\n value\n count\n }\n facetType\n }\n \n }\n \n\n payload {\n \n ... on RangeFacet {\n values {\n min\n max\n count\n }\n facetType\n }\n \n }\n \n\n maximumWeight {\n \n ... on RangeFacet {\n values {\n min\n max\n count\n }\n facetType\n }\n \n }\n \n\n loadspaceVolume {\n \n ... on RangeFacet {\n values {\n min\n max\n count\n }\n facetType\n }\n \n }\n \n\n loadspaceWidth {\n \n ... on RangeFacet {\n values {\n min\n max\n count\n }\n facetType\n }\n \n }\n \n\n loadspaceHeight {\n \n ... on RangeFacet {\n values {\n min\n max\n count\n }\n facetType\n }\n \n }\n \n\n vehicleHeight {\n \n ... on RangeFacet {\n values {\n min\n max\n count\n }\n facetType\n }\n \n }\n \n }\n \n navigation {\n currentLimit\n currentPage\n currentSortingCode\n totalPages\n totalResults\n }\n\n \n results {\n characteristics {\n stockCategories {\n code\n }\n highlights {\n label\n }\n campaigns {\n description\n footnote\n label\n }\n }\n colorVariants @include(if: $enableColorAggregation) {\n color {\n groupingId\n rgbValue\n text\n }\n estimatedArrivalDate\n identification {\n code\n }\n images {\n default\n }\n preProductionVehicle\n stockItemState\n }\n consignorCompanyId\n dealer {\n addressGlobalLanguage\n addressLocalLanguage\n category\n companyId\n nameGlobalLanguage\n nameLocalLanguage\n outletId\n }\n emissionAndConsumption {\n attributes {\n displayValue\n id\n label\n mustShowIn\n unit\n value\n }\n footnotes\n testProcedure\n }\n envkv {\n co2Classes {\n primary\n secondary\n }\n }\n estimatedArrivalDate\n identification {\n code\n commissionNumber\n dcpProductType\n dealerId\n dealerGroupName\n mpcId\n variantId\n vin\n vxVehicleId\n allDealers\n }\n images {\n default\n exterior @include(if: $isUcos)\n interior @include(if: $isUcos)\n misc @include(if: $isUcos)\n }\n productionDate\n preProductionVehicle\n stock {\n stockType\n }\n stockCategory\n technicalInformation {\n engine {\n fuelType {\n ...TechnicalData\n }\n power {\n ...PowerData\n }\n }\n transmission {\n ...TechnicalData\n }\n }\n usedVehicleData @include(if: $isUcos) {\n mileage {\n ...IntegerTechnicalData\n }\n firstRegistrationDate\n vehicleInspection {\n maintenance\n }\n warranty {\n status\n unlimitedDistance\n }\n }\n vehicleModel {\n baumuster\n bodyType {\n ...TechnicalData\n }\n brand {\n ...TechnicalData\n }\n category {\n ...TechnicalData\n }\n facelift\n generation\n modelYear\n modelYearCode\n motorization\n name\n steeringPosition {\n ...TechnicalData\n }\n typeClass\n vehicleClass {\n ...TechnicalData\n }\n }\n wholesale\n }\n \n }\n }\n \n fragment IntegerTechnicalData on IntegerTechnicalData {\n label\n formattedValue\n value\n unit\n }\n\n \n fragment PowerData on Power {\n label\n formattedValue\n combustionKw {\n ...IntegerTechnicalData\n }\n combustionHp {\n ...IntegerTechnicalData\n }\n electricKw {\n ...IntegerTechnicalData\n }\n electricHp {\n ...IntegerTechnicalData\n }\n combinedKw {\n ...IntegerTechnicalData\n }\n combinedHp {\n ...IntegerTechnicalData\n }\n }\n\n \n fragment TechnicalData on TechnicalData {\n label\n formattedValue\n value\n unit\n }\n\n ' diff --git a/src/oem_crawler/crawlers/porsche/__init__.py b/src/oem_crawler/crawlers/porsche/__init__.py new file mode 100644 index 0000000..8d20cd2 --- /dev/null +++ b/src/oem_crawler/crawlers/porsche/__init__.py @@ -0,0 +1,7 @@ +""" +Porsche Crawler Package exports. +""" +from oem_crawler.crawlers.porsche.crawler import PorscheStockCrawler +from oem_crawler.crawlers.porsche.parser import parse_porsche_vehicle + +__all__ = ["PorscheStockCrawler", "parse_porsche_vehicle"] diff --git a/src/oem_crawler/crawlers/porsche/crawler.py b/src/oem_crawler/crawlers/porsche/crawler.py new file mode 100644 index 0000000..987fcb5 --- /dev/null +++ b/src/oem_crawler/crawlers/porsche/crawler.py @@ -0,0 +1,266 @@ +""" +Porsche Finder Stock Crawler Implementation. +Interacts with Porsche Finder Next.js React Server Components (RSC) endpoints. +Supports both new and used vehicles from the unified backend. +""" +from __future__ import annotations + +import json +import logging +import time +from typing import Any, Dict, Generator, List, Optional, Tuple + +import httpx + +from oem_crawler.crawlers.base import BaseCrawler, CrawlerRegistry +from oem_crawler.crawlers.porsche.parser import parse_porsche_vehicle +from oem_crawler.db.schemas import VehicleIngestSchema + +logger = logging.getLogger(__name__) + + +@CrawlerRegistry.register("porsche") +class PorscheStockCrawler(BaseCrawler): + BASE_URL = "https://finder.porsche.com" + + def __init__( + self, + market_country: str = "DE", + proxy: Optional[str] = None, + request_delay: float = 0.5, + timeout: float = 25.0, + stock_type: str = "new", + fetch_details: bool = True, + order: str = "recommended_desc", + **kwargs: Any, + ): + super().__init__( + market_country=market_country, + proxy=proxy, + request_delay=request_delay, + timeout=timeout, + **kwargs, + ) + self.stock_type = stock_type.lower() + self.fetch_details = fetch_details + self.order = order + + market = self.market_country.lower() + lang_tag = f"{market}-{self.market_country.upper()}" + self.locale_path = f"{market}/{lang_tag}" + + self.headers = { + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", + "RSC": "1", + "Accept": "text/x-component", + "Accept-Language": f"{market}-{self.market_country.upper()},{market};q=0.9,en;q=0.8", + } + + @property + def brand_name(self) -> str: + return "PORSCHE" + + def _parse_search_rsc(self, text: str) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]: + """ + Extracts the pages pagination block and the items array from the RSC payload + using JSONDecoder.raw_decode. + """ + decoder = json.JSONDecoder() + pages: Dict[str, Any] = {"activePage": 1, "totalPages": 1} + items: List[Dict[str, Any]] = [] + + # 1. Parse pages + pos_pages = text.find('"pages":{') + if pos_pages != -1: + try: + pages, _ = decoder.raw_decode(text, pos_pages + 8) + except Exception as e: + logger.warning(f"Error decoding pages from Porsche RSC: {e}") + + # 2. Parse items + pos_items = text.find('"items":[') + if pos_items != -1: + try: + items, _ = decoder.raw_decode(text, pos_items + 8) + except Exception as e: + logger.warning(f"Error decoding items from Porsche RSC: {e}") + + return pages, items + + def _parse_detail_rsc(self, text: str) -> Dict[str, Any]: + """ + Extracts equipment, summary (characteristics), and info blocks from detail RSC payload. + """ + decoder = json.JSONDecoder() + detail_data: Dict[str, Any] = {} + + # 1. Equipment + pos_eq = text.find('"equipment":{') + if pos_eq != -1: + try: + detail_data["equipment"], _ = decoder.raw_decode(text, pos_eq + 12) + except Exception: + pass + + # 2. Summary (characteristics, EV specs) + pos_sum = text.find('"summary":{') + if pos_sum != -1: + try: + detail_data["summary"], _ = decoder.raw_decode(text, pos_sum + 10) + except Exception: + pass + + # 3. Info (pricing breakdown, seller, commission number) + pos_info = text.find('"info":{') + if pos_info != -1: + try: + detail_data["info"], _ = decoder.raw_decode(text, pos_info + 7) + except Exception: + pass + + return detail_data + + def fetch_search_page( + self, + client: httpx.Client, + condition: str = "new", + page: int = 1, + search_filter: Optional[Dict[str, Any]] = None, + max_retries: int = 3, + ) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]: + """ + Fetches one page of search results for a given condition ('new' or 'used'). + """ + url = f"{self.BASE_URL}/{self.locale_path}/search" + params: Dict[str, Any] = { + "condition": condition, + "page": page, + "order": self.order, + } + if search_filter: + params.update(search_filter) + + for attempt in range(1, max_retries + 1): + try: + time.sleep(self.request_delay) + logger.info(f"Fetching Porsche {condition} stock page {page}...") + resp = client.get(url, params=params, headers=self.headers) + resp.raise_for_status() + return self._parse_search_rsc(resp.text) + except httpx.HTTPError as e: + logger.warning(f"Attempt {attempt}/{max_retries} failed for Porsche {condition} page {page}: {e}") + if attempt == max_retries: + raise + time.sleep(attempt * 1.5) + + return {"activePage": page, "totalPages": 0}, [] + + def fetch_detail( + self, + client: httpx.Client, + listing_url_slug: str, + max_retries: int = 2, + ) -> Optional[Dict[str, Any]]: + """ + Fetches vehicle detail RSC payload for factory equipment and EV specs. + """ + url = f"{self.BASE_URL}/{self.locale_path}/details/{listing_url_slug}" + for attempt in range(1, max_retries + 1): + try: + time.sleep(self.request_delay) + resp = client.get(url, headers=self.headers) + resp.raise_for_status() + return self._parse_detail_rsc(resp.text) + except Exception as e: + logger.warning(f"Failed to fetch detail for {listing_url_slug} (attempt {attempt}): {e}") + if attempt == max_retries: + return None + time.sleep(1.0) + return None + + def _crawl_condition( + self, + client: httpx.Client, + condition: str, + max_vehicles: Optional[int], + search_filter: Optional[Dict[str, Any]], + ) -> Generator[VehicleIngestSchema, None, None]: + """ + Streams vehicles for a specific condition ('new' or 'used'). + """ + count = 0 + current_page = 1 + total_pages = 1 + + logger.info(f"Starting Porsche {condition.upper()} stock crawl (Market: {self.market_country})...") + + while current_page <= total_pages: + pages_info, items = self.fetch_search_page( + client=client, + condition=condition, + page=current_page, + search_filter=search_filter, + ) + + total_pages = pages_info.get("totalPages", total_pages) + if not items: + logger.info(f"No items found on page {current_page}. Ending {condition} crawl.") + break + + logger.info(f"Porsche {condition} reports {len(items)} items on page {current_page}/{total_pages}.") + + for item in items: + detail_data = None + slug = item.get("listingUrlSlug") + if self.fetch_details and slug: + detail_data = self.fetch_detail(client, slug) + + try: + vehicle = parse_porsche_vehicle( + search_item=item, + detail_data=detail_data, + market_country=self.market_country, + requested_stock_type=condition, + ) + yield vehicle + count += 1 + except Exception as e: + logger.error(f"Failed to parse Porsche vehicle {item.get('id')}: {e}", exc_info=True) + + if max_vehicles and count >= max_vehicles: + logger.info(f"Reached limit of {max_vehicles} {condition} vehicles.") + return + + current_page += 1 + + def crawl( + self, + max_vehicles: Optional[int] = None, + page_size: int = 15, + search_filter: Optional[Dict[str, Any]] = None, + ) -> Generator[VehicleIngestSchema, None, None]: + """ + Main crawl generator: supports stock_type in ('new', 'used', 'all'). + """ + client_kwargs: Dict[str, Any] = { + "timeout": self.timeout, + "http2": False, + } + if self.proxy: + client_kwargs["proxy"] = self.proxy + + with httpx.Client(**client_kwargs) as client: + if self.stock_type == "all": + half_limit = (max_vehicles // 2) if max_vehicles else None + new_count = 0 + for veh in self._crawl_condition(client, "new", half_limit, search_filter): + yield veh + new_count += 1 + + rem_limit = (max_vehicles - new_count) if max_vehicles else None + for veh in self._crawl_condition(client, "used", rem_limit, search_filter): + yield veh + elif self.stock_type == "used": + yield from self._crawl_condition(client, "used", max_vehicles, search_filter) + else: + yield from self._crawl_condition(client, "new", max_vehicles, search_filter) diff --git a/src/oem_crawler/crawlers/porsche/parser.py b/src/oem_crawler/crawlers/porsche/parser.py new file mode 100644 index 0000000..4365330 --- /dev/null +++ b/src/oem_crawler/crawlers/porsche/parser.py @@ -0,0 +1,392 @@ +""" +Parser and normalization logic for Porsche Finder responses. +Handles search RSC payloads and detail RSC payloads. +""" +from __future__ import annotations + +import logging +import re +from datetime import date, datetime +from decimal import Decimal +from typing import Any, Dict, List, Optional + +from oem_crawler.config import settings +from oem_crawler.db.schemas import ( + BatteryCapacityUnit, + DrivetrainType, + FuelType, + OptionSchema, + TransmissionType, + VehicleCondition, + VehicleIngestSchema, + VehicleStatus, +) + +logger = logging.getLogger(__name__) + + +def safe_decimal(val: Any) -> Optional[Decimal]: + """Safely extracts a Decimal from numeric, string, or dict values.""" + if val is None: + return None + if isinstance(val, dict): + val = val.get("value") or val.get("amount") or val.get("rawValue") + if val is None: + return None + try: + val_str = str(val).replace(".", "").replace(",", ".").replace("€", "").replace("\xa0", "").strip() + # Handle cases where number was already float/decimal with single decimal point + # e.g. "131063" or "220231.42" + # If original had a single dot and no comma: + raw_s = str(val).strip().replace("€", "").replace("\xa0", "").strip() + if "," not in raw_s and "." in raw_s: + return Decimal(raw_s) + if not val_str: + return None + return Decimal(val_str) + except Exception: + try: + return Decimal(str(val).strip()) + except Exception: + return None + + +def safe_int(val: Any) -> Optional[int]: + """Safely extracts an integer.""" + if val is None: + return None + if isinstance(val, dict): + val = val.get("value") + try: + val_str = str(val).split(".")[0].strip() + return int(val_str) + except Exception: + return None + + +def parse_date(val: Any) -> Optional[date]: + """Parses date string in ISO format YYYY-MM-DD.""" + if not val or not isinstance(val, str) or val == "$undefined": + return None + try: + clean = val.split("T")[0].strip() + return datetime.strptime(clean, "%Y-%m-%d").date() + except Exception: + return None + + +def map_porsche_fuel_type(engine_type_raw: Optional[str]) -> FuelType: + raw = (engine_type_raw or "").upper() + if "ELECTRIC" in raw or "ELEKTRO" in raw: + return FuelType.BEV + if "PLUG" in raw or "PHEV" in raw or "HYBRID" in raw and "MILD" not in raw: + return FuelType.PHEV + if "MILD" in raw or "MHEV" in raw: + return FuelType.MHEV + if "DIESEL" in raw: + return FuelType.DIESEL + if "PETROL" in raw or "BENZIN" in raw or "GASOLINE" in raw: + return FuelType.PETROL + return FuelType.PETROL + + +def map_porsche_transmission(trans_val: Optional[str]) -> Optional[TransmissionType]: + raw = (trans_val or "").upper() + if "PDK" in raw or "AUTOMATIK" in raw or "AUTOMATIC" in raw or "AUT" in raw: + return TransmissionType.AUTOMATIC + if "MANUELL" in raw or "MANUAL" in raw or "SCHALT" in raw: + return TransmissionType.MANUAL + return None + + +def map_porsche_drivetrain(dt_val: Optional[str]) -> Optional[DrivetrainType]: + raw = (dt_val or "").upper() + if "ALL_WHEEL" in raw or "ALLRAD" in raw or "AWD" in raw or "4S" in raw or "4" in raw and "DRIVE" in raw: + return DrivetrainType.AWD + if "REAR" in raw or "HECK" in raw or "RWD" in raw: + return DrivetrainType.RWD + if "FRONT" in raw or "FWD" in raw: + return DrivetrainType.FWD + return None + + +def map_porsche_series(model_category: Optional[str], title: Optional[str]) -> str: + cat = (model_category or "").strip() + tit = (title or "").lower() + + if cat: + cl = cat.lower() + if "911" in cl: + return "911" + if "taycan" in cl: + return "Taycan" + if "macan" in cl: + return "Macan" + if "cayenne" in cl: + return "Cayenne" + if "panamera" in cl: + return "Panamera" + if "718" in cl or "boxster" in cl or "cayman" in cl: + return "718" + return cat.capitalize() + + for s in ["911", "Taycan", "Macan", "Cayenne", "Panamera", "718"]: + if s.lower() in tit: + return s + return "Porsche" + + +def parse_porsche_vehicle( + search_item: Dict[str, Any], + detail_data: Optional[Dict[str, Any]] = None, + market_country: str = "DE", + requested_stock_type: str = "new", + include_raw: Optional[bool] = None, +) -> VehicleIngestSchema: + """ + Parses a Porsche vehicle from search item and optional detail payload. + """ + if include_raw is None: + include_raw = settings.ENABLE_RAW_EXTRACTION + meta = search_item.get("meta") or {} + car_meta = search_item.get("dataLayerListingMeta", {}).get("car", {}) + seller = meta.get("seller") or {} + addr_comp = seller.get("addressComponents") or {} + desc = search_item.get("description") or {} + + oem_id = str(search_item.get("id") or "").strip() + title = meta.get("title") or desc.get("title") or "Porsche" + series = map_porsche_series(meta.get("modelCategory"), title) + + # Condition determination + cond_raw = (meta.get("condition") or "").lower() + is_approved = car_meta.get("isPorscheApproved", False) or "approved" in (desc.get("subtitle") or "").lower() + + if requested_stock_type == "new" or cond_raw == "new": + condition = VehicleCondition.NEW + elif is_approved: + condition = VehicleCondition.CERTIFIED_PRE_OWNED + else: + condition = VehicleCondition.USED + + # Powertrain & Specs + fuel_type = map_porsche_fuel_type(meta.get("engineType")) + transmission = map_porsche_transmission(meta.get("transmission")) + drivetrain = map_porsche_drivetrain(meta.get("drivetrain")) + + power_kw = safe_int(meta.get("enginePower")) + power_hp = int(round(power_kw * 1.35962)) if power_kw else None + disp_cc = safe_int(meta.get("engineDisplacement")) + + mileage_val = safe_int(meta.get("mileage", {}).get("value")) + mileage_km = mileage_val if mileage_val is not None else 0 + + first_reg = parse_date(meta.get("firstRegistration")) + prev_owners = safe_int(meta.get("numberOfPreviousOwners")) + model_year = safe_int(meta.get("modelYear")) or safe_int(car_meta.get("modelModelYear")) + + # Pricing + price_gross = safe_decimal(meta.get("priceValue")) or safe_decimal(car_meta.get("priceTotalTotal")) + currency = meta.get("priceCurrency") or "EUR" + + # Monthly payment extraction + monthly_rate = None + periodic_payments = desc.get("periodicPayments") or {} + payment_label = periodic_payments.get("paymentLabel") if isinstance(periodic_payments, dict) else str(periodic_payments) + if payment_label: + # e.g. "765.26 per month" or "2808 per month" + rate_match = re.search(r"([0-9\.]+)", str(payment_label)) + if rate_match: + try: + monthly_rate = Decimal(rate_match.group(1)) + except Exception: + pass + + # Dealer / Location + dealer_name = seller.get("name") or desc.get("seller", {}).get("name") + dealer_id = str(seller.get("sellerId") or seller.get("porschePartnerNumber") or "").strip() or None + postal_code = addr_comp.get("postalCode") + city = addr_comp.get("city") + + # Images + gallery = search_item.get("gallery") or {} + image_ids = gallery.get("imageIds") or [] + image_urls = [f"https://images.finder.porsche.com/{img_id}/960.webp" for img_id in image_ids] + thumb = meta.get("imageUrl") or (image_urls[0] if image_urls else None) + + slug = search_item.get("listingUrlSlug") or f"porsche-{series.lower()}-{oem_id}" + vehicle_url = f"https://finder.porsche.com/de/de-DE/details/{slug}" + + # Initialize Option C fields + battery_capacity = None + battery_capacity_unit = None + charging_power_ac_kw = None + charging_power_dc_kw = None + charging_time_dc_minutes = None + electric_range_wltp_km = None + options: List[OptionSchema] = [] + + # Enrich from detail payload if available + raw_payload: Dict[str, Any] = {"search_item": search_item} if include_raw else {} + if detail_data and include_raw: + raw_payload["detail_data"] = detail_data + + # Characteristics (EV Specs, Engine, etc.) + summary = detail_data.get("summary") or {} + characteristics = summary.get("characteristics") or [] + for ch in characteristics: + title_ch = (ch.get("title") or "").strip() + val_ch = (ch.get("value") or "").strip() + + # Battery Capacity + if "Energieinhalt Batterie" in title_ch or "BatteriekapazitΓ€t" in title_ch: + cap_match = re.search(r"([0-9]+[,\.][0-9]+|[0-9]+)\s*kWh", val_ch, re.I) + if cap_match: + try: + clean_cap = cap_match.group(1).replace(",", ".") + battery_capacity = Decimal(clean_cap) + battery_capacity_unit = BatteryCapacityUnit.KWH + except Exception: + pass + + # WLTP Range + elif "Reichweite" in title_ch and "WLTP" in title_ch: + range_match = re.search(r"(\d+)\s*km", val_ch) + if range_match: + try: + electric_range_wltp_km = int(range_match.group(1)) + except Exception: + pass + + # DC Charging Time + elif "Gleichstrom (DC)" in title_ch or "Ladezeit" in title_ch and "DC" in title_ch: + time_match = re.search(r"([0-9]+[,\.][0-9]+|[0-9]+)\s*min", val_ch) + if time_match: + try: + clean_time = time_match.group(1).replace(",", ".").split(".")[0] + charging_time_dc_minutes = int(clean_time) + except Exception: + pass + + # Options / Special Equipment + equipment = detail_data.get("equipment") or {} + special_eq = equipment.get("specialEquipment") or [] + for cat in special_eq: + cat_name = cat.get("name") or "Sonderausstattung" + for item in cat.get("items") or []: + code = item.get("code") + name = item.get("name") or code or "Ausstattung" + description = item.get("description") + options.append( + OptionSchema( + option_code=code, + name=name, + category=cat_name, + description=description, + is_standard=False, + ) + ) + + # Check for AC charging upgrades (e.g. 22 kW On-Board AC-Lader) + if code == "KB3" or "22 kW" in name or description and "22 kW AC" in description: + charging_power_ac_kw = Decimal("22.00") + elif "11 kW" in name or description and "11 kW AC" in description: + if not charging_power_ac_kw: + charging_power_ac_kw = Decimal("11.00") + + # Important highlight options + important_eq = equipment.get("importantEquipmentOptions") or [] + existing_codes = {opt.option_code for opt in options if opt.option_code} + for imp in important_eq: + val = imp.get("value") + lbl = imp.get("label") or val + if val and val not in existing_codes: + options.append( + OptionSchema( + option_code=val, + name=lbl, + category="Highlights", + is_standard=False, + ) + ) + + # Standard Equipment + standard_eq = equipment.get("standardEquipment") or [] + for cat in standard_eq: + cat_name = cat.get("name") or "Serienausstattung" + for item in cat.get("items") or []: + name = item.get("name") or "Serienausstattung" + options.append( + OptionSchema( + option_code=None, + name=name, + category=cat_name, + is_standard=True, + ) + ) + + # If EV and default charging power wasn't overridden, standard Taycan / Macan EV AC is 11 kW + if fuel_type == FuelType.BEV: + if not charging_power_ac_kw: + charging_power_ac_kw = Decimal("11.00") + if not charging_power_dc_kw: + # Taycan II / Macan EV supports up to 270 kW / 320 kW DC fast charging + if series == "Taycan": + charging_power_dc_kw = Decimal("320.00") if "II" in (meta.get("model") or "") else Decimal("270.00") + elif series == "Macan": + charging_power_dc_kw = Decimal("270.00") + + # VIN resolution + vin = meta.get("vin") + if not vin or vin == "$undefined": + info = detail_data.get("info") or {} if detail_data else {} + vin = info.get("vin") + if not vin or vin == "$undefined" or not vin.strip(): + vin = None + + return VehicleIngestSchema( + brand="PORSCHE", + oem_vehicle_id=oem_id, + vin=vin, + series=series, + model_name=title, + trim_line=meta.get("model") or None, + body_type=meta.get("bodyType") or None, + condition=condition, + model_year=model_year, + first_registration_date=first_reg, + previous_owners_count=prev_owners, + fuel_type=fuel_type, + transmission=transmission, + drivetrain=drivetrain, + power_kw=power_kw, + power_hp=power_hp, + engine_displacement_cc=disp_cc, + battery_capacity=battery_capacity, + battery_capacity_unit=battery_capacity_unit, + charging_power_ac_kw=charging_power_ac_kw, + charging_power_dc_kw=charging_power_dc_kw, + charging_time_dc_minutes=charging_time_dc_minutes, + electric_range_wltp_km=electric_range_wltp_km, + consumption_combined=safe_decimal(meta.get("fuelConsumption")), + co2_emissions_g_km=safe_decimal(meta.get("co2Emissions")), + mileage_km=mileage_km, + exterior_color=meta.get("color"), + exterior_color_code=car_meta.get("exteriorColorCode"), + interior_color=meta.get("interiorColor"), + interior_material=meta.get("interiorName"), + currency=currency, + price_gross=price_gross, + monthly_rate=monthly_rate, + status=VehicleStatus.IN_STOCK, + dealer_name=dealer_name, + dealer_id=dealer_id, + location_city=city, + location_postal_code=postal_code, + location_country=market_country, + vehicle_url=vehicle_url, + image_urls=image_urls, + thumbnail_url=thumb, + options=options, + raw_payload=raw_payload, + ) diff --git a/src/oem_crawler/db/models.py b/src/oem_crawler/db/models.py new file mode 100644 index 0000000..a6db100 --- /dev/null +++ b/src/oem_crawler/db/models.py @@ -0,0 +1,279 @@ +""" +SQLAlchemy 2.0 ORM Models for OEM Vehicle Crawler. +""" +from __future__ import annotations + +import uuid +from datetime import date, datetime, timezone +from decimal import Decimal +from typing import List, Optional + +from sqlalchemy import ( + Boolean, + Date, + DateTime, + ForeignKey, + Index, + Integer, + JSON, + Numeric, + SmallInteger, + String, + Text, + UniqueConstraint, + Uuid, +) +from sqlalchemy.dialects.postgresql import JSONB, UUID as PGUUID +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + +from oem_crawler.db.schemas import ( + CrawlStage, + CrawlStatus, + FuelType, + LogLevel, + VehicleCondition, + VehicleStatus, +) + +# Cross-dialect JSON & UUID types (PostgreSQL JSONB / SQLite JSON fallback) +CompatibleJSON = JSONB().with_variant(JSON(), "sqlite") +CompatibleUUID = PGUUID(as_uuid=True).with_variant(Uuid(as_uuid=True), "sqlite") + + +class Base(DeclarativeBase): + pass + + +class CrawlRun(Base): + __tablename__ = "crawl_runs" + + id: Mapped[uuid.UUID] = mapped_column( + CompatibleUUID, primary_key=True, default=uuid.uuid4 + ) + brand: Mapped[str] = mapped_column(String(50), nullable=False) + market_country: Mapped[str] = mapped_column(String(2), default="DE", nullable=False) + crawler_name: Mapped[str] = mapped_column(String(100), nullable=False) + crawler_version: Mapped[Optional[str]] = mapped_column(String(20), nullable=True) + status: Mapped[str] = mapped_column( + String(20), default=CrawlStatus.PENDING.value, nullable=False + ) + started_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False + ) + finished_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True) + duration_ms: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + + items_discovered: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + items_inserted: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + items_updated: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + items_unmodified: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + items_delisted: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + items_failed: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + + error_summary: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + config_snapshot: Mapped[dict] = mapped_column(CompatibleJSON, default=dict, nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False + ) + + # Relationships + vehicles: Mapped[List["Vehicle"]] = relationship("Vehicle", back_populates="last_crawl_run") + price_history: Mapped[List["VehiclePriceHistory"]] = relationship("VehiclePriceHistory", back_populates="crawl_run") + logs: Mapped[List["CrawlLog"]] = relationship("CrawlLog", back_populates="crawl_run") + + +class Vehicle(Base): + __tablename__ = "vehicles" + __table_args__ = ( + UniqueConstraint("brand", "location_country", "oem_vehicle_id", name="uq_vehicles_brand_market_oem_id"), + Index("idx_vehicles_brand_series", "brand", "series"), + Index("idx_vehicles_fuel_transmission", "fuel_type", "transmission"), + Index("idx_vehicles_price_gross", "price_gross"), + Index("idx_vehicles_status_last_seen", "status", "last_seen_at"), + ) + + # Identifiers + id: Mapped[uuid.UUID] = mapped_column( + CompatibleUUID, primary_key=True, default=uuid.uuid4 + ) + brand: Mapped[str] = mapped_column(String(50), nullable=False) + oem_vehicle_id: Mapped[str] = mapped_column(String(100), nullable=False) + vin: Mapped[Optional[str]] = mapped_column(String(17), nullable=True) + + # Classification + series: Mapped[str] = mapped_column(String(100), nullable=False) + model_name: Mapped[str] = mapped_column(String(255), nullable=False) + trim_line: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + body_type: Mapped[Optional[str]] = mapped_column(String(50), nullable=True) + condition: Mapped[str] = mapped_column(String(30), default=VehicleCondition.NEW.value, nullable=False) + model_year: Mapped[Optional[int]] = mapped_column(SmallInteger, nullable=True) + first_registration_date: Mapped[Optional[date]] = mapped_column(Date, nullable=True) + previous_owners_count: Mapped[Optional[int]] = mapped_column(SmallInteger, nullable=True) + + # Technical Specs & Powertrain + fuel_type: Mapped[str] = mapped_column(String(30), default=FuelType.PETROL.value, nullable=False) + transmission: Mapped[Optional[str]] = mapped_column(String(30), nullable=True) + drivetrain: Mapped[Optional[str]] = mapped_column(String(20), nullable=True) + power_kw: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + power_hp: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + engine_displacement_cc: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + battery_capacity: Mapped[Optional[Decimal]] = mapped_column(Numeric(6, 2), nullable=True) + battery_capacity_unit: Mapped[Optional[str]] = mapped_column(String(10), nullable=True) + battery_capacity_gross_kwh: Mapped[Optional[Decimal]] = mapped_column(Numeric(6, 2), nullable=True) + battery_capacity_net_kwh: Mapped[Optional[Decimal]] = mapped_column(Numeric(6, 2), nullable=True) + charging_power_ac_kw: Mapped[Optional[Decimal]] = mapped_column(Numeric(5, 2), nullable=True) + charging_power_dc_kw: Mapped[Optional[Decimal]] = mapped_column(Numeric(5, 2), nullable=True) + charging_time_dc_minutes: Mapped[Optional[int]] = mapped_column(SmallInteger, nullable=True) + electric_range_wltp_km: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + consumption_combined: Mapped[Optional[Decimal]] = mapped_column(Numeric(5, 2), nullable=True) + co2_emissions_g_km: Mapped[Optional[Decimal]] = mapped_column(Numeric(5, 1), nullable=True) + mileage_km: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + + # Colors & Interior + exterior_color: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + exterior_color_code: Mapped[Optional[str]] = mapped_column(String(50), nullable=True) + exterior_color_generic: Mapped[Optional[str]] = mapped_column(String(50), nullable=True) + interior_color: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + interior_material: Mapped[Optional[str]] = mapped_column(String(50), nullable=True) + + # Pricing & Availability + currency: Mapped[str] = mapped_column(String(3), default="EUR", nullable=False) + base_price_gross: Mapped[Optional[Decimal]] = mapped_column(Numeric(12, 2), nullable=True) + options_price_gross: Mapped[Optional[Decimal]] = mapped_column(Numeric(12, 2), nullable=True) + price_gross: Mapped[Decimal] = mapped_column(Numeric(12, 2), nullable=False) + price_net: Mapped[Optional[Decimal]] = mapped_column(Numeric(12, 2), nullable=True) + msrp_gross: Mapped[Optional[Decimal]] = mapped_column(Numeric(12, 2), nullable=True) + discount_percentage: Mapped[Optional[Decimal]] = mapped_column(Numeric(5, 2), nullable=True) + vat_deductible: Mapped[Optional[bool]] = mapped_column(Boolean, nullable=True) + monthly_rate: Mapped[Optional[Decimal]] = mapped_column(Numeric(10, 2), nullable=True) + leasing_details: Mapped[dict] = mapped_column(CompatibleJSON, default=dict, nullable=False) + status: Mapped[str] = mapped_column(String(30), default=VehicleStatus.IN_STOCK.value, nullable=False) + availability_date: Mapped[Optional[date]] = mapped_column(Date, nullable=True) + delivery_time_days: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + + # Location & Media + dealer_name: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + dealer_id: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + location_city: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + location_postal_code: Mapped[Optional[str]] = mapped_column(String(20), nullable=True) + location_country: Mapped[str] = mapped_column(String(2), default="DE", nullable=False) + latitude: Mapped[Optional[Decimal]] = mapped_column(Numeric(9, 6), nullable=True) + longitude: Mapped[Optional[Decimal]] = mapped_column(Numeric(9, 6), nullable=True) + vehicle_url: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + image_urls: Mapped[list] = mapped_column(CompatibleJSON, default=list, nullable=False) + thumbnail_url: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + + # Timestamps & Tracking + first_seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False + ) + last_seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False + ) + delisted_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True) + last_crawl_run_id: Mapped[Optional[uuid.UUID]] = mapped_column( + CompatibleUUID, ForeignKey("crawl_runs.id", ondelete="SET NULL"), nullable=True + ) + content_hash: Mapped[str] = mapped_column(String(64), nullable=False) + raw_payload: Mapped[dict] = mapped_column(CompatibleJSON, default=dict, nullable=False) + raw_payload_path: Mapped[Optional[str]] = mapped_column(String(512), nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False + ) + + # Relationships + options: Mapped[List["VehicleOption"]] = relationship( + "VehicleOption", back_populates="vehicle", cascade="all, delete-orphan" + ) + price_history: Mapped[List["VehiclePriceHistory"]] = relationship( + "VehiclePriceHistory", back_populates="vehicle", cascade="all, delete-orphan" + ) + last_crawl_run: Mapped[Optional["CrawlRun"]] = relationship("CrawlRun", back_populates="vehicles") + + def get_raw_payload(self) -> dict: + """Loads raw payload from external compressed storage if path is set, else returns raw_payload.""" + if self.raw_payload_path: + from oem_crawler.storage import RawStorageManager + data = RawStorageManager.load_payload(self.raw_payload_path) + if data is not None: + return data + return self.raw_payload or {} + + +class VehicleOption(Base): + __tablename__ = "vehicle_options" + __table_args__ = ( + Index("idx_vehicle_options_vehicle_id", "vehicle_id"), + Index("idx_vehicle_options_code", "option_code"), + Index("idx_vehicle_options_category", "category"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + vehicle_id: Mapped[uuid.UUID] = mapped_column( + CompatibleUUID, ForeignKey("vehicles.id", ondelete="CASCADE"), nullable=False + ) + option_code: Mapped[Optional[str]] = mapped_column(String(50), nullable=True) + name: Mapped[str] = mapped_column(String(255), nullable=False) + category: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + price_gross: Mapped[Optional[Decimal]] = mapped_column(Numeric(10, 2), nullable=True) + price_net: Mapped[Optional[Decimal]] = mapped_column(Numeric(10, 2), nullable=True) + is_standard: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) + description: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + raw_data: Mapped[dict] = mapped_column(CompatibleJSON, default=dict, nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False + ) + + vehicle: Mapped["Vehicle"] = relationship("Vehicle", back_populates="options") + + +class VehiclePriceHistory(Base): + __tablename__ = "vehicle_price_history" + __table_args__ = ( + Index("idx_price_history_vehicle_recorded", "vehicle_id", "recorded_at"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + vehicle_id: Mapped[uuid.UUID] = mapped_column( + CompatibleUUID, ForeignKey("vehicles.id", ondelete="CASCADE"), nullable=False + ) + crawl_run_id: Mapped[uuid.UUID] = mapped_column( + CompatibleUUID, ForeignKey("crawl_runs.id", ondelete="CASCADE"), nullable=False + ) + price_gross: Mapped[Decimal] = mapped_column(Numeric(12, 2), nullable=False) + price_net: Mapped[Optional[Decimal]] = mapped_column(Numeric(12, 2), nullable=True) + monthly_rate: Mapped[Optional[Decimal]] = mapped_column(Numeric(10, 2), nullable=True) + status: Mapped[str] = mapped_column(String(30), nullable=False) + recorded_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False + ) + + vehicle: Mapped["Vehicle"] = relationship("Vehicle", back_populates="price_history") + crawl_run: Mapped["CrawlRun"] = relationship("CrawlRun", back_populates="price_history") + + +class CrawlLog(Base): + __tablename__ = "crawl_logs" + __table_args__ = ( + Index("idx_crawl_logs_run_level", "crawl_run_id", "level"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + crawl_run_id: Mapped[uuid.UUID] = mapped_column( + CompatibleUUID, ForeignKey("crawl_runs.id", ondelete="CASCADE"), nullable=False + ) + vehicle_id: Mapped[Optional[uuid.UUID]] = mapped_column( + CompatibleUUID, ForeignKey("vehicles.id", ondelete="SET NULL"), nullable=True + ) + level: Mapped[str] = mapped_column(String(10), default=LogLevel.INFO.value, nullable=False) + stage: Mapped[str] = mapped_column(String(50), default=CrawlStage.GENERAL.value, nullable=False) + message: Mapped[str] = mapped_column(Text, nullable=False) + error_context: Mapped[dict] = mapped_column(CompatibleJSON, default=dict, nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False + ) + + crawl_run: Mapped["CrawlRun"] = relationship("CrawlRun", back_populates="logs") diff --git a/src/oem_crawler/db/pipeline.py b/src/oem_crawler/db/pipeline.py new file mode 100644 index 0000000..74b1b86 --- /dev/null +++ b/src/oem_crawler/db/pipeline.py @@ -0,0 +1,413 @@ +""" +OEM Vehicle Stock Ingestion Pipeline +Handles database persistence, change detection, price history, delisting detection, and audit logging. +""" +from __future__ import annotations + +import logging +import uuid +from datetime import datetime, timezone +from typing import Generator, List, Optional + +from sqlalchemy import select, create_engine +from sqlalchemy.orm import Session, sessionmaker + +from oem_crawler.config import settings +from oem_crawler.db.models import ( + Base, + CrawlLog, + CrawlRun, + Vehicle, + VehicleOption, + VehiclePriceHistory, +) +from oem_crawler.db.schemas import ( + CrawlStage, + CrawlStatus, + LogLevel, + VehicleCondition, + VehicleIngestSchema, + VehicleStatus, + compute_content_hash, +) +from oem_crawler.db.session import SessionFactory, ensure_sqlite_dir, init_db +from oem_crawler.storage import RawStorageManager + +logger = logging.getLogger(__name__) + + +class StockIngestionPipeline: + def __init__(self, session_factory=None, db_url: Optional[str] = None): + if db_url: + ensure_sqlite_dir(db_url) + custom_engine = create_engine(db_url) + self.session_factory = sessionmaker(bind=custom_engine, autoflush=False, autocommit=False) + Base.metadata.create_all(custom_engine) + elif session_factory: + self.session_factory = session_factory + else: + self.session_factory = SessionFactory + init_db() + + def run_crawler_pipeline( + self, + brand: str, + crawler_name: str, + vehicle_stream: Generator[VehicleIngestSchema, None, None], + market_country: str = "DE", + config_snapshot: Optional[dict] = None, + ) -> CrawlRun: + """ + Executes a complete crawl ingestion run: + 1. Creates CrawlRun audit record (RUNNING) + 2. Streams vehicles, performs change-detection via content_hash + 3. Inserts new vehicles / options / price history + 4. Updates existing records on spec/price change + 5. Detects delisted vehicles (if full market scan) + 6. Finalizes CrawlRun (COMPLETED/FAILED) + """ + session: Session = self.session_factory() + start_time = datetime.now(timezone.utc) + + # 1. Initialize Crawl Run + crawl_run = CrawlRun( + id=uuid.uuid4(), + brand=brand, + market_country=market_country, + crawler_name=crawler_name, + status=CrawlStatus.RUNNING.value, + started_at=start_time, + config_snapshot=config_snapshot or {}, + ) + session.add(crawl_run) + session.commit() + logger.info(f"Started CrawlRun {crawl_run.id} for {brand} ({market_country})") + + seen_oem_ids = set() + + try: + for vehicle_data in vehicle_stream: + crawl_run.items_discovered += 1 + seen_oem_ids.add(vehicle_data.oem_vehicle_id) + + try: + self._ingest_vehicle(session, crawl_run, vehicle_data) + except Exception as e: + crawl_run.items_failed += 1 + logger.error(f"Error ingesting vehicle {vehicle_data.oem_vehicle_id}: {e}", exc_info=True) + self._log_event( + session=session, + crawl_run_id=crawl_run.id, + level=LogLevel.ERROR, + stage=CrawlStage.PERSIST, + message=f"Failed to persist vehicle {vehicle_data.oem_vehicle_id}: {str(e)}", + error_context={"oem_vehicle_id": vehicle_data.oem_vehicle_id, "error": str(e)}, + ) + + # Periodic batch commit + if crawl_run.items_discovered % 50 == 0: + session.commit() + + # Delisting detection for full market scans + if config_snapshot and config_snapshot.get("is_full_market_scan", False): + stock_type = config_snapshot.get("stock_type") + self._detect_delisted_vehicles(session, crawl_run, seen_oem_ids, brand, market_country, stock_type=stock_type) + + # Finalize CrawlRun + finish_time = datetime.now(timezone.utc) + crawl_run.finished_at = finish_time + crawl_run.duration_ms = int((finish_time - start_time).total_seconds() * 1000) + crawl_run.status = CrawlStatus.COMPLETED.value + session.commit() + logger.info( + f"CrawlRun {crawl_run.id} finished: discovered={crawl_run.items_discovered}, " + f"inserted={crawl_run.items_inserted}, updated={crawl_run.items_updated}, " + f"unmodified={crawl_run.items_unmodified}, delisted={crawl_run.items_delisted}, failed={crawl_run.items_failed}" + ) + return crawl_run + + except Exception as err: + session.rollback() + finish_time = datetime.now(timezone.utc) + crawl_run.finished_at = finish_time + crawl_run.duration_ms = int((finish_time - start_time).total_seconds() * 1000) + crawl_run.status = CrawlStatus.FAILED.value + crawl_run.error_summary = str(err) + session.commit() + logger.error(f"CrawlRun {crawl_run.id} failed: {err}", exc_info=True) + return crawl_run + finally: + session.close() + + def _ingest_vehicle( + self, + session: Session, + crawl_run: CrawlRun, + data: VehicleIngestSchema, + ) -> None: + """Ingests or updates a single vehicle using content_hash change detection.""" + content_hash = compute_content_hash(data) + now_ts = datetime.now(timezone.utc) + + stmt = select(Vehicle).where( + Vehicle.brand == data.brand, + Vehicle.location_country == data.location_country, + Vehicle.oem_vehicle_id == data.oem_vehicle_id, + ) + existing: Optional[Vehicle] = session.scalars(stmt).first() + + if existing is None: + # CASE 1: Brand new vehicle in stock + vehicle_id = uuid.uuid4() + options_gross_sum = sum( + opt.price_gross for opt in data.options if opt.price_gross is not None + ) + + new_veh = Vehicle( + id=vehicle_id, + brand=data.brand, + oem_vehicle_id=data.oem_vehicle_id, + vin=data.vin, + series=data.series, + model_name=data.model_name, + trim_line=data.trim_line, + body_type=data.body_type, + condition=data.condition.value, + model_year=data.model_year, + first_registration_date=data.first_registration_date, + previous_owners_count=data.previous_owners_count, + fuel_type=data.fuel_type.value, + transmission=data.transmission.value if data.transmission else None, + drivetrain=data.drivetrain.value if data.drivetrain else None, + power_kw=data.power_kw, + power_hp=data.power_hp, + engine_displacement_cc=data.engine_displacement_cc, + battery_capacity=data.battery_capacity, + battery_capacity_unit=data.battery_capacity_unit.value if data.battery_capacity_unit else None, + battery_capacity_gross_kwh=data.battery_capacity_gross_kwh, + battery_capacity_net_kwh=data.battery_capacity_net_kwh, + charging_power_ac_kw=data.charging_power_ac_kw, + charging_power_dc_kw=data.charging_power_dc_kw, + charging_time_dc_minutes=data.charging_time_dc_minutes, + electric_range_wltp_km=data.electric_range_wltp_km, + consumption_combined=data.consumption_combined, + co2_emissions_g_km=data.co2_emissions_g_km, + mileage_km=data.mileage_km, + exterior_color=data.exterior_color, + exterior_color_code=data.exterior_color_code, + exterior_color_generic=data.exterior_color_generic, + interior_color=data.interior_color, + interior_material=data.interior_material, + currency=data.currency, + base_price_gross=data.base_price_gross, + options_price_gross=data.options_price_gross or (options_gross_sum if options_gross_sum > 0 else None), + price_gross=data.price_gross, + price_net=data.price_net, + msrp_gross=data.msrp_gross, + discount_percentage=data.discount_percentage, + vat_deductible=data.vat_deductible, + monthly_rate=data.monthly_rate, + leasing_details=data.leasing_details, + status=data.status.value, + availability_date=data.availability_date, + delivery_time_days=data.delivery_time_days, + dealer_name=data.dealer_name, + dealer_id=data.dealer_id, + location_city=data.location_city, + location_postal_code=data.location_postal_code, + location_country=data.location_country, + latitude=data.latitude, + longitude=data.longitude, + vehicle_url=data.vehicle_url, + image_urls=data.image_urls, + thumbnail_url=data.thumbnail_url, + first_seen_at=now_ts, + last_seen_at=now_ts, + last_crawl_run_id=crawl_run.id, + content_hash=content_hash, + raw_payload={}, + raw_payload_path=( + data.raw_payload_path + or ( + RawStorageManager.save_payload( + brand=data.brand, + oem_vehicle_id=data.oem_vehicle_id, + payload=data.raw_payload, + timestamp=now_ts, + ) + if (settings.ENABLE_RAW_EXTRACTION and data.raw_payload) + else None + ) + ), + created_at=now_ts, + updated_at=now_ts, + ) + session.add(new_veh) + + # Insert Options + for opt in data.options: + session.add( + VehicleOption( + vehicle_id=vehicle_id, + option_code=opt.option_code, + name=opt.name, + category=opt.category, + price_gross=opt.price_gross, + price_net=opt.price_net, + is_standard=opt.is_standard, + description=opt.description, + raw_data={}, + ) + ) + + # Record Initial Price History + session.add( + VehiclePriceHistory( + vehicle_id=vehicle_id, + crawl_run_id=crawl_run.id, + price_gross=data.price_gross, + price_net=data.price_net, + monthly_rate=data.monthly_rate, + status=data.status.value, + recorded_at=now_ts, + ) + ) + + crawl_run.items_inserted += 1 + session.flush() + + elif existing.content_hash == content_hash: + # CASE 2: Vehicle observed with identical specs, options, and price + existing.last_seen_at = now_ts + existing.last_crawl_run_id = crawl_run.id + if existing.status == VehicleStatus.DELISTED.value: + existing.status = VehicleStatus.IN_STOCK.value + existing.delisted_at = None + crawl_run.items_unmodified += 1 + + else: + # CASE 3: Vehicle observed with updated price, specs, or equipment + old_price = existing.price_gross + old_rate = existing.monthly_rate + + existing.price_gross = data.price_gross + existing.price_net = data.price_net + existing.monthly_rate = data.monthly_rate + existing.msrp_gross = data.msrp_gross + existing.discount_percentage = data.discount_percentage + existing.power_kw = data.power_kw + existing.power_hp = data.power_hp + existing.battery_capacity = data.battery_capacity + existing.battery_capacity_unit = data.battery_capacity_unit.value if data.battery_capacity_unit else None + existing.battery_capacity_gross_kwh = data.battery_capacity_gross_kwh + existing.battery_capacity_net_kwh = data.battery_capacity_net_kwh + existing.charging_power_ac_kw = data.charging_power_ac_kw + existing.charging_power_dc_kw = data.charging_power_dc_kw + existing.charging_time_dc_minutes = data.charging_time_dc_minutes + existing.electric_range_wltp_km = data.electric_range_wltp_km + existing.mileage_km = data.mileage_km + existing.exterior_color = data.exterior_color + existing.exterior_color_code = data.exterior_color_code + existing.exterior_color_generic = data.exterior_color_generic + existing.status = data.status.value + existing.last_seen_at = now_ts + existing.last_crawl_run_id = crawl_run.id + existing.content_hash = content_hash + if data.raw_payload_path: + existing.raw_payload_path = data.raw_payload_path + elif settings.ENABLE_RAW_EXTRACTION and data.raw_payload: + existing.raw_payload_path = RawStorageManager.save_payload( + brand=data.brand, + oem_vehicle_id=data.oem_vehicle_id, + payload=data.raw_payload, + timestamp=now_ts, + ) + existing.raw_payload = {} + existing.updated_at = now_ts + + # Re-sync options if options changed + session.query(VehicleOption).filter(VehicleOption.vehicle_id == existing.id).delete() + for opt in data.options: + session.add( + VehicleOption( + vehicle_id=existing.id, + option_code=opt.option_code, + name=opt.name, + category=opt.category, + price_gross=opt.price_gross, + price_net=opt.price_net, + is_standard=opt.is_standard, + description=opt.description, + raw_data={}, + ) + ) + + # Record Price / Status snapshot if price, rate, or status changed + if old_price != data.price_gross or old_rate != data.monthly_rate or existing.status != data.status.value: + session.add( + VehiclePriceHistory( + vehicle_id=existing.id, + crawl_run_id=crawl_run.id, + price_gross=data.price_gross, + price_net=data.price_net, + monthly_rate=data.monthly_rate, + status=data.status.value, + recorded_at=now_ts, + ) + ) + + crawl_run.items_updated += 1 + + def _detect_delisted_vehicles( + self, + session: Session, + crawl_run: CrawlRun, + seen_oem_ids: set, + brand: str, + market_country: str, + stock_type: Optional[str] = None, + ) -> None: + """Marks active in-stock vehicles as delisted if they were absent from a full scan.""" + now_ts = datetime.now(timezone.utc) + stmt = ( + select(Vehicle) + .where( + Vehicle.brand == brand, + Vehicle.location_country == market_country, + Vehicle.status == VehicleStatus.IN_STOCK.value, + ) + ) + if stock_type == "new": + stmt = stmt.where(Vehicle.condition == VehicleCondition.NEW.value) + elif stock_type == "used": + stmt = stmt.where( + Vehicle.condition.in_([VehicleCondition.USED.value, VehicleCondition.CERTIFIED_PRE_OWNED.value]) + ) + + active_vehicles: List[Vehicle] = session.scalars(stmt).all() + for veh in active_vehicles: + if veh.oem_vehicle_id not in seen_oem_ids: + veh.status = VehicleStatus.DELISTED.value + veh.delisted_at = now_ts + veh.updated_at = now_ts + crawl_run.items_delisted += 1 + + def _log_event( + self, + session: Session, + crawl_run_id: uuid.UUID, + level: LogLevel, + stage: CrawlStage, + message: str, + error_context: Optional[dict] = None, + ) -> None: + """Appends a diagnostic log entry to crawl_logs.""" + log_entry = CrawlLog( + crawl_run_id=crawl_run_id, + level=level.value, + stage=stage.value, + message=message, + error_context=error_context or {}, + ) + session.add(log_entry) + session.commit() diff --git a/src/oem_crawler/db/schemas.py b/src/oem_crawler/db/schemas.py new file mode 100644 index 0000000..168df0b --- /dev/null +++ b/src/oem_crawler/db/schemas.py @@ -0,0 +1,233 @@ +""" +Pydantic schemas and deterministic content hashing logic. +""" +from __future__ import annotations + +import hashlib +import json +from datetime import date, datetime +from decimal import Decimal +from enum import Enum +from typing import Any, List, Optional + +from pydantic import BaseModel, Field, ConfigDict + + +# ============================================================================= +# Enums +# ============================================================================= + +class CrawlStatus(str, Enum): + PENDING = "PENDING" + RUNNING = "RUNNING" + COMPLETED = "COMPLETED" + FAILED = "FAILED" + PARTIAL = "PARTIAL" + + +class VehicleCondition(str, Enum): + NEW = "new" + USED = "used" + DEMONSTRATION = "demonstration" + DAY_REGISTRATION = "day_registration" + CERTIFIED_PRE_OWNED = "certified_pre_owned" + + +class FuelType(str, Enum): + BEV = "bev" + PHEV = "phev" + PETROL = "petrol" + DIESEL = "diesel" + MHEV = "mhev" + FCEV = "fcev" + CNG = "cng" + LPG = "lpg" + HYBRID_OTHER = "hybrid_other" + + +class BatteryCapacityUnit(str, Enum): + KWH = "kwh" + AH = "ah" + + +class TransmissionType(str, Enum): + AUTOMATIC = "automatic" + MANUAL = "manual" + DIRECT_DRIVE = "direct_drive" + DUAL_CLUTCH = "dual_clutch" + CVT = "cvt" + OTHER = "other" + + +class DrivetrainType(str, Enum): + AWD = "awd" + RWD = "rwd" + FWD = "fwd" + FOUR_BY_FOUR = "4x4" + OTHER = "other" + + +class VehicleStatus(str, Enum): + IN_STOCK = "in_stock" + IN_TRANSIT = "in_transit" + ON_ORDER = "on_order" + RESERVED = "reserved" + SOLD = "sold" + DELISTED = "delisted" + + +class LogLevel(str, Enum): + DEBUG = "DEBUG" + INFO = "INFO" + WARNING = "WARNING" + ERROR = "ERROR" + CRITICAL = "CRITICAL" + + +class CrawlStage(str, Enum): + FETCH = "FETCH" + PARSE = "PARSE" + VALIDATE = "VALIDATE" + PERSIST = "PERSIST" + DELIST_CHECK = "DELIST_CHECK" + GENERAL = "GENERAL" + + +# ============================================================================= +# Pydantic Ingestion / Validation Schemas +# ============================================================================= + +class OptionSchema(BaseModel): + model_config = ConfigDict(extra="ignore") + + option_code: Optional[str] = None + name: str + category: Optional[str] = None + price_gross: Optional[Decimal] = None + price_net: Optional[Decimal] = None + is_standard: bool = False + description: Optional[str] = None + raw_data: dict = Field(default_factory=dict) + + +class VehicleIngestSchema(BaseModel): + model_config = ConfigDict(extra="ignore") + + # Identifiers + brand: str + oem_vehicle_id: str + vin: Optional[str] = None + + # Classification + series: str + model_name: str + trim_line: Optional[str] = None + body_type: Optional[str] = None + condition: VehicleCondition = VehicleCondition.NEW + model_year: Optional[int] = None + first_registration_date: Optional[date] = None + previous_owners_count: Optional[int] = None + + # Powertrain & Specs + fuel_type: FuelType = FuelType.PETROL + transmission: Optional[TransmissionType] = None + drivetrain: Optional[DrivetrainType] = None + power_kw: Optional[int] = None + power_hp: Optional[int] = None + engine_displacement_cc: Optional[int] = None + battery_capacity: Optional[Decimal] = None + battery_capacity_unit: Optional[BatteryCapacityUnit] = None + battery_capacity_gross_kwh: Optional[Decimal] = None + battery_capacity_net_kwh: Optional[Decimal] = None + charging_power_ac_kw: Optional[Decimal] = None + charging_power_dc_kw: Optional[Decimal] = None + charging_time_dc_minutes: Optional[int] = None + electric_range_wltp_km: Optional[int] = None + consumption_combined: Optional[Decimal] = None + co2_emissions_g_km: Optional[Decimal] = None + mileage_km: int = 0 + + # Colors + exterior_color: Optional[str] = None + exterior_color_code: Optional[str] = None + exterior_color_generic: Optional[str] = None + interior_color: Optional[str] = None + interior_material: Optional[str] = None + + # Pricing & Availability + currency: str = "EUR" + base_price_gross: Optional[Decimal] = None + options_price_gross: Optional[Decimal] = None + price_gross: Decimal + price_net: Optional[Decimal] = None + msrp_gross: Optional[Decimal] = None + discount_percentage: Optional[Decimal] = None + vat_deductible: Optional[bool] = None + monthly_rate: Optional[Decimal] = None + leasing_details: dict = Field(default_factory=dict) + status: VehicleStatus = VehicleStatus.IN_STOCK + availability_date: Optional[date] = None + delivery_time_days: Optional[int] = None + + # Location & Media + dealer_name: Optional[str] = None + dealer_id: Optional[str] = None + location_city: Optional[str] = None + location_postal_code: Optional[str] = None + location_country: str = "DE" + latitude: Optional[Decimal] = None + longitude: Optional[Decimal] = None + vehicle_url: Optional[str] = None + image_urls: List[str] = Field(default_factory=list) + thumbnail_url: Optional[str] = None + + # Options & Equipment + options: List[OptionSchema] = Field(default_factory=list) + + # Raw Payload & Storage + raw_payload: dict = Field(default_factory=dict) + raw_payload_path: Optional[str] = None + + +def compute_content_hash(vehicle: VehicleIngestSchema) -> str: + """ + Computes a deterministic SHA-256 hash of core vehicle attributes, + prices, and equipment options to detect stock updates and changes quickly. + """ + sorted_options = sorted( + [ + { + "code": opt.option_code or "", + "name": opt.name, + "price": str(opt.price_gross) if opt.price_gross is not None else "", + "is_std": opt.is_standard, + } + for opt in vehicle.options + ], + key=lambda x: (x["code"], x["name"]), + ) + + hash_dict = { + "brand": vehicle.brand.lower(), + "oem_vehicle_id": vehicle.oem_vehicle_id, + "vin": vehicle.vin or "", + "series": vehicle.series, + "model_name": vehicle.model_name, + "condition": vehicle.condition.value, + "fuel_type": vehicle.fuel_type.value, + "transmission": vehicle.transmission.value if vehicle.transmission else "", + "drivetrain": vehicle.drivetrain.value if vehicle.drivetrain else "", + "power_kw": vehicle.power_kw, + "battery_capacity": str(vehicle.battery_capacity) if vehicle.battery_capacity is not None else "", + "battery_capacity_unit": vehicle.battery_capacity_unit.value if vehicle.battery_capacity_unit else "", + "mileage_km": vehicle.mileage_km, + "price_gross": str(vehicle.price_gross), + "price_net": str(vehicle.price_net) if vehicle.price_net is not None else "", + "monthly_rate": str(vehicle.monthly_rate) if vehicle.monthly_rate is not None else "", + "status": vehicle.status.value, + "exterior_color": vehicle.exterior_color or "", + "options": sorted_options, + } + + serialized = json.dumps(hash_dict, sort_keys=True) + return hashlib.sha256(serialized.encode("utf-8")).hexdigest() diff --git a/src/oem_crawler/db/session.py b/src/oem_crawler/db/session.py new file mode 100644 index 0000000..3156cde --- /dev/null +++ b/src/oem_crawler/db/session.py @@ -0,0 +1,38 @@ +""" +Database session and connection management. +""" +from typing import Generator +from sqlalchemy import create_engine +from sqlalchemy.orm import Session, sessionmaker + +from pathlib import Path + +from oem_crawler.config import settings +from oem_crawler.db.models import Base + + +def ensure_sqlite_dir(url: str) -> None: + """Ensures parent directories exist for local SQLite file paths.""" + if url.startswith("sqlite:///"): + path_str = url.replace("sqlite:///", "") + if path_str and not path_str.startswith(":memory:"): + Path(path_str).parent.mkdir(parents=True, exist_ok=True) + + +ensure_sqlite_dir(settings.DATABASE_URL) +engine = create_engine(settings.DATABASE_URL, echo=settings.DB_ECHO_SQL) +SessionFactory = sessionmaker(bind=engine, autoflush=False, autocommit=False) + + +def init_db(): + """Initializes schema tables if not present.""" + Base.metadata.create_all(engine) + + +def get_db_session() -> Generator[Session, None, None]: + """Yields an isolated database session for transactional tasks.""" + session = SessionFactory() + try: + yield session + finally: + session.close() diff --git a/src/oem_crawler/storage.py b/src/oem_crawler/storage.py new file mode 100644 index 0000000..6f74c99 --- /dev/null +++ b/src/oem_crawler/storage.py @@ -0,0 +1,123 @@ +""" +Storage manager for raw OEM payloads. +Provides compressed file persistence (.json.gz) outside of the relational database. +""" +from __future__ import annotations + +import gzip +import json +import logging +import os +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, Optional + +from oem_crawler.config import settings + +logger = logging.getLogger(__name__) + + +class RawStorageManager: + """Manages writing, reading, and locating compressed raw payloads on the filesystem.""" + + @staticmethod + def get_storage_path( + brand: str, + oem_vehicle_id: str, + timestamp: Optional[datetime] = None, + ) -> str: + """ + Computes the relative path for a vehicle's raw payload file. + Format: {brand}/{YYYY-MM-DD}/{safe_id}.json.gz + """ + date_str = (timestamp or datetime.now(timezone.utc)).strftime("%Y-%m-%d") + safe_brand = brand.strip().lower() + safe_id = "".join(c if c.isalnum() or c in ("-", "_") else "_" for c in str(oem_vehicle_id)) + return f"{safe_brand}/{date_str}/{safe_id}.json.gz" + + @classmethod + def save_payload( + cls, + brand: str, + oem_vehicle_id: str, + payload: Dict[str, Any], + timestamp: Optional[datetime] = None, + base_dir: Optional[str] = None, + compression_level: Optional[int] = None, + ) -> Optional[str]: + """ + Saves a compressed JSON payload (.json.gz) to disk if extraction is enabled. + Returns the relative file path if saved, or None if disabled or empty. + """ + if not settings.ENABLE_RAW_EXTRACTION or not payload: + return None + + rel_path = cls.get_storage_path(brand=brand, oem_vehicle_id=oem_vehicle_id, timestamp=timestamp) + storage_root = Path(base_dir or settings.RAW_STORAGE_DIR) + target_path = storage_root / rel_path + target_path.parent.mkdir(parents=True, exist_ok=True) + + level = compression_level if compression_level is not None else settings.RAW_COMPRESSION_LEVEL + + # Write to temporary file first for atomic commit + temp_path = target_path.with_suffix(".tmp") + try: + with gzip.open(temp_path, "wt", encoding="utf-8", compresslevel=level) as f: + json.dump(payload, f, ensure_ascii=False) + temp_path.replace(target_path) + return rel_path + except Exception as e: + if temp_path.exists(): + try: + temp_path.unlink() + except OSError: + pass + logger.error(f"Failed to save raw payload for {brand} {oem_vehicle_id}: {e}") + raise + + @classmethod + def load_payload( + cls, + rel_path: str, + base_dir: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + """ + Reads and decompresses a raw payload from disk. + """ + if not rel_path: + return None + + storage_root = Path(base_dir or settings.RAW_STORAGE_DIR) + full_path = storage_root / rel_path + + if not full_path.exists(): + return None + + try: + with gzip.open(full_path, "rt", encoding="utf-8") as f: + return json.load(f) + except Exception as e: + logger.error(f"Failed to read raw payload from {full_path}: {e}") + return None + + @classmethod + def delete_payload( + cls, + rel_path: str, + base_dir: Optional[str] = None, + ) -> bool: + """Deletes a raw payload file if it exists.""" + if not rel_path: + return False + + storage_root = Path(base_dir or settings.RAW_STORAGE_DIR) + full_path = storage_root / rel_path + + if full_path.exists(): + try: + full_path.unlink() + return True + except OSError as e: + logger.error(f"Failed to delete {full_path}: {e}") + return False + diff --git a/src/oem_crawler/tools/README.md b/src/oem_crawler/tools/README.md new file mode 100644 index 0000000..c19e778 --- /dev/null +++ b/src/oem_crawler/tools/README.md @@ -0,0 +1,130 @@ +# πŸ› οΈ OEM Crawler Developer Tools + +This folder contains inspection, auditing, and debugging utilities for the OEM Crawler project. + +--- + +## πŸ” Vehicle Stock Inspector (`view_stock.py`) + +`view_stock.py` is a terminal utility that inspects the local database, shows high-level statistics across all manufacturer crawl runs, breaks down inventory by brand, and displays vehicle cards with their factory options and EV specifications. + +### Running the Tool + +You can run the inspector either via the CLI command `oem-inspect` or directly with Python: + +```bash +# 1. Using the CLI shortcut +uv run oem-inspect + +# 2. Or using python module execution +uv run python -m oem_crawler.tools.view_stock + +# 3. Or running the script directly +uv run python src/oem_crawler/tools/view_stock.py +``` + +--- + +## πŸ“‹ Command-Line Options + +| Option | Default | Description | +| :--- | :--- | :--- | +| `--summary` | `False` | Displays the high-level aggregate summary and brand breakdown table only, without listing individual vehicles. | +| `--brand ` | `None` | Filters vehicles by manufacturer (`AUDI`, `BMW`, `MERCEDES`, `PORSCHE`). | +| `--limit ` | `10` | Number of vehicle cards to display in the list. | +| `--db-url ` | `sqlite:///data/crawler_stock.db` | SQLAlchemy connection URL (can point to local SQLite file or PostgreSQL). | + +### Examples + +```bash +# Display only the high-level summary and brand table +uv run oem-inspect --summary + +# Inspect the 15 latest Audi vehicles +uv run oem-inspect --brand audi --limit 15 + +# Inspect Porsche inventory +uv run oem-inspect --brand porsche --limit 10 + +# Inspect against a Docker PostgreSQL database +uv run oem-inspect --db-url postgresql+psycopg://crawler_user:crawler_password@localhost:5432/oem_stock_db +``` + +--- + +## πŸ’» Direct Database Queries via `sqlite3` CLI + +For ad-hoc queries on the local SQLite development database (`data/crawler_stock.db`): + +```bash +# 1. Inventory count and pricing stats per brand +sqlite3 -table data/crawler_stock.db " +SELECT brand, + count(*) AS vehicles, + printf('%.0f €', avg(price_gross)) AS avg_price, + printf('%.0f €', min(price_gross)) AS min_price, + printf('%.0f €', max(price_gross)) AS max_price +FROM vehicles +GROUP BY brand;" + +# 2. Powertrain and Fuel Type breakdown +sqlite3 -table data/crawler_stock.db " +SELECT brand, fuel_type, condition, count(*) AS count +FROM vehicles +GROUP BY brand, fuel_type, condition +ORDER BY brand, count DESC;" + +# 3. Search for specific factory options +sqlite3 -table data/crawler_stock.db " +SELECT v.brand, v.model_name, vo.option_code, vo.name, vo.category +FROM vehicle_options vo +JOIN vehicles v ON vo.vehicle_id = v.id +WHERE vo.name LIKE '%panoramadach%' +LIMIT 10;" + +# 4. View latest crawl run audit history +sqlite3 -table data/crawler_stock.db " +SELECT id, brand, status, items_discovered, items_inserted, + printf('%.1fs', duration_ms/1000.0) AS duration, started_at +FROM crawl_runs +ORDER BY started_at DESC +LIMIT 8;" +``` + +--- + +## πŸ”„ Database Restore & Upsert from Raw Payloads (`restore_from_raw.py`) + +`restore_from_raw.py` reads and decompresses raw OEM JSON files (`.json.gz`) from your storage directory and upserts them into the database. It is designed for disaster recovery, rebuilding a database from scratch, or re-indexing without having to re-crawl OEM portals. + +### Running the Tool + +```bash +# 1. Using the CLI shortcut +uv run oem-restore + +# 2. Or using the script launcher +uv run python scripts/restore_from_raw.py + +# 3. Dry-run validation (parse and validate without DB changes) +uv run oem-restore --dry-run + +# 4. Restore only a specific brand +uv run oem-restore --brand porsche + +# 5. Restore into a fresh or remote database +uv run oem-restore --db-url "sqlite:///data/restored_stock.db" +``` + +### Options + +| Option | Default | Description | +| :--- | :--- | :--- | +| `--raw-dir ` | `data/raw` | Source directory containing compressed `.json.gz` files. | +| `--db-url ` | `sqlite:///data/crawler_stock.db` | Destination database connection string. | +| `--brand ` | `all` | Filter by brand (`bmw`, `audi`, `mercedes-benz`, `porsche`, or comma-separated). | +| `--limit ` | `None` | Process up to N files per brand (useful for testing). | +| `--batch-size ` | `500` | Number of records per commit transaction. | +| `--dry-run` | `False` | Validate raw files without altering database state. | +| `--country ` | `DE` | Market ISO code. | + diff --git a/src/oem_crawler/tools/__init__.py b/src/oem_crawler/tools/__init__.py new file mode 100644 index 0000000..c51dbc0 --- /dev/null +++ b/src/oem_crawler/tools/__init__.py @@ -0,0 +1,7 @@ +""" +OEM Crawler Developer & Inspection Tools. +""" +from oem_crawler.tools.restore_from_raw import restore_database +from oem_crawler.tools.view_stock import inspect_stock, main + +__all__ = ["inspect_stock", "main", "restore_database"] diff --git a/src/oem_crawler/tools/restore_from_raw.py b/src/oem_crawler/tools/restore_from_raw.py new file mode 100644 index 0000000..b5ea542 --- /dev/null +++ b/src/oem_crawler/tools/restore_from_raw.py @@ -0,0 +1,328 @@ +""" +Restore & Upsert Tool for OEM Crawler +Rebuilds, repairs, or re-populates the database by reading and parsing raw compressed payload files (.json.gz). +Useful for disaster recovery, migrating databases, or re-indexing after schema changes. +""" +from __future__ import annotations + +import argparse +import gzip +import json +import logging +import sys +import time +from pathlib import Path +from typing import Any, Dict, Generator, List, Optional, Tuple + +from oem_crawler.config import settings +from oem_crawler.crawlers.audi.parser import parse_audi_vehicle +from oem_crawler.crawlers.bmw.parser import parse_bmw_vehicle +from oem_crawler.crawlers.mercedes.parser import parse_mercedes_used_dict, parse_mercedes_vehicle +from oem_crawler.crawlers.porsche.parser import parse_porsche_vehicle +from oem_crawler.db.pipeline import StockIngestionPipeline +from oem_crawler.db.schemas import VehicleIngestSchema +from oem_crawler.db.session import ensure_sqlite_dir + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + datefmt="%H:%M:%S", +) +logger = logging.getLogger("restore_from_raw") + + +def parse_raw_payload( + brand: str, + payload: Dict[str, Any], + market_country: str = "DE", + rel_path: Optional[str] = None, +) -> Optional[VehicleIngestSchema]: + """ + Normalizes a raw payload dictionary into a validated VehicleIngestSchema based on brand. + """ + brand_norm = brand.strip().lower() + + if "audi" in brand_norm: + basic_item = payload.get("vehicleBasic", payload) + detail_data = payload.get("detail") + veh = parse_audi_vehicle( + basic_item=basic_item, + detail_data=detail_data, + market_country=market_country, + include_raw=False, + ) + elif "bmw" in brand_norm: + veh = parse_bmw_vehicle( + raw=payload, + market_country=market_country, + include_raw=False, + ) + elif "porsche" in brand_norm: + search_item = payload.get("search_item", payload) + detail_data = payload.get("detail_data") + veh = parse_porsche_vehicle( + search_item=search_item, + detail_data=detail_data, + market_country=market_country, + include_raw=False, + ) + elif "mercedes" in brand_norm: + if "search" in payload: + veh = parse_mercedes_vehicle( + raw_search=payload["search"], + raw_price=payload.get("price"), + raw_vpc=payload.get("vpc"), + market_country=market_country, + include_raw=False, + ) + elif "card_id" in payload or "basic_data" in payload or "card_attributes" in payload: + veh = parse_mercedes_used_dict( + data=payload, + market_country=market_country, + include_raw=False, + ) + else: + return None + else: + logger.warning(f"Unsupported brand '{brand}' for raw payload.") + return None + + if veh and rel_path: + veh.raw_payload_path = rel_path + + return veh + + +def stream_brand_files( + brand_dir: Path, + raw_root: Path, + market_country: str = "DE", + limit: Optional[int] = None, +) -> Generator[Tuple[VehicleIngestSchema, str], None, None]: + """ + Yields parsed (VehicleIngestSchema, rel_path) tuples from a brand directory. + """ + brand = brand_dir.name + count = 0 + + # Scan all .json.gz files recursively (partitioned by date) + for file_path in sorted(brand_dir.glob("**/*.json.gz")): + rel_path = str(file_path.relative_to(raw_root)) + try: + with gzip.open(file_path, "rt", encoding="utf-8") as f: + payload = json.load(f) + + if not isinstance(payload, dict): + continue + + # Skip dummy test fixtures + if payload.get("id") == "TEST-MB-EQE-001" and len(payload) == 1: + continue + + veh = parse_raw_payload( + brand=brand, + payload=payload, + market_country=market_country, + rel_path=rel_path, + ) + + if veh: + yield veh, rel_path + count += 1 + if limit and count >= limit: + break + except Exception as e: + logger.warning(f"Error parsing raw file {rel_path}: {e}") + + +def restore_database( + raw_dir: Path, + db_url: str, + target_brands: Optional[List[str]] = None, + batch_size: int = 500, + limit: Optional[int] = None, + dry_run: bool = False, + market_country: str = "DE", +) -> None: + """ + Scans raw directory and upserts vehicles into the database using StockIngestionPipeline. + """ + raw_root = raw_dir.resolve() + if not raw_root.exists(): + logger.error(f"Raw storage directory not found: {raw_root}") + sys.exit(1) + + # Discover brand subdirectories + available_brand_dirs = [d for d in sorted(raw_root.iterdir()) if d.is_dir() and not d.name.startswith(".")] + if target_brands: + filter_set = {b.strip().lower() for b in target_brands} + brand_dirs = [d for d in available_brand_dirs if d.name.lower() in filter_set] + else: + brand_dirs = available_brand_dirs + + if not brand_dirs: + logger.warning(f"No matching brand directories found in {raw_root}") + return + + logger.info("=" * 70) + logger.info("OEM VEHICLE STOCK DATABASE RESTORE & UPSERT TOOL") + logger.info("=" * 70) + logger.info(f"Source Directory: {raw_root}") + logger.info(f"Target Database: {db_url}") + logger.info(f"Brands to Scan: {', '.join(d.name.upper() for d in brand_dirs)}") + logger.info(f"Dry Run Mode: {dry_run}") + if limit: + logger.info(f"Per-Brand Limit: {limit:,}") + logger.info("=" * 70 + "\n") + + ensure_sqlite_dir(db_url) + pipeline = StockIngestionPipeline(db_url=db_url) if not dry_run else None + + overall_start = time.time() + grand_totals = { + "discovered": 0, + "parsed": 0, + "inserted": 0, + "updated": 0, + "unmodified": 0, + "failed": 0, + } + + for b_dir in brand_dirs: + brand_name = b_dir.name.upper() + files = list(b_dir.glob("**/*.json.gz")) + total_files = len(files) + logger.info(f"[{brand_name}] Found {total_files:,} raw payload files. Starting ingestion...") + + brand_start = time.time() + brand_parsed = 0 + + def vehicle_generator(): + nonlocal brand_parsed + for veh, _ in stream_brand_files(b_dir, raw_root, market_country=market_country, limit=limit): + brand_parsed += 1 + yield veh + + if dry_run: + # In dry-run mode, iterate and validate without pipeline + for veh, rel_path in stream_brand_files(b_dir, raw_root, market_country=market_country, limit=limit): + brand_parsed += 1 + if brand_parsed % 1000 == 0 or brand_parsed == total_files: + logger.info(f"[{brand_name}] [Dry Run] Validated {brand_parsed:,}/{total_files:,} payloads...") + + elapsed = time.time() - brand_start + rate = brand_parsed / elapsed if elapsed > 0 else 0 + logger.info(f"[{brand_name}] [Dry Run] Finished validating {brand_parsed:,} vehicles ({rate:.0f} veh/sec).") + grand_totals["discovered"] += total_files + grand_totals["parsed"] += brand_parsed + else: + # Ingest via pipeline + run_result = pipeline.run_crawler_pipeline( + brand=brand_name, + crawler_name="restore_from_raw", + vehicle_stream=vehicle_generator(), + market_country=market_country, + ) + + elapsed = time.time() - brand_start + rate = run_result.items_discovered / elapsed if elapsed > 0 else 0 + logger.info( + f"[{brand_name}] Ingestion complete: " + f"Discovered={run_result.items_discovered:,} | " + f"Inserted={run_result.items_inserted:,} | " + f"Updated={run_result.items_updated:,} | " + f"Unmodified={run_result.items_unmodified:,} | " + f"Failed={run_result.items_failed:,} " + f"({rate:.0f} veh/sec in {elapsed:.1f}s)" + ) + + grand_totals["discovered"] += total_files + grand_totals["parsed"] += run_result.items_discovered + grand_totals["inserted"] += run_result.items_inserted + grand_totals["updated"] += run_result.items_updated + grand_totals["unmodified"] += run_result.items_unmodified + grand_totals["failed"] += run_result.items_failed + + total_time = time.time() - overall_start + overall_rate = grand_totals["parsed"] / total_time if total_time > 0 else 0 + + print("\n" + "=" * 70) + print("RESTORE SUMMARY") + print("=" * 70) + print(f"Total Raw Files Scanned: {grand_totals['discovered']:,}") + print(f"Total Vehicles Parsed: {grand_totals['parsed']:,}") + if not dry_run: + print(f"Vehicles Inserted (New): {grand_totals['inserted']:,}") + print(f"Vehicles Updated: {grand_totals['updated']:,}") + print(f"Vehicles Unmodified: {grand_totals['unmodified']:,}") + print(f"Failed Records: {grand_totals['failed']:,}") + print(f"Total Duration: {total_time:.1f} seconds ({overall_rate:.0f} veh/sec)") + print("=" * 70 + "\n") + + +def main(): + parser = argparse.ArgumentParser( + description="Upsert and restore OEM inventory database from external raw compressed payload files (.json.gz)" + ) + parser.add_argument( + "--raw-dir", + type=Path, + default=Path(settings.RAW_STORAGE_DIR), + help=f"Directory containing compressed raw JSON payloads (default: {settings.RAW_STORAGE_DIR})", + ) + parser.add_argument( + "--db-url", + type=str, + default=settings.DATABASE_URL, + help=f"Target database URL (default: {settings.DATABASE_URL})", + ) + parser.add_argument( + "--brand", + type=str, + default="all", + help="Brand to restore (e.g. 'bmw', 'audi', 'mercedes-benz', 'porsche', or 'all'; comma-separated allowed)", + ) + parser.add_argument( + "--batch-size", + type=int, + default=500, + help="Transaction batch size (default: 500)", + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help="Limit number of vehicles to process per brand (default: unlimited)", + ) + parser.add_argument( + "--country", + type=str, + default=settings.DEFAULT_MARKET, + help=f"Market country code (default: {settings.DEFAULT_MARKET})", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Parse and validate files without making any database changes", + ) + + args = parser.parse_args() + + target_brands = None + if args.brand and args.brand.strip().lower() not in ("all", "*"): + target_brands = [b.strip().lower() for b in args.brand.split(",") if b.strip()] + + restore_database( + raw_dir=args.raw_dir, + db_url=args.db_url, + target_brands=target_brands, + batch_size=args.batch_size, + limit=args.limit, + dry_run=args.dry_run, + market_country=args.country, + ) + + +if __name__ == "__main__": + main() + diff --git a/src/oem_crawler/tools/view_stock.py b/src/oem_crawler/tools/view_stock.py new file mode 100644 index 0000000..ce4ef79 --- /dev/null +++ b/src/oem_crawler/tools/view_stock.py @@ -0,0 +1,128 @@ +""" +Vehicle Stock Inspector & Query Tool +Prints vehicle inventory, pricing summaries, and options from the database. +""" +from __future__ import annotations + +import argparse +from typing import Optional + +from sqlalchemy import create_engine, func, select +from sqlalchemy.orm import sessionmaker + +from oem_crawler.config import settings +from oem_crawler.db.models import CrawlRun, Vehicle, VehicleOption, VehiclePriceHistory +from oem_crawler.db.session import ensure_sqlite_dir + + +def inspect_stock(db_url: str, limit: int = 10, brand: Optional[str] = None, summary_only: bool = False): + ensure_sqlite_dir(db_url) + engine = create_engine(db_url) + Session = sessionmaker(bind=engine) + + with Session() as session: + # Total counts + total_vehicles = session.scalar(select(func.count(Vehicle.id))) or 0 + total_runs = session.scalar(select(func.count(CrawlRun.id))) or 0 + total_options = session.scalar(select(func.count(VehicleOption.id))) or 0 + + print("\n" + "=" * 80) + print(f"πŸ“Š DATABASE SUMMARY ({db_url})") + print(f" β€’ Total Crawl Runs: {total_runs}") + print(f" β€’ Total Vehicles in Stock: {total_vehicles:,}") + print(f" β€’ Total Normalized Options: {total_options:,}") + print("=" * 80) + + if total_vehicles == 0: + print("\n⚠️ No vehicles found in database yet. Run a crawl first:") + print(" uv run oem-crawler --brand porsche --limit 20\n") + return + + # Brand Breakdown + brand_stats = session.execute( + select( + Vehicle.brand, + func.count(Vehicle.id).label("total"), + func.avg(Vehicle.price_gross).label("avg_price"), + func.min(Vehicle.price_gross).label("min_price"), + func.max(Vehicle.price_gross).label("max_price"), + ) + .group_by(Vehicle.brand) + .order_by(func.count(Vehicle.id).desc()) + ).all() + + print("\n🏷️ INVENTORY BREAKDOWN BY BRAND:") + print(f"{'Brand':<18} {'Count':<10} {'Avg Price':<14} {'Min Price':<14} {'Max Price':<14}") + print("-" * 74) + for row in brand_stats: + b_name, b_count, b_avg, b_min, b_max = row + avg_s = f"{b_avg:,.2f} €" if b_avg else "N/A" + min_s = f"{b_min:,.2f} €" if b_min else "N/A" + max_s = f"{b_max:,.2f} €" if b_max else "N/A" + print(f"{b_name:<18} {b_count:<10} {avg_s:<14} {min_s:<14} {max_s:<14}") + print("-" * 74) + + if summary_only: + print("\n") + return + + # Query latest vehicles + query = select(Vehicle) + if brand: + query = query.where(func.lower(Vehicle.brand).like(f"%{brand.lower()}%")) + query = query.order_by(Vehicle.created_at.desc()).limit(limit) + + vehicles = session.scalars(query).all() + + filter_lbl = f" for '{brand.upper()}'" if brand else "" + print(f"\nπŸš— LATEST VEHICLES IN STOCK (Showing up to {limit}{filter_lbl}):\n") + for idx, v in enumerate(vehicles, 1): + opt_count = session.scalar( + select(func.count(VehicleOption.id)).where(VehicleOption.vehicle_id == v.id) + ) or 0 + + condition_badge = f"[{v.condition.upper()}]" + mileage_info = f"{v.mileage_km:,} km" if v.mileage_km > 0 else "0 km (New)" + first_reg = f" | First Reg: {v.first_registration_date}" if v.first_registration_date else "" + + print(f"[{idx}] {v.brand} - {v.series} ({v.model_name}) {condition_badge}") + print(f" β€’ OEM ID: {v.oem_vehicle_id} | Status: {v.status} | Mileage: {mileage_info}{first_reg}") + print(f" β€’ Powertrain: {v.fuel_type.upper()} ({v.power_kw or '?'} kW / {v.power_hp or '?'} HP) | Drivetrain: {v.drivetrain or 'N/A'}") + if v.battery_capacity: + print(f" β€’ EV Battery: {v.battery_capacity} {v.battery_capacity_unit or 'kwh'} | Range WLTP: {v.electric_range_wltp_km or '?'} km | Charging: {v.charging_power_ac_kw or '?'} kW AC / {v.charging_power_dc_kw or '?'} kW DC") + print(f" β€’ Price Gross: {v.price_gross:,.2f} {v.currency} | Monthly Rate: {v.monthly_rate or 0:,.2f} {v.currency}") + print(f" β€’ Color: {v.exterior_color or 'N/A'}") + loc_coords = f" [Lat: {v.latitude}, Lon: {v.longitude}]" if v.latitude and v.longitude else "" + print(f" β€’ Location: {v.dealer_name or 'Dealer'} in {v.location_city or 'N/A'} ({v.location_postal_code or 'N/A'}){loc_coords}") + print(f" β€’ Options: {opt_count} factory options recorded") + print(f" β€’ URL: {v.vehicle_url or 'N/A'}") + print("-" * 80) + + # Show Sample options for the first vehicle + if vehicles: + first = vehicles[0] + options = session.scalars( + select(VehicleOption).where(VehicleOption.vehicle_id == first.id).limit(6) + ).all() + if options: + print(f"\nπŸ“¦ Sample Options for '{first.model_name}' ({first.oem_vehicle_id}):") + for opt in options: + badge = "[Standard]" if opt.is_standard else "[Extra]" + price = f" (+{opt.price_gross:,.2f} EUR)" if opt.price_gross else "" + print(f" - {opt.option_code or '---'}: {opt.name} {badge}{price} (Category: {opt.category or 'General'})") + print("=" * 80 + "\n") + + +def main(): + parser = argparse.ArgumentParser(description="View vehicles and options stored in the database") + parser.add_argument("--db-url", type=str, default=settings.DATABASE_URL, help=f"Database connection URL (default: {settings.DATABASE_URL})") + parser.add_argument("--limit", type=int, default=10, help="Number of vehicles to display (default: 10)") + parser.add_argument("--brand", type=str, default=None, help="Filter by brand (e.g. BMW, PORSCHE, AUDI, MERCEDES)") + parser.add_argument("--summary", action="store_true", help="Only show high-level summary and brand table without vehicle list") + args = parser.parse_args() + + inspect_stock(db_url=args.db_url, limit=args.limit, brand=args.brand, summary_only=args.summary) + + +if __name__ == "__main__": + main() diff --git a/tests/test_audi_parser.py b/tests/test_audi_parser.py new file mode 100644 index 0000000..63ed0c3 --- /dev/null +++ b/tests/test_audi_parser.py @@ -0,0 +1,325 @@ +""" +Unit tests for the Audi Stock Car Service parser and normalization logic. +""" +from datetime import date +from decimal import Decimal +import pytest + +from oem_crawler.crawlers.base import CrawlerRegistry +from oem_crawler.crawlers.audi.parser import ( + map_audi_drivetrain, + map_audi_fuel_type, + map_audi_series, + map_audi_transmission, + parse_audi_vehicle, + parse_german_date, + parse_power, +) +from oem_crawler.db.schemas import ( + BatteryCapacityUnit, + DrivetrainType, + FuelType, + TransmissionType, + VehicleCondition, + VehicleStatus, +) + + +class TestAudiParser: + def test_registry_contains_audi(self): + assert "audi" in CrawlerRegistry.supported_brands() + crawler_cls = CrawlerRegistry.get("audi") + crawler = crawler_cls(market_country="DE") + assert crawler.brand_name == "AUDI" + assert crawler.get_market_code("new") == "de" + assert crawler.get_market_code("used") == "deuc" + + def test_map_fuel_type(self): + assert map_audi_fuel_type({"code": "E", "description": "Elektro"}) == FuelType.BEV + assert map_audi_fuel_type("Elektro") == FuelType.BEV + assert map_audi_fuel_type({"code": "H", "description": "Hybrid (Benzin/Elektro)"}) == FuelType.PHEV + assert map_audi_fuel_type({"code": "B", "description": "Benzin"}) == FuelType.PETROL + assert map_audi_fuel_type({"code": "D", "description": "Diesel"}) == FuelType.DIESEL + assert map_audi_fuel_type("Mild-Hybrid") == FuelType.MHEV + assert map_audi_fuel_type("g-tron Erdgas") == FuelType.CNG + + def test_map_transmission(self): + assert map_audi_transmission({"code": "s-tronic", "description": "S tronic"}) == TransmissionType.AUTOMATIC + assert map_audi_transmission("tiptronic") == TransmissionType.AUTOMATIC + assert map_audi_transmission("Automatik") == TransmissionType.AUTOMATIC + assert map_audi_transmission({"code": "manual", "description": "Handschaltung"}) == TransmissionType.MANUAL + assert map_audi_transmission("Schaltgetriebe") == TransmissionType.MANUAL + assert map_audi_transmission(None) is None + + def test_map_drivetrain(self): + assert map_audi_drivetrain({"code": "quattro", "description": "quattro"}) == DrivetrainType.AWD + assert map_audi_drivetrain("Allradantrieb") == DrivetrainType.AWD + assert map_audi_drivetrain({"code": "fwd", "description": "Frontantrieb"}) == DrivetrainType.FWD + assert map_audi_drivetrain("Heckantrieb") == DrivetrainType.RWD + assert map_audi_drivetrain(None) is None + + def test_map_series(self): + assert map_audi_series({"description": "A3"}) == "A3" + assert map_audi_series({"description": "Q4 e-tron"}) == "Q4 e-tron" + assert map_audi_series(None, "Audi A1 Sportback 25 TFSI") == "A1" + assert map_audi_series(None, "Audi e-tron GT quattro 350 kW") == "e-tron GT" + assert map_audi_series(None, "Audi RS6 Avant performance TFSI") == "RS6" + assert map_audi_series(None, "Audi Q8 50 TDI quattro") == "Q8" + + def test_parse_power(self): + kw, hp = parse_power("110 kW (150 PS)") + assert kw == 110 + assert hp == 150 + + kw2, hp2 = parse_power("70 kW") + assert kw2 == 70 + assert hp2 == 95 + + kw3, hp3 = parse_power(None) + assert kw3 is None + assert hp3 is None + + def test_parse_german_date(self): + assert parse_german_date("21.07.2015") == date(2015, 7, 21) + assert parse_german_date("2021-10-20") == date(2021, 10, 20) + assert parse_german_date(1319068800000) == date(2011, 10, 20) + assert parse_german_date(None) is None + + def test_parse_audi_new_ice_vehicle(self): + basic_item = { + "id": "REVVMjA2NjUxNzdBQlNXMzQyMDI1=", + "carId": "DEU20665177ABSW342025", + "decodedCarId": "DEU20665177ABSW342025", + "commissionNumber": "177ABSW342025", + "model": { + "code": "aacv", + "description": "Audi A1 Sportback 25 TFSI 70 kW (95 PS) Schaltgetriebe", + }, + "symbolicCarlineGroup": { + "description": "A1", + }, + "bodyType": { + "description": "Sportback", + }, + "driveType": { + "code": "drive.fwd", + "description": "Frontantrieb", + }, + "gearType": { + "code": "gear-type.manual", + "description": "Handschaltung", + }, + "fuel": { + "code": "B", + "description": "Benzin", + }, + "powerDisplay": "70 kW (95 PS)", + "modelYear": 2025, + "typedPrices": [ + {"amount": 356.0, "currencyCode": "EUR", "type": "financeRate"}, + {"amount": 21480.0, "currencyCode": "EUR", "type": "retail"}, + ], + "dealer": { + "id": "20665", + "name": "Autohaus Schnitzler GmbH & Co. KG", + "street": "Westring 20-22", + "zipCode": "40721", + "city": "Hilden", + "geoLocation": { + "lat": 51.17704, + "lon": 6.91641, + }, + }, + "used": {}, + "extColor": {"description": "Gletscherweiß Metallic", "code": "2Y2Y"}, + "intColor": {"description": "Schwarz"}, + "tilesPictures": ["https://mediaservice.audi.com/media/tile.jpg"], + "pictures": ["https://mediaservice.audi.com/media/full.jpg"], + } + + v = parse_audi_vehicle(basic_item=basic_item, market_country="DE", requested_stock_type="new") + + assert v.brand == "AUDI" + assert v.oem_vehicle_id == "DEU20665177ABSW342025" + assert v.series == "A1" + assert v.model_name == "Audi A1 Sportback 25 TFSI 70 kW (95 PS) Schaltgetriebe" + assert v.body_type == "Sportback" + assert v.condition == VehicleCondition.NEW + assert v.fuel_type == FuelType.PETROL + assert v.transmission == TransmissionType.MANUAL + assert v.drivetrain == DrivetrainType.FWD + assert v.power_kw == 70 + assert v.power_hp == 95 + assert v.model_year == 2025 + assert v.mileage_km == 0 + assert v.price_gross == Decimal("21480.0") + assert v.monthly_rate == Decimal("356.0") + assert v.dealer_name == "Autohaus Schnitzler GmbH & Co. KG" + assert v.dealer_id == "20665" + assert v.location_city == "Hilden" + assert v.location_postal_code == "40721" + assert v.latitude == Decimal("51.17704") + assert v.longitude == Decimal("6.91641") + assert v.exterior_color == "Gletscherweiß Metallic" + assert v.status == VehicleStatus.IN_STOCK + assert "REVVMjA2NjUxNzdBQlNXMzQyMDI1=" in v.vehicle_url + assert len(v.image_urls) == 2 + + def test_parse_audi_used_vehicle(self): + basic_item = { + "id": "REVVMjIyNjNHVzE4ODMw=", + "carId": "DEU22263GW18830", + "model": { + "description": "Audi A3 Sportback 1.4 TFSI attraction", + }, + "symbolicCarlineGroup": { + "description": "A3", + }, + "fuel": { + "code": "B", + "description": "Benzin", + }, + "gearType": { + "description": "Handschaltung", + }, + "driveType": { + "description": "Frontantrieb", + }, + "powerDisplay": "92 kW (125 PS)", + "typedPrices": [ + {"amount": 10890.0, "currencyCode": "EUR", "type": "retail"} + ], + "dealer": { + "name": "Autohaus Pietsch.BΓΌnde GmbH", + "city": "BΓΌnde", + "zipCode": "32257", + "geoLocation": {"lat": 52.2025, "lon": 8.57877}, + }, + "warrantyPlus": True, + "used": { + "initialRegistration": "21.07.2015", + "mileage": 108859.0, + "numPreviousOwners": 2, + "pictureUrls": ["https://vtpimages.audi.com/carimg2/used.jpg"], + }, + } + + v = parse_audi_vehicle(basic_item=basic_item, market_country="DE", requested_stock_type="used") + + assert v.brand == "AUDI" + assert v.series == "A3" + assert v.condition == VehicleCondition.CERTIFIED_PRE_OWNED # warrantyPlus is True + assert v.mileage_km == 108859 + assert v.first_registration_date == date(2015, 7, 21) + assert v.previous_owners_count == 2 + assert v.price_gross == Decimal("10890.0") + assert v.latitude == Decimal("52.2025") + assert v.longitude == Decimal("8.57877") + assert "gebrauchtwagenboerse" in v.vehicle_url + + def test_parse_audi_ev_with_detail(self): + basic_item = { + "id": "REVVNDcyNTIxNzdaWlpZNDMyMDI2=", + "carId": "DEU47252177ZZZY432026", + "model": { + "description": "Audi Q4 Sportback e-tron 150 kW", + }, + "symbolicCarlineGroup": { + "description": "Q4 e-tron", + }, + "fuel": { + "code": "E", + "description": "Elektro", + }, + "gearType": { + "description": "Automatik", + }, + "powerDisplay": "150 kW (204 PS)", + "typedPrices": [ + {"amount": 48900.0, "currencyCode": "EUR", "type": "retail"} + ], + "dealer": { + "name": "Audi Zentrum Hamburg", + "city": "Hamburg", + "zipCode": "22525", + "geoLocation": {"lat": 53.58, "lon": 9.92}, + }, + } + + detail_data = { + "vehicle": { + "detail": { + "techData": { + "fields": [ + {"name": "Netto-Batterie-KapazitΓ€t", "value": "59 kWh"}, + {"name": "Brutto-Batterie-KapazitΓ€t", "value": "63 kWh"}, + {"name": "Max. Ladeleistung AC / HPC", "value": "11 kW / 160 kW"}, + {"name": "Ladedauer DC 10-80% bei max. Ladeleistung", "value": "27 min"}, + {"name": "Elektrische Reichweite kombiniert (WLTP)", "value": "415 km"}, + ] + }, + "features": [ + { + "key": "5SL", + "prno": "MASL5SL", + "name": "Außenspiegel links, asphΓ€risch", + "groupName": "Außenausstattung", + "kind": "serie", + }, + { + "key": "KA2", + "prno": "MKA2KA2", + "name": "RΓΌckfahrkamera", + "groupName": "Assistenzsysteme", + "kind": "sonder", + }, + ], + } + } + } + + v = parse_audi_vehicle( + basic_item=basic_item, + detail_data=detail_data, + market_country="DE", + requested_stock_type="new", + ) + + assert v.brand == "AUDI" + assert v.series == "Q4 e-tron" + assert v.fuel_type == FuelType.BEV + assert v.battery_capacity == Decimal("59") + assert v.battery_capacity_net_kwh == Decimal("59") + assert v.battery_capacity_gross_kwh == Decimal("63") + assert v.battery_capacity_unit == BatteryCapacityUnit.KWH + assert v.charging_power_ac_kw == Decimal("11") + assert v.charging_power_dc_kw == Decimal("160") + assert v.charging_time_dc_minutes == 27 + assert v.electric_range_wltp_km == 415 + + # Check options + assert len(v.options) == 2 + assert v.options[0].option_code == "5SL" + assert v.options[0].is_standard is True + assert v.options[1].option_code == "KA2" + assert v.options[1].is_standard is False + assert v.options[1].category == "Assistenzsysteme" + + def test_fetch_search_page_pagination_params(self): + from unittest.mock import MagicMock + from oem_crawler.crawlers.audi.crawler import AudiStockCrawler + + crawler = AudiStockCrawler(market_country="DE") + mock_client = MagicMock() + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"totalCount": 100, "vehicleBasic": []} + mock_client.get.return_value = mock_resp + + crawler.fetch_search_page(mock_client, market_code="de", page=2, size=50) + + mock_client.get.assert_called_once() + call_kwargs = mock_client.get.call_args.kwargs + assert call_kwargs["params"]["from"] == 100 + assert call_kwargs["params"]["size"] == 50 + diff --git a/tests/test_bmw_parser.py b/tests/test_bmw_parser.py new file mode 100644 index 0000000..792744e --- /dev/null +++ b/tests/test_bmw_parser.py @@ -0,0 +1,74 @@ +""" +Unit tests for BMW Parser and Registry. +""" +from decimal import Decimal +import unittest + +from oem_crawler.crawlers import CrawlerRegistry +from oem_crawler.crawlers.bmw.crawler import BMWStockCrawler +from oem_crawler.crawlers.bmw.parser import parse_bmw_vehicle + + +class TestBMWParserAndRegistry(unittest.TestCase): + + def test_registry_contains_bmw(self): + supported = CrawlerRegistry.supported_brands() + self.assertIn("bmw", supported) + cls = CrawlerRegistry.get("bmw") + self.assertEqual(cls, BMWStockCrawler) + + def test_bmw_vehicle_parsing(self): + raw_bmw = { + "documentId": "2454561-61134216", + "vin": "WBA11111111111111", + "vehicleSpecification": { + "modelAndOption": { + "brand": "BMW", + "baseFuelType": "GASOLINE", + "is48Volt": True, + "transmission": "AUTOMATIC", + "marketingDriveType": "SDRIVE", + "model": {"modelName": "120"}, + "modelRange": {"description": {"de_DE": "BMW 1er"}}, + "color": {"name": {"de_DE": "Schwarz Uni"}, "code": "668"}, + "equipments": { + "S05DM": { + "name": {"de_DE": "Parking Assistant"}, + "marketingText": {"category": {"de_DE": "Fahrerassistenz"}}, + "standard": True + } + } + }, + "technicalAndEmission": { + "technicalData": { + "power": {"value": 115}, + "powerInHp": {"value": 156} + } + } + }, + "pricing": { + "price": {"value": 30960.00}, + "monthlyInstallment": {"value": 199.00} + }, + "ordering": { + "retailData": { + "locationOutletName": "BMW Autohaus Test", + "locationOutletAddress": {"town": "MΓΌnchen", "postalCode": "80809", "countryCode": "DE"} + } + } + } + + parsed = parse_bmw_vehicle(raw_bmw, market_country="DE") + self.assertEqual(parsed.brand, "BMW") + self.assertEqual(parsed.oem_vehicle_id, "2454561-61134216") + self.assertEqual(parsed.series, "BMW 1er") + self.assertEqual(parsed.model_name, "120") + self.assertEqual(parsed.fuel_type.value, "mhev") + self.assertEqual(parsed.drivetrain.value, "fwd") + self.assertEqual(parsed.power_kw, 115) + self.assertEqual(parsed.power_hp, 156) + self.assertEqual(parsed.price_gross, Decimal("30960.00")) + self.assertEqual(parsed.monthly_rate, Decimal("199.00")) + self.assertEqual(len(parsed.options), 1) + self.assertEqual(parsed.options[0].option_code, "S05DM") + self.assertTrue(parsed.options[0].is_standard) diff --git a/tests/test_mercedes_parser.py b/tests/test_mercedes_parser.py new file mode 100644 index 0000000..d4f7167 --- /dev/null +++ b/tests/test_mercedes_parser.py @@ -0,0 +1,338 @@ +""" +Unit tests for Mercedes-Benz stock parser. +""" +from decimal import Decimal +import unittest + +from oem_crawler.db.schemas import ( + BatteryCapacityUnit, + DrivetrainType, + FuelType, + TransmissionType, + VehicleCondition, + VehicleStatus, +) +from oem_crawler.crawlers.mercedes.parser import ( + extract_city_postal_code, + map_mercedes_drivetrain, + map_mercedes_fuel_type, + map_mercedes_transmission, + map_mercedes_used_fuel_type, + map_mercedes_used_series, + parse_mercedes_used_vehicle, + parse_mercedes_vehicle, +) + + +class TestMercedesParser(unittest.TestCase): + + def test_extract_city_postal_code(self): + city, zip_code = extract_city_postal_code("Mercedesstraße 120, 70372 Stuttgart") + self.assertEqual(city, "Stuttgart") + self.assertEqual(zip_code, "70372") + + city2, zip_code2 = extract_city_postal_code("Am Seestern 1, 40547 DΓΌsseldorf") + self.assertEqual(city2, "DΓΌsseldorf") + self.assertEqual(zip_code2, "40547") + + def test_map_fuel_type(self): + self.assertEqual(map_mercedes_fuel_type("ELECTRIC", []), FuelType.BEV) + self.assertEqual(map_mercedes_fuel_type("PETROL", [{"id": "90W-BI_MILD"}]), FuelType.MHEV) + self.assertEqual(map_mercedes_fuel_type("DIESEL", []), FuelType.DIESEL) + self.assertEqual(map_mercedes_fuel_type("PLUGIN", []), FuelType.PHEV) + + def test_map_transmission_and_drivetrain(self): + self.assertEqual(map_mercedes_transmission("AUTOMATIC"), TransmissionType.AUTOMATIC) + self.assertEqual(map_mercedes_transmission("MANUAL"), TransmissionType.MANUAL) + self.assertEqual(map_mercedes_drivetrain("C 400 4MATIC"), DrivetrainType.AWD) + self.assertEqual(map_mercedes_drivetrain("A 180", "W177"), DrivetrainType.FWD) + self.assertEqual(map_mercedes_drivetrain("C 200", "W206"), DrivetrainType.RWD) + + def test_parse_mercedes_vehicle(self): + raw_search = { + "identification": { + "code": "MB8SXSK01", + "commissionNumber": "0555673413", + "vin": None, + }, + "vehicleModel": { + "name": "A 180", + "motorization": "A 180", + "vehicleClass": {"formattedValue": "A-Klasse"}, + "bodyType": {"formattedValue": "Kompaktwagen"}, + "modelYearCode": "806", + "typeClass": "W177", + }, + "technicalInformation": { + "engine": { + "fuelType": {"value": "PETROL"}, + "power": { + "combinedKw": {"value": 110}, + "combinedHp": {"value": 150}, + }, + }, + "transmission": {"value": "AUTOMATIC"}, + }, + "dealer": { + "nameLocalLanguage": "Mercedes-Benz AG", + "outletId": "GS00113259", + "addressLocalLanguage": "Mercedesstraße 120, 70372 Stuttgart", + }, + "emissionAndConsumption": { + "attributes": [ + {"id": "90W-BI_MILD", "value": "5.9", "unit": "l/100km"}, + {"id": "85W", "value": "133.0", "unit": "g/km"}, + ] + }, + "stock": {"stockType": "AVAILABLE"}, + "characteristics": { + "highlights": [{"label": "Progressive"}, {"label": "Spiegel-Paket"}] + }, + "images": { + "default": "https://media.oneweb.mercedes-benz.com/images/default.jpg" + }, + } + + raw_price = { + "currency": "EUR", + "purchasePrice": {"amount": "38167.76"}, + "priceBreakDown": [{"type": "BASE_PRICE", "amount": "37453.76"}], + "financeLeasingInfo": { + "vehicle": { + "prices": [ + {"id": "grossListPrice", "rawValue": "49938.35"}, + {"id": "baseListPrice", "rawValue": "39513.95"}, + ] + } + }, + "taxProfile": "GROSS", + } + + raw_vpc = { + "color": { + "id": "LU-787", + "text": {"de": "mountaingrau"}, + "group": {"de": "Grau"}, + }, + "upholstery": { + "id": "AU-311", + "group": {"de": "Schwarz"}, + "polish": {"de": "Ledernachbildung ARTICO"}, + }, + "components": { + "SA-R31": { + "code": "R31", + "codeType": "SA", + "name": {"de": "18\" LeichtmetallrΓ€der"}, + "standard": True, + } + }, + } + + veh = parse_mercedes_vehicle(raw_search, raw_price=raw_price, raw_vpc=raw_vpc) + + self.assertEqual(veh.brand, "Mercedes-Benz") + self.assertEqual(veh.oem_vehicle_id, "MB8SXSK01") + self.assertEqual(veh.series, "A-Klasse") + self.assertEqual(veh.model_name, "A 180") + self.assertEqual(veh.trim_line, "Progressive") + self.assertEqual(veh.body_type, "Kompaktwagen") + self.assertEqual(veh.condition, VehicleCondition.NEW) + self.assertEqual(veh.model_year, 2026) + self.assertEqual(veh.fuel_type, FuelType.MHEV) + self.assertEqual(veh.transmission, TransmissionType.AUTOMATIC) + self.assertEqual(veh.drivetrain, DrivetrainType.FWD) + self.assertEqual(veh.power_kw, 110) + self.assertEqual(veh.power_hp, 150) + self.assertEqual(veh.price_gross, Decimal("38167.76")) + self.assertEqual(veh.msrp_gross, Decimal("49938.35")) + self.assertEqual(veh.discount_percentage, Decimal("23.57")) + self.assertEqual(veh.location_city, "Stuttgart") + self.assertEqual(veh.location_postal_code, "70372") + self.assertEqual(veh.exterior_color, "mountaingrau") + self.assertEqual(veh.exterior_color_code, "LU-787") + self.assertEqual(veh.interior_color, "Schwarz") + self.assertEqual(len(veh.options), 1) + self.assertEqual(veh.options[0].option_code, "R31") + + def test_parse_mercedes_electric_vehicle(self): + raw_search = { + "identification": {"code": "MB_EV_001"}, + "vehicleModel": { + "name": "EQE 350+ 4MATIC", + "vehicleClass": {"formattedValue": "EQE"}, + "bodyType": {"formattedValue": "Limousine"}, + }, + "technicalInformation": { + "engine": { + "fuelType": {"value": "ELECTRIC"}, + "power": {"combinedKw": {"value": 215}, "combinedHp": {"value": 292}}, + }, + "transmission": {"value": "AUTOMATIC"}, + }, + "emissionAndConsumption": { + "attributes": [ + {"id": "237W", "value": "16.5", "unit": "kWh/100km"}, + {"id": "193W", "value": "620", "unit": "km"}, + {"id": "85W", "value": "0", "unit": "g/km"}, + ] + }, + } + + raw_vpc = { + "technicalInformation": { + "engine": { + "batteryCapacity": {"value": 141, "unit": "Ah"}, + "chargingCapacityAC": {"value": 11, "unit": "kW"}, + "chargingCapacityDC": {"value": 170, "unit": "kW"}, + }, + "chargeTime": { + "chargeTimeDCHigh": {"value": 22.0, "unit": "min"} + }, + } + } + + raw_price = { + "purchasePrice": {"amount": "74500.00"}, + } + + veh = parse_mercedes_vehicle(raw_search, raw_price=raw_price, raw_vpc=raw_vpc) + + self.assertEqual(veh.fuel_type, FuelType.BEV) + self.assertEqual(veh.drivetrain, DrivetrainType.AWD) + self.assertEqual(veh.electric_range_wltp_km, 620) + self.assertEqual(veh.consumption_combined, Decimal("16.5")) + self.assertEqual(veh.battery_capacity, Decimal("141")) + self.assertEqual(veh.battery_capacity_unit, BatteryCapacityUnit.AH) + self.assertEqual(veh.charging_power_ac_kw, Decimal("11")) + self.assertEqual(veh.charging_power_dc_kw, Decimal("170")) + self.assertEqual(veh.charging_time_dc_minutes, 22) + + def test_map_mercedes_used_series(self): + self.assertEqual(map_mercedes_used_series("V 300 d Avantgarde lang"), "V-Klasse") + self.assertEqual(map_mercedes_used_series("EQV 300 lang"), "EQV") + self.assertEqual(map_mercedes_used_series("EQE 350 4MATIC"), "EQE") + self.assertEqual(map_mercedes_used_series("C 200 T-Modell AMG Line"), "C-Klasse") + self.assertEqual(map_mercedes_used_series("A 180 Progressive"), "A-Klasse") + + def test_map_mercedes_used_fuel_type(self): + self.assertEqual(map_mercedes_used_fuel_type("Elektro"), FuelType.BEV) + self.assertEqual(map_mercedes_used_fuel_type("Diesel"), FuelType.DIESEL) + self.assertEqual(map_mercedes_used_fuel_type("Benzin"), FuelType.PETROL) + self.assertEqual(map_mercedes_used_fuel_type("Plug-in-Hybrid Benzin"), FuelType.PHEV) + + def test_parse_mercedes_used_vehicle_ice(self): + card_html = """ +
+
Mercedes-Benz
V 300 d Avantgarde lang
+
Fahrzeugnummer 55229GK030
+
95.490 €
+
699,00 €
+
    +

  • Van/Kleinbus

    • +

  • 25.07.2025

    • +

  • 174 kW (237 PS)

    • +

  • 10.018 km

    • +

  • Diesel

    • +
  • Mercedes-Benz AG71034 BΓΆblingen
    • +
+
+ """ + detail_html = """ + + +
Erstzulassung
25.07.2025
+
Modeljahr
2025
+
Laufleistung
10.018 km
+
Hubraum
1.950 ccm
+
Getriebe
Automatikgetriebe
+
Garantie
Mercedes-Benz Junge Sterne
+
Außenfarbe (grau)
Alpingrau
+
Polster (Leder)
Leder Lugano schwarz
+ +
Leichtmetallfelgen 19 Zoll | R1W
+ + + """ + + veh = parse_mercedes_used_vehicle(card=card_html, detail_html=detail_html) + + self.assertEqual(veh.oem_vehicle_id, "19170569") + self.assertEqual(veh.raw_payload.get("commission_number"), "55229GK030") + self.assertEqual(veh.series, "V-Klasse") + self.assertEqual(veh.model_name, "V 300 d Avantgarde lang") + self.assertEqual(veh.trim_line, "Avantgarde") + self.assertEqual(veh.body_type, "Van/Kleinbus") + self.assertEqual(veh.condition, VehicleCondition.CERTIFIED_PRE_OWNED) + self.assertEqual(veh.model_year, 2025) + self.assertEqual(veh.fuel_type, FuelType.DIESEL) + self.assertEqual(veh.transmission, TransmissionType.AUTOMATIC) + self.assertEqual(veh.power_kw, 174) + self.assertEqual(veh.power_hp, 237) + self.assertEqual(veh.engine_displacement_cc, 1950) + self.assertEqual(veh.mileage_km, 10018) + self.assertEqual(veh.price_gross, Decimal("95490")) + self.assertEqual(veh.monthly_rate, Decimal("699.00")) + self.assertEqual(veh.exterior_color, "Alpingrau") + self.assertEqual(veh.interior_material, "Leder Lugano schwarz") + self.assertEqual(veh.dealer_name, "Mercedes-Benz AG") + self.assertEqual(veh.location_postal_code, "71034") + self.assertEqual(veh.location_city, "BΓΆblingen") + self.assertEqual(veh.latitude, Decimal("48.682361000000000")) + self.assertEqual(veh.longitude, Decimal("8.982236000000000")) + self.assertEqual(len(veh.options), 1) + self.assertEqual(veh.options[0].option_code, "R1W") + + def test_parse_mercedes_used_vehicle_ev(self): + card_html = """ +
+
Mercedes-Benz
EQV 300 lang
+
Fahrzeugnummer 56F27180
+
33.444 €
+
    +

  • 150 kW (204 PS)

    • +

  • 121.962 km

    • +

  • Elektro

    • +
  • KTW31137 Hildesheim
    • +
+
+ """ + detail_html = """ + + +
Erstzulassung
23.05.2022
+
Modeljahr
2024
+
Kraftstoffart
Elektro
+
Außenfarbe (blau)
Cavansitblau
+
Elektrische Reichweite
348 km
+
Energie
90 kWh
+

Die Ladeleistung von 110 kW ermΓΆglicht schnelles Laden an DC-Stationen. An Wallboxen mit bis zu 11 kW AC laden.

+ + + + """ + + veh = parse_mercedes_used_vehicle(card=card_html, detail_html=detail_html) + + self.assertEqual(veh.oem_vehicle_id, "19169425") + self.assertEqual(veh.raw_payload.get("commission_number"), "56F27180") + self.assertEqual(veh.series, "EQV") + self.assertEqual(veh.model_name, "EQV 300 lang") + self.assertEqual(veh.fuel_type, FuelType.BEV) + self.assertEqual(veh.battery_capacity, Decimal("90")) + self.assertEqual(veh.battery_capacity_unit, BatteryCapacityUnit.KWH) + self.assertEqual(veh.battery_capacity_net_kwh, Decimal("90")) + self.assertEqual(veh.electric_range_wltp_km, 348) + self.assertEqual(veh.charging_power_ac_kw, Decimal("11")) + self.assertEqual(veh.charging_power_dc_kw, Decimal("110")) + self.assertEqual(veh.latitude, Decimal("52.167469000000000")) + self.assertEqual(veh.longitude, Decimal("9.920900000000000")) + + +if __name__ == "__main__": + unittest.main() + diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py new file mode 100644 index 0000000..e66d28e --- /dev/null +++ b/tests/test_pipeline.py @@ -0,0 +1,145 @@ +""" +Unit tests for the StockIngestionPipeline. +""" +from decimal import Decimal +import uuid +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from oem_crawler.db.models import Base, Vehicle, VehicleOption, VehiclePriceHistory, CrawlRun +from oem_crawler.db.pipeline import StockIngestionPipeline +from oem_crawler.db.schemas import ( + OptionSchema, + VehicleIngestSchema, + VehicleStatus, + compute_content_hash, +) + + +@pytest.fixture +def in_memory_pipeline(): + engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all(engine) + test_session_factory = sessionmaker(bind=engine, autoflush=False, autocommit=False) + pipeline = StockIngestionPipeline(session_factory=test_session_factory) + return pipeline, test_session_factory + + +def test_pipeline_insert_update_and_idempotency(in_memory_pipeline): + pipeline, session_factory = in_memory_pipeline + + test_vehicle = VehicleIngestSchema( + oem_vehicle_id="TEST-MB-EQE-001", + brand="MERCEDES-BENZ", + model_name="EQE 350 4MATIC", + series="EQE", + stock_type="used", + status=VehicleStatus.IN_STOCK, + price_gross=Decimal("65900.00"), + price_net=Decimal("55378.15"), + currency="EUR", + mileage_km=15200, + power_kw=215, + power_hp=292, + battery_capacity=Decimal("90.56"), + battery_capacity_unit="kwh", + charging_power_ac_kw=Decimal("11.00"), + charging_power_dc_kw=Decimal("170.00"), + electric_range_wltp_km=620, + dealer_name="Mercedes-Benz Niederlassung Berlin", + location_city="Berlin", + location_postal_code="10115", + latitude=Decimal("52.5200"), + longitude=Decimal("13.4050"), + options=[ + OptionSchema(option_code="P20", name="Fahrassistenz-Paket Plus", category="Packages", is_standard=False), + OptionSchema(option_code="413", name="Panorama-Schiebedach", category="Roof", is_standard=False), + ], + raw_payload={"id": "TEST-MB-EQE-001"}, + ) + + # 1. First Run -> Insert + def stream_once(): + yield test_vehicle + + run1 = pipeline.run_crawler_pipeline( + brand="MERCEDES-BENZ", + crawler_name="test-crawler", + vehicle_stream=stream_once(), + market_country="DE", + ) + + assert run1.items_discovered == 1 + assert run1.items_inserted == 1 + assert run1.items_updated == 0 + assert run1.items_unmodified == 0 + assert run1.items_failed == 0 + + with session_factory() as s: + veh = s.query(Vehicle).filter_by(oem_vehicle_id="TEST-MB-EQE-001").first() + assert veh is not None + assert veh.price_gross == Decimal("65900.00") + assert veh.battery_capacity == Decimal("90.56") + assert veh.battery_capacity_unit == "kwh" + assert veh.charging_power_ac_kw == Decimal("11.00") + assert veh.charging_power_dc_kw == Decimal("170.00") + assert veh.electric_range_wltp_km == 620 + assert veh.location_city == "Berlin" + assert veh.latitude == Decimal("52.5200") + assert veh.longitude == Decimal("13.4050") + + # Check options + options = s.query(VehicleOption).filter_by(vehicle_id=veh.id).all() + assert len(options) == 2 + assert {opt.option_code for opt in options} == {"P20", "413"} + + # Check price history + hist = s.query(VehiclePriceHistory).filter_by(vehicle_id=veh.id).all() + assert len(hist) == 1 + assert hist[0].price_gross == Decimal("65900.00") + + # 2. Second Run -> Unmodified + run2 = pipeline.run_crawler_pipeline( + brand="MERCEDES-BENZ", + crawler_name="test-crawler", + vehicle_stream=stream_once(), + market_country="DE", + ) + assert run2.items_discovered == 1 + assert run2.items_inserted == 0 + assert run2.items_updated == 0 + assert run2.items_unmodified == 1 + assert run2.items_failed == 0 + + with session_factory() as s: + veh = s.query(Vehicle).filter_by(oem_vehicle_id="TEST-MB-EQE-001").first() + hist = s.query(VehiclePriceHistory).filter_by(vehicle_id=veh.id).all() + assert len(hist) == 1 # No duplicate price history + + # 3. Third Run -> Price Drop Update + test_vehicle_updated = test_vehicle.model_copy() + test_vehicle_updated.price_gross = Decimal("63900.00") + + def stream_updated(): + yield test_vehicle_updated + + run3 = pipeline.run_crawler_pipeline( + brand="MERCEDES-BENZ", + crawler_name="test-crawler", + vehicle_stream=stream_updated(), + market_country="DE", + ) + assert run3.items_discovered == 1 + assert run3.items_inserted == 0 + assert run3.items_updated == 1 + assert run3.items_unmodified == 0 + assert run3.items_failed == 0 + + with session_factory() as s: + veh = s.query(Vehicle).filter_by(oem_vehicle_id="TEST-MB-EQE-001").first() + assert veh.price_gross == Decimal("63900.00") + hist = s.query(VehiclePriceHistory).filter_by(vehicle_id=veh.id).order_by(VehiclePriceHistory.recorded_at).all() + assert len(hist) == 2 + assert hist[0].price_gross == Decimal("65900.00") + assert hist[1].price_gross == Decimal("63900.00") diff --git a/tests/test_porsche_parser.py b/tests/test_porsche_parser.py new file mode 100644 index 0000000..05df9d7 --- /dev/null +++ b/tests/test_porsche_parser.py @@ -0,0 +1,285 @@ +""" +Unit tests for the Porsche Finder parser and normalization logic. +""" +from decimal import Decimal +import pytest + +from oem_crawler.crawlers.base import CrawlerRegistry +from oem_crawler.crawlers.porsche.parser import ( + map_porsche_drivetrain, + map_porsche_fuel_type, + map_porsche_series, + map_porsche_transmission, + parse_porsche_vehicle, +) +from oem_crawler.db.schemas import ( + BatteryCapacityUnit, + DrivetrainType, + FuelType, + TransmissionType, + VehicleCondition, + VehicleStatus, +) + + +class TestPorscheParser: + def test_registry_contains_porsche(self): + assert "porsche" in CrawlerRegistry.supported_brands() + crawler_cls = CrawlerRegistry.get("porsche") + crawler = crawler_cls(market_country="DE") + assert crawler.brand_name == "PORSCHE" + + def test_map_fuel_type(self): + assert map_porsche_fuel_type("ELECTRIC") == FuelType.BEV + assert map_porsche_fuel_type("Elektro") == FuelType.BEV + assert map_porsche_fuel_type("PLUG_IN_HYBRID") == FuelType.PHEV + assert map_porsche_fuel_type("MILD_HYBRID") == FuelType.MHEV + assert map_porsche_fuel_type("DIESEL") == FuelType.DIESEL + assert map_porsche_fuel_type("PETROL") == FuelType.PETROL + assert map_porsche_fuel_type("Benzin") == FuelType.PETROL + + def test_map_transmission(self): + assert map_porsche_transmission("PDK (Automatik)") == TransmissionType.AUTOMATIC + assert map_porsche_transmission("Automatik") == TransmissionType.AUTOMATIC + assert map_porsche_transmission("Manuell") == TransmissionType.MANUAL + assert map_porsche_transmission(None) is None + + def test_map_drivetrain(self): + assert map_porsche_drivetrain("ALL_WHEEL_DRIVE") == DrivetrainType.AWD + assert map_porsche_drivetrain("Allradantrieb") == DrivetrainType.AWD + assert map_porsche_drivetrain("REAR_WHEEL_DRIVE") == DrivetrainType.RWD + assert map_porsche_drivetrain("Heckantrieb") == DrivetrainType.RWD + + def test_map_series(self): + assert map_porsche_series("Taycan", None) == "Taycan" + assert map_porsche_series("911", None) == "911" + assert map_porsche_series("Macan", None) == "Macan" + assert map_porsche_series("Cayenne", None) == "Cayenne" + assert map_porsche_series("Panamera", None) == "Panamera" + assert map_porsche_series("718", None) == "718" + assert map_porsche_series(None, "Porsche 911 Carrera 4S") == "911" + assert map_porsche_series(None, "Porsche Taycan 4S") == "Taycan" + + def test_parse_porsche_new_ice_vehicle(self): + search_item = { + "id": "RNR7NZ", + "listingUrlSlug": "porsche-911-carrera-4s-cabriolet-neu-RNR7NZ", + "meta": { + "title": "911 Carrera 4S Cabriolet", + "condition": "new", + "priceValue": 220231.42, + "priceCurrency": "EUR", + "numberOfPreviousOwners": 0, + "mileage": {"value": 0, "unitCode": "KMT"}, + "model": "992 II", + "interiorColor": "Lederausstattung Schwarz", + "color": "Schiefergrau Neo", + "transmission": "PDK (Automatik)", + "modelYear": 2026, + "engineType": "PETROL", + "modelCategory": "911", + "bodyType": "Cabriolet", + "drivetrain": "ALL_WHEEL_DRIVE", + "enginePower": 353, + "engineDisplacement": 2981, + "co2Emissions": 245, + "fuelConsumption": 10.8, + "seller": { + "sellerId": "8066", + "porschePartnerNumber": "1110050", + "name": "Porsche Zentrum Berlin-Potsdam", + "addressComponents": { + "postalCode": "14532", + "city": "Kleinmachnow", + "street": "Albert-Einstein-Ring 49", + }, + }, + }, + "dataLayerListingMeta": { + "car": { + "isPorscheApproved": False, + "exteriorColorCode": "2T", + "interiorColorCode": "VF", + } + }, + "gallery": { + "imageIds": ["dcf0d51e-230b-444f-bcb2-d37c8f631d98"] + }, + "description": { + "title": "Porsche 911 Carrera 4S Cabriolet", + "periodicPayments": { + "paymentLabel": "2808 per month", + }, + }, + } + + detail_data = { + "equipment": { + "specialEquipment": [ + { + "name": "Exterieur", + "items": [ + {"code": "KQ2", "name": "β€žPORSCHEβ€œ Schriftzug lackiert in Schwarz", "description": "Hochglanz"}, + {"code": "3FG", "name": "Panoramadach", "description": "Glas"}, + ], + } + ], + "importantEquipmentOptions": [ + {"value": "bose-sound-system", "label": "BOSE Surround Sound-System"} + ], + }, + "summary": { + "characteristics": [ + {"title": "Motor", "value": "Benzin"}, + {"title": "Leistung Verbrennungsmotor", "value": "353 kW / 480 PS"}, + ] + }, + "info": { + "stockNumber": "D27829", + "commissionNumber": "D27829", + "vin": "$undefined", + }, + } + + veh = parse_porsche_vehicle(search_item, detail_data, market_country="DE", requested_stock_type="new") + + assert veh.brand == "PORSCHE" + assert veh.oem_vehicle_id == "RNR7NZ" + assert veh.vin is None + assert veh.series == "911" + assert veh.model_name == "911 Carrera 4S Cabriolet" + assert veh.trim_line == "992 II" + assert veh.body_type == "Cabriolet" + assert veh.condition == VehicleCondition.NEW + assert veh.fuel_type == FuelType.PETROL + assert veh.transmission == TransmissionType.AUTOMATIC + assert veh.drivetrain == DrivetrainType.AWD + assert veh.power_kw == 353 + assert veh.power_hp == 480 + assert veh.engine_displacement_cc == 2981 + assert veh.price_gross == Decimal("220231.42") + assert veh.monthly_rate == Decimal("2808") + assert veh.dealer_name == "Porsche Zentrum Berlin-Potsdam" + assert veh.location_city == "Kleinmachnow" + assert veh.location_postal_code == "14532" + assert veh.location_country == "DE" + assert len(veh.options) == 3 + option_codes = {opt.option_code for opt in veh.options} + assert "KQ2" in option_codes + assert "3FG" in option_codes + assert "bose-sound-system" in option_codes + assert veh.image_urls == ["https://images.finder.porsche.com/dcf0d51e-230b-444f-bcb2-d37c8f631d98/960.webp"] + + def test_parse_porsche_used_certified_vehicle(self): + search_item = { + "id": "P00XDK", + "listingUrlSlug": "porsche-macan-gebraucht-P00XDK", + "meta": { + "title": "Macan", + "condition": "preowned", + "priceValue": 56900, + "priceCurrency": "EUR", + "numberOfPreviousOwners": 1, + "mileage": {"value": 35050, "unitCode": "KMT"}, + "model": "H1 III", + "color": "Tiefschwarzmetallic", + "firstRegistration": "2022-01-04", + "modelYear": 2022, + "engineType": "PETROL", + "modelCategory": "Macan", + "bodyType": "SUV", + "drivetrain": "ALL_WHEEL_DRIVE", + "enginePower": 195, + "seller": { + "sellerId": "15713", + "name": "Porsche Zentrum Essen", + "addressComponents": {"postalCode": "45143", "city": "Essen"}, + }, + }, + "dataLayerListingMeta": { + "car": { + "isPorscheApproved": True, + } + }, + "description": { + "title": "Porsche Macan", + "periodicPayments": { + "paymentLabel": "765.26 per month", + }, + }, + } + + veh = parse_porsche_vehicle(search_item, detail_data=None, market_country="DE", requested_stock_type="used") + + assert veh.brand == "PORSCHE" + assert veh.oem_vehicle_id == "P00XDK" + assert veh.series == "Macan" + assert veh.condition == VehicleCondition.CERTIFIED_PRE_OWNED + assert veh.mileage_km == 35050 + assert veh.price_gross == Decimal("56900") + assert veh.monthly_rate == Decimal("765.26") + assert veh.dealer_name == "Porsche Zentrum Essen" + assert veh.location_city == "Essen" + assert veh.location_postal_code == "45143" + + def test_parse_porsche_electric_option_c(self): + search_item = { + "id": "EK6EW6", + "listingUrlSlug": "porsche-taycan-sport-turismo-black-edition-neu-EK6EW6", + "meta": { + "title": "Taycan Sport Turismo Black Edition", + "condition": "new", + "priceValue": 131063, + "priceCurrency": "EUR", + "mileage": {"value": 0, "unitCode": "KMT"}, + "model": "J1 II", + "color": "Eisgraumetallic", + "engineType": "ELECTRIC", + "modelCategory": "Taycan", + "bodyType": "Sport Turismo", + "drivetrain": "REAR_WHEEL_DRIVE", + "enginePower": 320, + "seller": { + "name": "Porsche Zentrum Berlin-Adlershof", + "addressComponents": {"postalCode": "12487", "city": "Berlin"}, + }, + }, + "dataLayerListingMeta": { + "car": {"isPorscheApproved": False} + }, + } + + detail_data = { + "equipment": { + "specialEquipment": [ + { + "name": "Motor, Getriebe und Fahrwerk", + "items": [ + {"code": "KB3", "name": "On-Board AC-Lader mit 22 kW", "description": "Schnelles Laden"}, + ], + } + ] + }, + "summary": { + "characteristics": [ + {"title": "Motor", "value": "Elektro"}, + {"title": "Reichweite kombiniert (WLTP)", "value": "636 km"}, + {"title": "Brutto-Energieinhalt Batterie", "value": "105,0 kWh"}, + {"title": "Ladezeit fΓΌr Gleichstrom (DC) mit maximaler Ladeleistung (10 auf bis zu 80%)", "value": "18,00 min"}, + {"title": "Antrieb", "value": "Heckantrieb"}, + ] + }, + } + + veh = parse_porsche_vehicle(search_item, detail_data, market_country="DE", requested_stock_type="new") + + assert veh.brand == "PORSCHE" + assert veh.series == "Taycan" + assert veh.fuel_type == FuelType.BEV + assert veh.drivetrain == DrivetrainType.RWD + assert veh.battery_capacity == Decimal("105.0") + assert veh.battery_capacity_unit == BatteryCapacityUnit.KWH + assert veh.electric_range_wltp_km == 636 + assert veh.charging_time_dc_minutes == 18 + assert veh.charging_power_ac_kw == Decimal("22.00") + assert veh.charging_power_dc_kw == Decimal("320.00") diff --git a/tests/test_raw_storage.py b/tests/test_raw_storage.py new file mode 100644 index 0000000..4f1ece8 --- /dev/null +++ b/tests/test_raw_storage.py @@ -0,0 +1,206 @@ +""" +Tests for RawStorageManager, external compressed storage, and raw extraction toggle. +""" +from __future__ import annotations + +import gzip +import json +import shutil +from datetime import datetime, timezone +from decimal import Decimal +from pathlib import Path + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from oem_crawler.config import settings +from oem_crawler.crawlers.audi.parser import parse_audi_vehicle +from oem_crawler.crawlers.bmw.parser import parse_bmw_vehicle +from oem_crawler.db.models import Base, Vehicle, VehicleOption +from oem_crawler.db.pipeline import StockIngestionPipeline +from oem_crawler.db.schemas import ( + OptionSchema, + VehicleCondition, + VehicleIngestSchema, + VehicleStatus, +) +from oem_crawler.storage import RawStorageManager + + +@pytest.fixture +def temp_raw_dir(tmp_path): + orig_dir = settings.RAW_STORAGE_DIR + orig_enabled = settings.ENABLE_RAW_EXTRACTION + raw_dir = tmp_path / "raw_storage" + settings.RAW_STORAGE_DIR = str(raw_dir) + settings.ENABLE_RAW_EXTRACTION = True + yield raw_dir + settings.RAW_STORAGE_DIR = orig_dir + settings.ENABLE_RAW_EXTRACTION = orig_enabled + + +def test_raw_storage_manager_save_and_load(temp_raw_dir): + payload = {"brand": "BMW", "specs": {"engine": "v8", "power": 390}} + timestamp = datetime(2026, 11, 7, 12, 0, 0, tzinfo=timezone.utc) + + # 1. Save + rel_path = RawStorageManager.save_payload( + brand="BMW", + oem_vehicle_id="VIN-123456", + payload=payload, + timestamp=timestamp, + ) + + assert rel_path == "bmw/2026-11-07/VIN-123456.json.gz" + full_path = temp_raw_dir / rel_path + assert full_path.exists() + + # 2. Verify compression + with gzip.open(full_path, "rt", encoding="utf-8") as f: + loaded_raw = json.load(f) + assert loaded_raw == payload + + # 3. Load via Manager + loaded = RawStorageManager.load_payload(rel_path) + assert loaded == payload + + # 4. Delete + deleted = RawStorageManager.delete_payload(rel_path) + assert deleted is True + assert not full_path.exists() + + +def test_raw_storage_manager_disabled(temp_raw_dir): + settings.ENABLE_RAW_EXTRACTION = False + payload = {"key": "val"} + + rel_path = RawStorageManager.save_payload( + brand="AUDI", + oem_vehicle_id="AUDI-001", + payload=payload, + ) + assert rel_path is None + assert not (temp_raw_dir / "audi").exists() + + +def test_parser_include_raw_toggle(): + # BMW parser with include_raw=False + raw_bmw = { + "documentId": "TEST-BMW-RAW", + "vehicleSpecification": { + "modelAndOption": { + "model": {"modelName": "BMW 330i"}, + "modelRange": {"description": {"de_DE": "3er"}}, + } + }, + "pricing": {"price": {"value": 55000}}, + "extra_huge_catalog": {"items": [1, 2, 3] * 1000}, + } + + veh_with_raw = parse_bmw_vehicle(raw_bmw, include_raw=True) + assert veh_with_raw.raw_payload == raw_bmw + + veh_no_raw = parse_bmw_vehicle(raw_bmw, include_raw=False) + assert veh_no_raw.raw_payload == {} + assert veh_no_raw.model_name == "BMW 330i" + assert veh_no_raw.series == "3er" + + +def test_pipeline_ingestion_with_raw_storage(temp_raw_dir, tmp_path): + db_file = tmp_path / "test_stock.db" + db_url = f"sqlite:///{db_file}" + engine = create_engine(db_url) + Base.metadata.create_all(engine) + session_factory = sessionmaker(bind=engine) + + pipeline = StockIngestionPipeline(session_factory=session_factory) + + test_vehicle = VehicleIngestSchema( + brand="BMW", + oem_vehicle_id="BMW-INGEST-01", + series="3er", + model_name="320d", + condition=VehicleCondition.NEW, + fuel_type="diesel", + mileage_km=0, + price_gross=Decimal("45000.00"), + status=VehicleStatus.IN_STOCK, + options=[ + OptionSchema(option_code="S01", name="Sport Line", raw_data={"pr": "S01"}), + ], + raw_payload={"full_spec": "all details here", "code": "BMW-INGEST-01"}, + ) + + def stream(): + yield test_vehicle + + run = pipeline.run_crawler_pipeline( + brand="BMW", + crawler_name="test-crawler", + vehicle_stream=stream(), + market_country="DE", + ) + assert run.items_inserted == 1 + + with session_factory() as session: + veh = session.query(Vehicle).filter_by(oem_vehicle_id="BMW-INGEST-01").first() + assert veh is not None + # In-database raw_payload is kept empty to save disk space + assert veh.raw_payload == {} + # Relative path points to the compressed file + assert veh.raw_payload_path is not None + assert veh.raw_payload_path.startswith("bmw/") + assert veh.raw_payload_path.endswith(".json.gz") + + # Options raw_data in database is kept empty + opt = session.query(VehicleOption).filter_by(vehicle_id=veh.id).first() + assert opt.name == "Sport Line" + assert opt.raw_data == {} + + # Loading via helper method reads from disk seamlessly + recovered_payload = veh.get_raw_payload() + assert recovered_payload == {"full_spec": "all details here", "code": "BMW-INGEST-01"} + + +def test_restore_database_from_raw(temp_raw_dir, tmp_path): + from oem_crawler.tools.restore_from_raw import restore_database + + # Create dummy raw payload files in temp_raw_dir + payload = { + "documentId": "BMW-RESTORE-99", + "vehicleSpecification": { + "modelAndOption": { + "model": {"modelName": "BMW 520d"}, + "modelRange": {"description": {"de_DE": "5er"}}, + } + }, + "pricing": {"price": {"value": 62000}}, + } + RawStorageManager.save_payload( + brand="bmw", + oem_vehicle_id="BMW-RESTORE-99", + payload=payload, + ) + + db_file = tmp_path / "restored.db" + db_url = f"sqlite:///{db_file}" + + # Restore into fresh DB + restore_database( + raw_dir=temp_raw_dir, + db_url=db_url, + target_brands=["bmw"], + ) + + engine = create_engine(db_url) + session_factory = sessionmaker(bind=engine) + with session_factory() as session: + veh = session.query(Vehicle).filter_by(oem_vehicle_id="BMW-RESTORE-99").first() + assert veh is not None + assert veh.model_name == "BMW 520d" + assert veh.price_gross == Decimal("62000") + assert veh.raw_payload_path.startswith("bmw/") + assert veh.raw_payload == {} + + diff --git a/tests/test_schema.py b/tests/test_schema.py new file mode 100644 index 0000000..a9b2c6d --- /dev/null +++ b/tests/test_schema.py @@ -0,0 +1,82 @@ +""" +Unit tests for data schemas and content hashing. +""" +from decimal import Decimal +import unittest + +from oem_crawler.db.schemas import ( + OptionSchema, + VehicleCondition, + VehicleIngestSchema, + compute_content_hash, +) + + +class TestSchemas(unittest.TestCase): + + def setUp(self): + self.sample_payload = { + "brand": "BMW", + "oem_vehicle_id": "BMW-DE-STOCK-839210", + "vin": "WBA33AY08NFP12345", + "series": "3 Series", + "model_name": "330e xDrive Touring M Sport", + "condition": "new", + "fuel_type": "phev", + "transmission": "automatic", + "power_kw": 215, + "mileage_km": 10, + "price_gross": Decimal("69250.00"), + "options": [ + { + "option_code": "05AU", + "name": "Driving Assistant Professional", + "category": "Driver Assistance", + "price_gross": Decimal("2200.00"), + "is_standard": False, + }, + { + "option_code": "0402", + "name": "Panorama Glass Roof", + "price_gross": Decimal("1500.00"), + "is_standard": False, + }, + ], + } + + def test_schema_parsing(self): + vehicle = VehicleIngestSchema(**self.sample_payload) + self.assertEqual(vehicle.brand, "BMW") + self.assertEqual(len(vehicle.options), 2) + self.assertEqual(vehicle.options[0].option_code, "05AU") + + def test_content_hash_stability(self): + v1 = VehicleIngestSchema(**self.sample_payload) + v2 = VehicleIngestSchema(**self.sample_payload) + self.assertEqual(compute_content_hash(v1), compute_content_hash(v2)) + + def test_hash_sensitivity_on_price_change(self): + v1 = VehicleIngestSchema(**self.sample_payload) + data2 = dict(self.sample_payload, price_gross=Decimal("68000.00")) + v2 = VehicleIngestSchema(**data2) + self.assertNotEqual(compute_content_hash(v1), compute_content_hash(v2)) + + def test_ev_battery_and_charging_schema(self): + ev_payload = dict( + self.sample_payload, + fuel_type="bev", + battery_capacity=Decimal("141.00"), + battery_capacity_unit="ah", + battery_capacity_net_kwh=Decimal("108.40"), + charging_power_ac_kw=Decimal("11.00"), + charging_power_dc_kw=Decimal("170.00"), + charging_time_dc_minutes=22, + ) + ev_vehicle = VehicleIngestSchema(**ev_payload) + self.assertEqual(ev_vehicle.battery_capacity, Decimal("141.00")) + self.assertEqual(ev_vehicle.battery_capacity_unit.value, "ah") + self.assertEqual(ev_vehicle.battery_capacity_net_kwh, Decimal("108.40")) + self.assertEqual(ev_vehicle.charging_power_ac_kw, Decimal("11.00")) + self.assertEqual(ev_vehicle.charging_power_dc_kw, Decimal("170.00")) + self.assertEqual(ev_vehicle.charging_time_dc_minutes, 22) + diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..3183a60 --- /dev/null +++ b/uv.lock @@ -0,0 +1,773 @@ +version = 1 +revision = 3 +requires-python = ">=3.10" +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version >= '3.11.5' and python_full_version < '3.12'", + "python_full_version < '3.11.5'", +] + +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + +[[package]] +name = "anyio" +version = "4.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, +] + +[[package]] +name = "beautifulsoup4" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soupsieve", version = "2.10", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11.5'" }, + { name = "soupsieve", version = "3.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11.5'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" }, +] + +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.11.5'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, +] + +[[package]] +name = "greenlet" +version = "3.5.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/d8/7cc97c142388aef03f622e001c572c4f84e9252a439549d483f555771970/greenlet-3.5.5.tar.gz", hash = "sha256:adb4bae02e91a8e863e48b177e4014bdcac8a6b5e047ea1df687a61534b85e6c", size = 207585, upload-time = "2026-08-10T15:09:36.136Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/67/07ecd6d85c0f253363cbc4ac9e7dd048ca571a267fbccfea084d6009ac3b/greenlet-3.5.5-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:816230f469381ad0a43abc9fa8dda5a699e32fb78958dde32ded93213b70a667", size = 292971, upload-time = "2026-08-10T13:28:09.837Z" }, + { url = "https://files.pythonhosted.org/packages/24/35/426733bc24247ee17bb76df90f550c299ff5f8572b2bdd7cacb5c53fd994/greenlet-3.5.5-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a5433cf291e0ef9114bd14d0d824db6e5e4a43033234bca48181a9597acca07b", size = 609290, upload-time = "2026-08-10T14:14:32.273Z" }, + { url = "https://files.pythonhosted.org/packages/9c/dc/17d3a5acceb2fd0bbc9682a228117b719cdfb68085e6dbb33fa325755f27/greenlet-3.5.5-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:19d59f068887d8c5907fc177f27683413ace3011b6ed646c0b309266e74a6502", size = 622650, upload-time = "2026-08-10T14:27:22.004Z" }, + { url = "https://files.pythonhosted.org/packages/74/3f/b31e6bd6adb8f80492efb6a47e8f9cd0e7335db5aac2795b1876d5afcfee/greenlet-3.5.5-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:86c5113d698cb8d927b2750bb1f1d59eefe3a37e0e0217491aee29a7f84ef52c", size = 629560, upload-time = "2026-08-10T14:30:04.733Z" }, + { url = "https://files.pythonhosted.org/packages/51/91/00b3c0566316c6f383ea16ee05388ec4f57f6e0afa49e72ae471eda8c47f/greenlet-3.5.5-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ff00e12102358292087274dfb1669132387ff6e7920ebf9d85f4826ce0d3a56", size = 622819, upload-time = "2026-08-10T13:40:46.562Z" }, + { url = "https://files.pythonhosted.org/packages/18/5e/58c561efde575c4ca070030e2e0513e8d1b07b76d8a2d84a9bcef1becd42/greenlet-3.5.5-cp310-cp310-manylinux_2_39_riscv64.whl", hash = "sha256:c69bed34470abfcd456984fdadaa18e62169af4480335c45f3c32d1d9c12e638", size = 425489, upload-time = "2026-08-10T14:29:59.768Z" }, + { url = "https://files.pythonhosted.org/packages/3c/78/52de4f7ac9152ad1dbc8f437895c1e573d30ee1e1427e0f91a280e2417b6/greenlet-3.5.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:523bb8e27614d77101ea7a8cf59f8d91219b72d5c29f6a038c92b50828bfa8d0", size = 1582164, upload-time = "2026-08-10T14:15:02.645Z" }, + { url = "https://files.pythonhosted.org/packages/f6/ff/c3855a00c2417e8f61c7b9f0bc4f8f599c6928f5c15681ad251e78a91e05/greenlet-3.5.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f1e2db190db51c17433eee424803818cf0670bf049d9cfe0dd07be111d1aa7c4", size = 1648807, upload-time = "2026-08-10T13:40:27.471Z" }, + { url = "https://files.pythonhosted.org/packages/70/21/d29ff6a79dbd1ec1fe74c155d25d4c5a85e221d6b9ffc2cc7a709e7c8c39/greenlet-3.5.5-cp310-cp310-win_amd64.whl", hash = "sha256:740e544169527b82695ce76af2f7ad6f030904658f2f3921a1d245771fb88cfc", size = 322832, upload-time = "2026-08-10T13:28:10.85Z" }, + { url = "https://files.pythonhosted.org/packages/4e/a3/07297917485ee2ca85bc3c8dc6ed85ad3fffcf424047fba62671dba68e97/greenlet-3.5.5-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:be63afcbbccfad3dd95a1ba12ada84dab2ef32031973d80b5b92df67fa763a61", size = 294165, upload-time = "2026-08-10T13:25:17.987Z" }, + { url = "https://files.pythonhosted.org/packages/db/51/6f732f9314cda54c5fd48a7620c7160f4f286967e8045ad94b9d66ce80b7/greenlet-3.5.5-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a268024ce2d7d2b04694bf1594058981a9fa663d1df4b762dee499211ed7c1c", size = 613610, upload-time = "2026-08-10T14:14:33.829Z" }, + { url = "https://files.pythonhosted.org/packages/d8/c0/b27589e25d220289edcd4d582b2b17b83058d1a56d53d971b6ea1a34f10d/greenlet-3.5.5-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:35cbb8bf55ace57fbccb4fb8622c4521713acd8691e77f4696d416ea7ca527da", size = 625481, upload-time = "2026-08-10T14:27:23.647Z" }, + { url = "https://files.pythonhosted.org/packages/39/82/5c873dbb4fb001d22fbbd50e80d4c1b0181ddae106856132160f84b94e88/greenlet-3.5.5-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:abc8bc8d9f935cd685457545b6a53863a877fdc12c2c0f5ee9beee18d9db139c", size = 633329, upload-time = "2026-08-10T14:30:06.062Z" }, + { url = "https://files.pythonhosted.org/packages/51/2d/f2c928218ac52f26d7a2c188c171d1b7e728b23782cb3347e7b4fce1493a/greenlet-3.5.5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74cc6df89ec5302337adc9cf096221cbed2510fd444b0e0f1586cf0470740864", size = 624562, upload-time = "2026-08-10T13:40:48.064Z" }, + { url = "https://files.pythonhosted.org/packages/70/12/f7df98e72a8eb4a7edfec5a08d6d1a4ab53a52c95ba0b2ea6c10b8dd9bd0/greenlet-3.5.5-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:3134291427bb0f3526e9d90311988caf336eb43730e95244997a4fb15f45144f", size = 428145, upload-time = "2026-08-10T14:30:01.071Z" }, + { url = "https://files.pythonhosted.org/packages/3e/4a/92fc51d5d35912f4f06eec037ba347985defd0be47463a010a325634d9d2/greenlet-3.5.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d9b454c5fc48aeaa7c4337813dbf513a6870468e426438a04d922c6d0fe63db", size = 1584909, upload-time = "2026-08-10T14:15:04.343Z" }, + { url = "https://files.pythonhosted.org/packages/ac/58/ed98b80ac5738c149a5258544843c45601ade1fd70f61740cdaead6351b3/greenlet-3.5.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03551ed792cb1b4fc0277a0c60dfd8c343894a0ba06fe60dcd22f568b433da39", size = 1651184, upload-time = "2026-08-10T13:40:28.879Z" }, + { url = "https://files.pythonhosted.org/packages/d8/be/b582ceb80cefdf9d8da34078714e4b12b3d16f509dee0f65e40a5cc8fc7d/greenlet-3.5.5-cp311-cp311-win_amd64.whl", hash = "sha256:ab3df3dffb58bf70564e93a5cec7941e4d9faa5a36cc4234a10d3131afe04f53", size = 323280, upload-time = "2026-08-10T13:26:07.495Z" }, + { url = "https://files.pythonhosted.org/packages/4d/18/5313c4c58598c38b0373c013e4ff2b3e6d258aaaa338f373335ebecdaddd/greenlet-3.5.5-cp311-cp311-win_arm64.whl", hash = "sha256:2b70a766135540c472ac1393d57c2e1b4a2eb85bf526a1e41e6d096173a8cee5", size = 307785, upload-time = "2026-08-10T13:28:34.874Z" }, + { url = "https://files.pythonhosted.org/packages/2e/7e/9ecd0285e3153532ae07aeb88063c43c72b4221cf0d4d123b02f3682e3ff/greenlet-3.5.5-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:49520f0c95a48b42cf55414b8e8479beb274ea70431afc33e3f79903c71f4380", size = 295809, upload-time = "2026-08-10T13:25:34.023Z" }, + { url = "https://files.pythonhosted.org/packages/35/73/60e4bbcc89252037b18087f2ec16405d5b2d5be42dde191bbf3667e96102/greenlet-3.5.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55272212cbc5f43d1d723725ab931f1939969b7e9523882ca58b55061769d053", size = 611910, upload-time = "2026-08-10T14:14:35.18Z" }, + { url = "https://files.pythonhosted.org/packages/a4/17/cd5134be659cd4a443e7a61ae670dabec165a814c51162916d637b6dd38e/greenlet-3.5.5-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:655bca754a2ef4efcb0eb48a94d3f4593536d0f3d48f8ed44343c01d16a92f95", size = 624198, upload-time = "2026-08-10T14:27:25.229Z" }, + { url = "https://files.pythonhosted.org/packages/9b/30/87c212b5c684d0e72974f1063b7a9687631e8985902c06e1016542c874e7/greenlet-3.5.5-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6ca5d6ae0739e5764f2cfcfaa562ac5a990cbdaedca93251c5e3cf07c362371f", size = 629504, upload-time = "2026-08-10T14:30:07.967Z" }, + { url = "https://files.pythonhosted.org/packages/78/ac/5c5b959999b6f09c3026b5dfe171575bc3121c5236ce74f495096f25b203/greenlet-3.5.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:147b25a42e5ca5be3d42356e8f608b37af715a1c196e9bf9d1627f3341adfe1d", size = 621439, upload-time = "2026-08-10T13:40:49.391Z" }, + { url = "https://files.pythonhosted.org/packages/63/2c/eb487fafc9f50ffff2b1e0b697f70fb34bf150821c08ab225aacf5583a7e/greenlet-3.5.5-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:1b5ed9162c0c098e0bbc2cf88a94f433c1b8926f831745252e099e5d83e17759", size = 432462, upload-time = "2026-08-10T14:30:02.309Z" }, + { url = "https://files.pythonhosted.org/packages/c8/8b/6acf112ed8aee499f25b4d6949820fb02ac950ff9c1f3d793bd5be0599f2/greenlet-3.5.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:27493374cff1d1b7919dc8126547f2aea582737e3046147b434b1e12de56389b", size = 1581342, upload-time = "2026-08-10T14:15:05.653Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d7/734e5f198888876b42d7616ff6644c075baf6b8a2412deadd6b0e1b8b20c/greenlet-3.5.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:12e2ee66c2aba86133f10fd99d6a8856c6d351ffb7be0e4d52ef2cc5fbb705b2", size = 1645744, upload-time = "2026-08-10T13:40:30.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/30/1f42b88dc587b5899ee50616ad56ee40cafaf225df4fb829f10183c62a5c/greenlet-3.5.5-cp312-cp312-win_amd64.whl", hash = "sha256:49ddacd36af37735fab103846f4ee4d18a492dde72730d1699c0c8ebe30d9f18", size = 324171, upload-time = "2026-08-10T13:28:44.472Z" }, + { url = "https://files.pythonhosted.org/packages/76/e5/4dee4d8d2e603fe5fdd7b444e63219f7b9bd852c60c6214511c7157cbe88/greenlet-3.5.5-cp312-cp312-win_arm64.whl", hash = "sha256:5f1b1ff4828cdc1aba4266aff814085d04a1d07959287219af021b838b265d52", size = 308362, upload-time = "2026-08-10T13:26:46.839Z" }, + { url = "https://files.pythonhosted.org/packages/fb/3d/8cef5f724ec0d4add2af8961d504535ec60c3cca9e464f6d03bdba29d85b/greenlet-3.5.5-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:b79fd2a5bc099b5e744f34c4c9a58954a5f4cb7529fb4b6e8446057d61b6edaa", size = 294730, upload-time = "2026-08-10T13:27:51.206Z" }, + { url = "https://files.pythonhosted.org/packages/88/4b/8e7aa3f514273aecff30a16ab1bac09ff54cfc7e6860fdd8058c37ff2499/greenlet-3.5.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:634cf15a233a949136879dd388e25d3296e16f3f1e217d2456797b8579ebc6ed", size = 614536, upload-time = "2026-08-10T14:14:36.589Z" }, + { url = "https://files.pythonhosted.org/packages/85/48/4e95e9dd5a8a397dc6a6345dd7f1935113d0fca4f85e89d3976da9cd988d/greenlet-3.5.5-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:499adea519f748407fc6806d20eedabac2884fd73b9f38d81236e190ba20dfef", size = 626924, upload-time = "2026-08-10T14:27:27.048Z" }, + { url = "https://files.pythonhosted.org/packages/0e/84/eaa476d6bf3816828d0d70e80dcc36bf30a058233bd889e707e693f6e860/greenlet-3.5.5-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7278591501941bb2456af102bb9cd59aab48c6cfd6e2dd68fa1290bb0c49a42", size = 632726, upload-time = "2026-08-10T14:30:09.874Z" }, + { url = "https://files.pythonhosted.org/packages/89/5d/398a1c71fa7a277deeb376c999979de6786f08fc2d5747a0b9d6e11738dd/greenlet-3.5.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2eabb980975cba5b93a95f6f69287d05fc05ac955bfd6a320a7c083eeb52c0b0", size = 623906, upload-time = "2026-08-10T13:40:50.501Z" }, + { url = "https://files.pythonhosted.org/packages/d0/f2/0cc2849ede68579291e9c59b3ab6ec1958f98681cca5b14d8fc75bf674a4/greenlet-3.5.5-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:4dfc7c4470354e7b09184d1a3a985761053a2fd694ddb5b5c80242afc2c8c90b", size = 434966, upload-time = "2026-08-10T14:30:03.729Z" }, + { url = "https://files.pythonhosted.org/packages/04/1b/745450fc5ea9e0cb17d840d248f284db3363de736d362c7d2d883e3eadba/greenlet-3.5.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:03115c2e0a371999bf8ae616aa8d653f96641d4705c457aebaa187276e9f7537", size = 1581430, upload-time = "2026-08-10T14:15:06.853Z" }, + { url = "https://files.pythonhosted.org/packages/d4/29/d51b296e3191bb15d3d81ec375af1909e4466c0f395d744ed475801798a9/greenlet-3.5.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4441153ffba21b90d3ca89fe3d31f5c093ae6c0bf0cfdfc98f54cde22f95b62e", size = 1645684, upload-time = "2026-08-10T13:40:32.133Z" }, + { url = "https://files.pythonhosted.org/packages/12/63/369f1a1625e64e9e31df3963c6044056e3fdfa3fa3fdba3c54ffefa6e987/greenlet-3.5.5-cp313-cp313-win_amd64.whl", hash = "sha256:95c5b1f4b3a193f8a0c2de4bfdcb48d119f7f1063941f1de1f2168051b3e52dd", size = 324075, upload-time = "2026-08-10T13:26:58.974Z" }, + { url = "https://files.pythonhosted.org/packages/45/78/649cb5c09d4d81f6dd1444e75474a7206784743283a21d24171562ac4899/greenlet-3.5.5-cp313-cp313-win_arm64.whl", hash = "sha256:1af90aa4bc129883b340cdd6957a3bc74f60528a4993bbd1f53aaebe1d9981cc", size = 308260, upload-time = "2026-08-10T13:27:50.795Z" }, + { url = "https://files.pythonhosted.org/packages/7f/8c/080e881fa2be95ff1ddbd6994b2bab3b1a78df3b3fcab39306011764fcc7/greenlet-3.5.5-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d4a389a852e392a6366058651a20fa5ba40d979865aa81bea2ccbdc44805070d", size = 295309, upload-time = "2026-08-10T13:26:03.032Z" }, + { url = "https://files.pythonhosted.org/packages/25/cc/0ac614e6586c0e42d4cc281a5819150f4f43685744a4c5ff77139286409d/greenlet-3.5.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70b157cd319873e8b544ddc2de158f55bbd0a9b0218c8ce9332039801518e328", size = 661185, upload-time = "2026-08-10T14:14:37.867Z" }, + { url = "https://files.pythonhosted.org/packages/5e/b9/6808725354be8ad305dfe5172377664fc9642d4fc043be246b3314cf4482/greenlet-3.5.5-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8bdfd1424abcf26832961e766570cae79efdb9599d709088c9cb6ef82b194926", size = 673419, upload-time = "2026-08-10T14:27:28.652Z" }, + { url = "https://files.pythonhosted.org/packages/eb/52/f005d579acde46c3d1cc3cab1c9f3d5708c8a3006a4120e8cf5da801afe9/greenlet-3.5.5-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d98ef6f92e67c6dbf299dbfd8facc1b0d2d9cedf91e325e73b3d0373fe4309d8", size = 677863, upload-time = "2026-08-10T14:30:11.663Z" }, + { url = "https://files.pythonhosted.org/packages/42/2e/40c509967da7f254680826a2fa0dd22138ec79946c70b97542d74cde8b43/greenlet-3.5.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:182de51c6b572a705f2fafaab2e783bcf7d2760940229dfe73086cbae037af3e", size = 670822, upload-time = "2026-08-10T13:40:51.833Z" }, + { url = "https://files.pythonhosted.org/packages/c4/8a/a75f8a2bdcef3c358a3147cdc9db3aa83755f0a038f766ab0bedb66f512c/greenlet-3.5.5-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:159df1942d88e8f784cbb38d6f18bdb365cd11319cfbb3e89623de2b97892d53", size = 480554, upload-time = "2026-08-10T14:30:05.171Z" }, + { url = "https://files.pythonhosted.org/packages/2d/22/c3c2eee4a8fe191d6d1d183086c56133d646024e3d70bfd414829f64560b/greenlet-3.5.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8fec3f165dfe332e490c3247c0f6c23b0bfc45f06496ad7f00ddb00e3d35e4dc", size = 1628469, upload-time = "2026-08-10T14:15:08.11Z" }, + { url = "https://files.pythonhosted.org/packages/f7/87/25babd09b94cb1f03e71db815fde463f0262e40cfbd953d58a8d77311351/greenlet-3.5.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c6ce25fee6cabc8bf22cb8b52e642cbb821be5b9aec8094d07ff03378141b8e9", size = 1691952, upload-time = "2026-08-10T13:40:33.502Z" }, + { url = "https://files.pythonhosted.org/packages/2e/3d/5cc9701117ea4dc0eb7bf1f4f9b7888a6e2e5277ddfae095805ace50f2b6/greenlet-3.5.5-cp314-cp314-win_amd64.whl", hash = "sha256:7dffc5c859fe6059974df1e37d7923d654a83e2ae18fdd616994270e001115e1", size = 327458, upload-time = "2026-08-10T13:27:02.868Z" }, + { url = "https://files.pythonhosted.org/packages/a7/6b/594fa2de7fae7629168a404a4305d7d7e31a5742c50a801b1839543cb93d/greenlet-3.5.5-cp314-cp314-win_arm64.whl", hash = "sha256:5e2afcfc4d4305dd715809b03da5cbe437c8984f61d8917751eb5fe4aefa3e07", size = 311146, upload-time = "2026-08-10T13:27:25.046Z" }, + { url = "https://files.pythonhosted.org/packages/24/e0/50cd600b469e5734c72709b6b1838b6bc63f307b573c772c3132d6ecfe92/greenlet-3.5.5-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:0e5a7de979d764aea1f5b6e95cf92b5b37741b9823702041f34b126e7f690277", size = 305471, upload-time = "2026-08-10T13:26:20.568Z" }, + { url = "https://files.pythonhosted.org/packages/75/a3/77acd66dfc6387b5219b2080806c0cabb73c10eb1bb44b413c40a62015ba/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fef01bd457f11fc158b130ca0027a3c365693280e8e231b65bdaf57999f39f5b", size = 672470, upload-time = "2026-08-10T14:14:39.058Z" }, + { url = "https://files.pythonhosted.org/packages/b9/71/0d178142dca3ec19f46fb2212ae73d30ad53b9d548dc64804086033a7089/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5173a72310725a74afc82c164f0e52cb8ad0de62f2bb623f24f6c0cc07d80272", size = 679973, upload-time = "2026-08-10T14:27:30.072Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ac/0d7887aa4bbfc9eba075cc428244dfc96f623478454d5ec81180d0d6bd5a/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5e9ec2e7c98e895fcea0c5cc57b2606cf86ece6d0a56578f3eb225e2af4f0387", size = 681587, upload-time = "2026-08-10T14:30:13.519Z" }, + { url = "https://files.pythonhosted.org/packages/6e/31/46eb8567302eaf787abf88d09df014e14ae3baf460af1b8b0efdbd3efcd5/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44f08341873200ba8a60a8bc14ace3d91f1754f7fa7bc66157714a8cd420a476", size = 676634, upload-time = "2026-08-10T13:40:53.004Z" }, + { url = "https://files.pythonhosted.org/packages/4f/18/8d58ba1c429b0383e3219a3d0e0bba241d0444d8ed05b73349953c7d7c7b/greenlet-3.5.5-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:102817506f6090b5176c746a82603341a549b40e5c3d5b72a4c672228a918c41", size = 510175, upload-time = "2026-08-10T14:30:07.047Z" }, + { url = "https://files.pythonhosted.org/packages/a3/e9/b88bbf5b29970cb84172dc2c32aa3e5e579ceb94c808e81c826454138850/greenlet-3.5.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d246c0db9a2513cd45f019ba178ea4d4d4705bd210ee465e2c15d76a1ab13874", size = 1637320, upload-time = "2026-08-10T14:15:09.317Z" }, + { url = "https://files.pythonhosted.org/packages/6d/8c/7631ed29cc6f0392f11830076e172ce4885e70b0bc2c1bce1731176d4b4e/greenlet-3.5.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:72507285b5caa1d17904a3f7c322ca780823a54170a0e04ec3f37bcc60d4db71", size = 1697412, upload-time = "2026-08-10T13:40:34.924Z" }, + { url = "https://files.pythonhosted.org/packages/da/0f/f7dd935f9c4cb1be49098770587f54d8a78518e55c89bce86c4fb4109057/greenlet-3.5.5-cp314-cp314t-win_amd64.whl", hash = "sha256:7805655781fb8f28a55d05fe57ed61f5f10f1892fb587673e3bb5264f28041f0", size = 331514, upload-time = "2026-08-10T13:29:20.611Z" }, + { url = "https://files.pythonhosted.org/packages/b7/e5/681b01f8fbc1b55232822f99e8f8afeb78a55a7c76a7bf9dbdc7ccb03a6d/greenlet-3.5.5-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:c0db80fcd5b8aece93f66c64f78a786bbb6b96c5fe63ef5a5a4581ecf8bab206", size = 295975, upload-time = "2026-08-10T13:28:45.985Z" }, + { url = "https://files.pythonhosted.org/packages/11/f2/69b488cd9e7267bf4b0fe8cdebf25d8d6df680d21bdf41150d23e23d6652/greenlet-3.5.5-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b241c32f912ada659808d68e308c568baf577eebf757d15471472de0c18cfad", size = 666823, upload-time = "2026-08-10T14:14:40.222Z" }, + { url = "https://files.pythonhosted.org/packages/84/d4/d5bc2fdebbdda0c94555925ba79948b8395d75a7f6a36cc85dce5bab9f11/greenlet-3.5.5-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ef6a08349401d8eaf3cb12688ac8557de95788556b8631ef17555a4a173022c0", size = 677613, upload-time = "2026-08-10T14:27:31.543Z" }, + { url = "https://files.pythonhosted.org/packages/65/53/4e13642efc4d7ad6554ecb2242a5be42666b2e1a067323e88dfc0124a04b/greenlet-3.5.5-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:37faa97daccb6d9f4c2141ce3118d023c3c5506864a7d8bdf726f665018c1f76", size = 681436, upload-time = "2026-08-10T14:30:14.839Z" }, + { url = "https://files.pythonhosted.org/packages/bd/93/542d8a3a90f3b35c6ad8bf7e56a03010287f2cafa289a5b7985b5207db39/greenlet-3.5.5-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f2e3d061b8e13aec2f0441689b3c71b244a20e5d274a52cb0f7e31bd1d139552", size = 675930, upload-time = "2026-08-10T13:40:54.205Z" }, + { url = "https://files.pythonhosted.org/packages/cd/32/188447c9a468d6977d2989397226b0c6b65ab6f4cf943f931643328512fc/greenlet-3.5.5-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:b18007dc2473a7942fd157366b55f01da6fed7ce85318591005b419e0a439474", size = 487404, upload-time = "2026-08-10T14:30:08.903Z" }, + { url = "https://files.pythonhosted.org/packages/52/b5/89c9f2e8460d71101037d47a1feed11928615a5edd42370be290e0657eeb/greenlet-3.5.5-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:9ab5f5b93655e77fe0d6c2dfd22b5eac751bb1f876d8ec21761b7c1fb9266007", size = 1633878, upload-time = "2026-08-10T14:15:10.693Z" }, + { url = "https://files.pythonhosted.org/packages/b8/60/297de93f3b02ac78a5e04d32bb8bbe3080f4a73d8ed95016561463b70618/greenlet-3.5.5-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:f0e5a21bd4452a88cf032fc43c4a5b307ab1380eacb63b5988f9c0317885e773", size = 1696597, upload-time = "2026-08-10T13:40:36.252Z" }, + { url = "https://files.pythonhosted.org/packages/18/25/54c6eaff4f337fb670215e89eb2d00d9499487b658e709d4b477be4a342e/greenlet-3.5.5-cp315-cp315-win_amd64.whl", hash = "sha256:469dbb0a78625642f4a626cfd0c6e8bccc0385b5e49189b6308bbe849ec88a8e", size = 327700, upload-time = "2026-08-10T13:28:06.752Z" }, + { url = "https://files.pythonhosted.org/packages/67/67/857e88a36301caa0e029870132c2478bd55d896630321432afab03a3115f/greenlet-3.5.5-cp315-cp315-win_arm64.whl", hash = "sha256:2d57406c3efd32d7a81e17a674314e8bd00792cdab49ea3228a49aa1bfb2e769", size = 311750, upload-time = "2026-08-10T13:34:08.815Z" }, + { url = "https://files.pythonhosted.org/packages/10/e2/3144c0a116067ac1e30457b0139a94d60d1d36a86e015de68e9ac87cb3bc/greenlet-3.5.5-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:68184dfcf50ccaa8e864770fe0633a7e27250ea9329f8192ef47ee9ecfd78e1c", size = 306387, upload-time = "2026-08-10T13:27:00.897Z" }, + { url = "https://files.pythonhosted.org/packages/5c/a1/cb4223a7e9b9f43b8807e8eb212358bfe2dfaa174a9ea2889eb1714dcba2/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9ec0dc0e59dc9c61af5c47348365ccbbd7addfafe0a93b00336ff3da2907bdc6", size = 676472, upload-time = "2026-08-10T14:14:41.417Z" }, + { url = "https://files.pythonhosted.org/packages/9e/cd/a154b4498e5d8f12ada291cfb3b8d596eadde2177f5bf09a9be699d2a446/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e604f58e35833fc46ef20302bcb314dddbfd3fcf33a4f936216d51dd678d63ae", size = 684238, upload-time = "2026-08-10T14:27:32.946Z" }, + { url = "https://files.pythonhosted.org/packages/ce/f4/e450a68a152f819491d8c7df6a8254e761d87e6a78759268961f8c5bd4dd/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2888a3a38bc5ee5bb6c438372197152e815837e4fab7ed7a1f86ef18ffd58ad1", size = 686022, upload-time = "2026-08-10T14:30:15.96Z" }, + { url = "https://files.pythonhosted.org/packages/bf/bb/b0031d260c2968a3c87deebc51d80c64e499377f993aafe06ee3b7488cc2/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:40239b5384f96da3963585cc6d7eaa9b56f8ae67e8d92cc82dd9e202fc847de3", size = 681246, upload-time = "2026-08-10T13:40:55.402Z" }, + { url = "https://files.pythonhosted.org/packages/18/23/17e63d6bf3b9c9b9dbea981b7f643a71f79603bdfb4f1c3a9cf353e22aed/greenlet-3.5.5-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:1e8d9391fe77f15649589a907cef972dbbd6352ef7ff7dc0492f658c0c26495f", size = 516951, upload-time = "2026-08-10T14:30:10.907Z" }, + { url = "https://files.pythonhosted.org/packages/9a/07/da554b71ab88e649da146e1065d86a48a5c5d92e50ab74ef41b504aa7f56/greenlet-3.5.5-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:a1eaccf5c3a1d3e46dead602c72e6836731e8e245c9de6a27764567b6b62d4c0", size = 1642735, upload-time = "2026-08-10T14:15:11.92Z" }, + { url = "https://files.pythonhosted.org/packages/78/76/26a3782a051677668af9d92beaa47cd87ba9dd5072f762961144a03dd4c6/greenlet-3.5.5-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:19e4e026fe20691f333b8eb1a3bc9625eceba8c3f9d62ec5a6f8581afbc6b5a5", size = 1700925, upload-time = "2026-08-10T13:40:37.656Z" }, + { url = "https://files.pythonhosted.org/packages/28/d9/fe7baf4190c2ae71f267efb9de21b3172bb35bc0ed1ef53dd6027d658e33/greenlet-3.5.5-cp315-cp315t-win_amd64.whl", hash = "sha256:712aee154f648bde84634654bb38bb78c69ac640c37a45c9effed800735049d8", size = 331829, upload-time = "2026-08-10T13:26:48.851Z" }, + { url = "https://files.pythonhosted.org/packages/df/af/419a4e383bd600858a9b67e9b280a60fdc383ee3f2fe5b6c0c1ef04e74d1/greenlet-3.5.5-cp315-cp315t-win_arm64.whl", hash = "sha256:7f049911ee81a16a03c33d5450d8d5867d27f596ca5fb201b86f4524e874468b", size = 315093, upload-time = "2026-08-10T13:29:34.949Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "h2" +version = "4.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "hpack" }, + { name = "hyperframe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e7/85/7c366e69d84c17bb778fe41419e1fbcce3033d5b7ce29bbffff0a98b859f/h2-4.4.1.tar.gz", hash = "sha256:4e866ffb1a869ae14dd9b5e6beb5c24a13da0495ad72b65925ded182521c1516", size = 2157281, upload-time = "2026-08-03T11:45:09.509Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/22/e85faf23bd72a92d1921e37d674ca56eb298a3c8be31fdecef0ff2b3aaac/h2-4.4.1-py3-none-any.whl", hash = "sha256:0e25f1462b23c9cb82d9eb02e28bc706dac2a68cb457c6a0d74d63c8a2a5d0e6", size = 62636, upload-time = "2026-08-03T11:44:59.164Z" }, +] + +[[package]] +name = "hpack" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/26/5b/fcabf6028144a8723726318b07a32c2f3314acdff6265743cf08a344b18e/hpack-4.2.0.tar.gz", hash = "sha256:0895cfa3b5531fc65fe439c05eb65144f123bf7a394fcaa56aa423548d8e45c0", size = 51300, upload-time = "2026-06-23T18:34:46.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/b4/4a9fcfb2aef6ba44d9073ecd301443aa00b3dac95de5619f2a7de7ec8a91/hpack-4.2.0-py3-none-any.whl", hash = "sha256:858ac0b02280fa582b5080d68db0899c62a80375e0e5413a74970c5e518b6986", size = 34246, upload-time = "2026-06-23T18:34:45.472Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[package.optional-dependencies] +http2 = [ + { name = "h2" }, +] + +[[package]] +name = "hyperframe" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, +] + +[[package]] +name = "idna" +version = "3.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/f7/abb373e5757eaec4b922b92f97ec8d6d7e057cf06778247604fbc4e7c3f3/idna-3.19.tar.gz", hash = "sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15", size = 215237, upload-time = "2026-08-18T05:14:24.27Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4", size = 68550, upload-time = "2026-08-18T05:14:22.343Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "oem-crawler" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "httpx", extra = ["http2"] }, + { name = "playwright" }, + { name = "psycopg", extra = ["binary"] }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "sqlalchemy" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "beautifulsoup4", specifier = ">=4.15.0" }, + { name = "httpx", extras = ["http2"], specifier = ">=0.27.0" }, + { name = "playwright", specifier = ">=1.64.1" }, + { name = "psycopg", extras = ["binary"], specifier = ">=3.1.18" }, + { name = "pydantic", specifier = ">=2.7.0" }, + { name = "pydantic-settings", specifier = ">=2.2.0" }, + { name = "sqlalchemy", specifier = ">=2.0.30" }, +] + +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=8.0.0" }] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "playwright" +version = "1.64.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet" }, + { name = "pyee", version = "13.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "pyee", version = "14.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/c8/cc677856d89a0c077673b40ab915ed5597bf1347b205cabe70121c431ca9/playwright-1.64.1-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:29453ff76e25c17d0b45658cacd6b92e9eec46e8db21e134fb3b7143df50f013", size = 44508971, upload-time = "2026-10-22T16:29:58.12Z" }, + { url = "https://files.pythonhosted.org/packages/72/4f/572bbbe60a5787ca61a84bad934c3ce19c08c2bc9c0e9781bf0d99cad1a6/playwright-1.64.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:59f7b041284ce8d1b72098ae50ca48a2d83692eec25154ab5ba79d4b6b4fe8a9", size = 43155232, upload-time = "2026-10-22T16:30:02.415Z" }, + { url = "https://files.pythonhosted.org/packages/44/fa/468f313a56d33a6420705147c8d47bef558f087856edf38c5b13eef8aa6c/playwright-1.64.1-py3-none-macosx_11_0_universal2.whl", hash = "sha256:6a6f99adad1ae40e6ee62efe89785db861ce7ddd57651521cb31894183cc4ae3", size = 44508977, upload-time = "2026-10-22T16:30:07.632Z" }, + { url = "https://files.pythonhosted.org/packages/15/b9/e481a34572933b5cf179d5637c98da488c732ee339dff5b73a0624d2e42b/playwright-1.64.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:577b8d88778c245cfbd145ae17cf53044f084b1b46c55c69081d4069d56057e2", size = 48471931, upload-time = "2026-10-22T16:30:12.831Z" }, + { url = "https://files.pythonhosted.org/packages/0c/11/545f25e47ec54e1e8a2892ef9994fc2b298f909ab37b88578e3a1f87ac2e/playwright-1.64.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a29fda44725e11b0b625c4bff6458a1f6dbee97b88c9788d12e18a8344458c2d", size = 48150994, upload-time = "2026-10-22T16:30:18.297Z" }, + { url = "https://files.pythonhosted.org/packages/33/87/8ff7af99387ee72ae9b46a66ece7a0c0aa63f73320ecf6dd3dd087fa840a/playwright-1.64.1-py3-none-win32.whl", hash = "sha256:d3c48b31da426c2f3562d0919cb9ea4e5415f6dbd29d7211fa4d66cca24fbd2f", size = 38801828, upload-time = "2026-10-22T16:30:23.154Z" }, + { url = "https://files.pythonhosted.org/packages/96/0a/64db6851aa1852a4f76be28eea71d4e2bf9f3ce493ee2e66939426729046/playwright-1.64.1-py3-none-win_amd64.whl", hash = "sha256:5e240a79464434c9260957b9ef59a5808e02e96e3fcabdc717fd7485260b9730", size = 38801836, upload-time = "2026-10-22T16:30:28.092Z" }, + { url = "https://files.pythonhosted.org/packages/c5/4b/d7ddde9467a5fcfe6a0d3c954732332e98a4f4a9025fd42c8d091c16e5f9/playwright-1.64.1-py3-none-win_arm64.whl", hash = "sha256:983ccd085453ea5a7588193657fc349a6ffbf7bf7ba663fd58dc81e4aa3af9d9", size = 34693875, upload-time = "2026-10-22T16:30:32.724Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "psycopg" +version = "3.3.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/2f/cb91e5502ec9de1de6f1b76cfbf69531932725361168bb06963620c77e2e/psycopg-3.3.4.tar.gz", hash = "sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc", size = 165799, upload-time = "2026-05-01T23:31:55.179Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/e0/7b3dee031daae7743609ce3c746565d4a3ed7c2c186479eb48e34e838c64/psycopg-3.3.4-py3-none-any.whl", hash = "sha256:b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a", size = 213001, upload-time = "2026-05-01T23:20:50.816Z" }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.3.4" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/bf/70d8a60488f9955cbbcd538beae44d56bb2f1d19e673b72788f2d343ff55/psycopg_binary-3.3.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b7bfff1ca23732b488cbca3076fc11bc98d520ee122514fdb17a8e20d3338f5a", size = 4609750, upload-time = "2026-05-01T23:24:20.06Z" }, + { url = "https://files.pythonhosted.org/packages/db/b0/29e98ba210c9dbc75a6dc91e3f99b9e06ea901a62ca95804e02a1ae13e6b/psycopg_binary-3.3.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:32a6fbf8481e3a370d0d72b860d35948a693cb01281da217f7b2f307636e591a", size = 4676700, upload-time = "2026-05-01T23:25:21.727Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ab/3df087b3c12bf74e47c08204172b2fabb5a144679110d5c7ad12d9201323/psycopg_binary-3.3.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bdef84570ebbce1d42b4e7ea952d21c414c5f118ad02fee00c5625f35e134429", size = 5496319, upload-time = "2026-05-01T23:25:28.271Z" }, + { url = "https://files.pythonhosted.org/packages/87/9a/f088207b4cd6772f9e0d8a91807e79fa2458d4eb9eb1ae406c68415f2bec/psycopg_binary-3.3.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa1cbc10768a796c96d3243656016bf4e337c81c71097270bb7b0ad6210d9765", size = 5171906, upload-time = "2026-05-01T23:25:34.004Z" }, + { url = "https://files.pythonhosted.org/packages/48/45/4523a857f253871d75c22e1c2e79fd47e599e736bcba1bad58d83e24be02/psycopg_binary-3.3.4-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cf7f73a4a792bc5db58a4b385d8a1467e8d468f7548702fb0ed1e9b7501b1c13", size = 6762621, upload-time = "2026-05-01T23:25:41.392Z" }, + { url = "https://files.pythonhosted.org/packages/7c/d1/925bf776503345bef428e6c45fb017d0139ddbe0e211814b585c4253dca8/psycopg_binary-3.3.4-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d7b4d40c153fa352ab3cca530f3a0baedf7621b2ebcbd7f084009522c21788fc", size = 5006319, upload-time = "2026-05-01T23:25:51.419Z" }, + { url = "https://files.pythonhosted.org/packages/6f/aa/99727337206fbba357ca084bf4ea8b29dc986f61842a2685859af61416db/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9b1c2533af01cd7648378599f82b0b8ae32f293296e6eec5753a625bc97ef28", size = 4535388, upload-time = "2026-05-01T23:25:57.957Z" }, + { url = "https://files.pythonhosted.org/packages/0b/a4/567ba2c37d19d8c2f63d836385dfd2495aa5897bbee6cfab104d9ee58624/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ad3bc94054876155549fdaedf4a46d1ec69d39a5bcee377148afe498e84c4b8e", size = 4224544, upload-time = "2026-05-01T23:26:03.832Z" }, + { url = "https://files.pythonhosted.org/packages/b7/23/86457f5a82731685d7701de7bfaa5eb783dd1fecbf875321897d9d9ce33a/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:eb4eed2079c01a4850bf467deacfab56d356d4225040170af03dc9958321242d", size = 3956282, upload-time = "2026-05-01T23:26:09.983Z" }, + { url = "https://files.pythonhosted.org/packages/a7/d8/249456df16d47de082abd9b73bce8ccdeb0293eb12e590f9150c7cbdb788/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f80e3f2b5331dbbf0901bcb658056c03eeb2c1ef31d774afb0d61598b242e744", size = 4261736, upload-time = "2026-05-01T23:26:16.798Z" }, + { url = "https://files.pythonhosted.org/packages/15/6b/c4abe228acafd8a385c1fb615d4f1e3c9b8ad7a4e4f0e84118ba3ffeed9c/psycopg_binary-3.3.4-cp310-cp310-win_amd64.whl", hash = "sha256:574ea21a9651958f1535c5a1c649c7409e9168bcbffa29a3f2f961f58b322949", size = 3570620, upload-time = "2026-05-01T23:26:22.655Z" }, + { url = "https://files.pythonhosted.org/packages/b6/82/df3312c0ca083d5b43b352f27d4dd8b1e614bd334473074715d9e0000da4/psycopg_binary-3.3.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:612a627d733f695b1de1f9b4bd511c15f999a5d8b915d444bbd7dd71cf3370da", size = 4609813, upload-time = "2026-05-01T23:26:30.612Z" }, + { url = "https://files.pythonhosted.org/packages/1f/b5/d74d542458d3e8ac0571d8a88f57ca369999b9a82f4fa528052d0d7d3e4c/psycopg_binary-3.3.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:13a7f380824c35896dcac7fe0f61440f7ca49d6dc73f3c13a9a4471e6a3b302e", size = 4676799, upload-time = "2026-05-01T23:26:38.475Z" }, + { url = "https://files.pythonhosted.org/packages/09/67/06bab9c60671999f4c6ceff1b334f3ac1f9fc5789eb467c714623ea21de9/psycopg_binary-3.3.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:276904e3452d6a23d474ef9a21eee19f20eed3d53ddd2576af033827e0ba0992", size = 5497050, upload-time = "2026-05-01T23:26:47.061Z" }, + { url = "https://files.pythonhosted.org/packages/72/9b/023433e2b20f970de1e22d29132a95281277646da0b2e2879dd4ee94b8c1/psycopg_binary-3.3.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab8cca8ef8fb1ccf5b048ae5bd78ba55b9e4b5d472e3ce5ca39ff4d2a9c249e4", size = 5172428, upload-time = "2026-05-01T23:26:56.708Z" }, + { url = "https://files.pythonhosted.org/packages/08/cd/ae16da8fde228a38b2fe9269bbc13cf89e0186173f2265600f02d6a71e64/psycopg_binary-3.3.4-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7465bfe6087d2d5b42d4c53b9b11ca9f218e477317a4a162a10e3c19e984ba8e", size = 6762746, upload-time = "2026-05-01T23:27:07.023Z" }, + { url = "https://files.pythonhosted.org/packages/4f/81/0ba09fa5f5f88779093a2541a8e02489825721f258ab88058b11d68b3eb5/psycopg_binary-3.3.4-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:22cdbf5f91ef7bb91fe0c5757e1962d3127a8010256eefd9c61fcaf441802097", size = 5006033, upload-time = "2026-05-01T23:27:12.221Z" }, + { url = "https://files.pythonhosted.org/packages/73/6a/629136040cc3497adb442a305710b5913f2a754d4630fc3d3717c4c0df65/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e2631da29253a98bd496e6c4813b24e09a4fe3fb2a9e88513305d6f8747cce95", size = 4534175, upload-time = "2026-05-01T23:27:18.248Z" }, + { url = "https://files.pythonhosted.org/packages/7c/32/1027f843c6dc2d5d51960ee62cc0c2cf755a4c39455aff1371173edbef7d/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7f7668f30b9dd5163197e5cbf4e0efd54e00f0a859cc566ce56cfc31f4054839", size = 4224203, upload-time = "2026-05-01T23:27:24.3Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e1/380a724d9093c74adb14d4fce920ea8327838abb61f760b1448586b14a8e/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:cffc3408d77a27973f33e5d909b624cce683db5fc25964b02fe0aae7886c1007", size = 3954509, upload-time = "2026-05-01T23:27:30.815Z" }, + { url = "https://files.pythonhosted.org/packages/db/cd/895893ae575a09c97ccfd5def070d88993d955ef34df45a881fd5ff506d6/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0579252a1202cd73e4da137a1426e2dae993ae44e757605344282af3a082848c", size = 4259551, upload-time = "2026-05-01T23:27:38.828Z" }, + { url = "https://files.pythonhosted.org/packages/dd/c6/2330a20794e37a3ec609ef2fd8522919ec7a4395a1abf979a8e2d1775cd5/psycopg_binary-3.3.4-cp311-cp311-win_amd64.whl", hash = "sha256:41f2ec0fea529832982bcb6c9415de3c86264ebe562b77a467c0fbcd7efbba8d", size = 3572054, upload-time = "2026-05-01T23:27:45.455Z" }, + { url = "https://files.pythonhosted.org/packages/95/7d/03818e13ba7f36de93573c93ee3482006d3dfa8b0f8d28df511bad0a1a92/psycopg_binary-3.3.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5ab28a2a7649df3b72e6b674b4c190e448e8e77cf496a65bd846472048de2089", size = 4591122, upload-time = "2026-05-01T23:27:56.162Z" }, + { url = "https://files.pythonhosted.org/packages/a5/b9/11b341edf8d54e2694726b273fe9652b254d989f4f63e3ac6816ad6b55f4/psycopg_binary-3.3.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6402a9d8146cf4b3974ded3fd28a971e83dc6a0333eb7822524a3aa20b546578", size = 4669943, upload-time = "2026-05-01T23:28:04.522Z" }, + { url = "https://files.pythonhosted.org/packages/8b/18/4665bacd65e7865b4372fcd8abb8b9186ada4b0025f8c2ca691b364a556c/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:580ae30a5f95ccd90008ec697d3ed6a4a2047a516407ad904283fa42086936e9", size = 5469697, upload-time = "2026-05-01T23:28:11.337Z" }, + { url = "https://files.pythonhosted.org/packages/7c/b1/b83136c6e510593d9b0c759ba5384337bc4ad82d19fda675adc4b2703c84/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e7510c37550f91a187e3660a8cc50d4b760f8c3b8b2f89ebc5698cd2c7f2c85d", size = 5152995, upload-time = "2026-05-01T23:28:20.529Z" }, + { url = "https://files.pythonhosted.org/packages/67/8d/a9821e2a648afe6091989929982a3b0f00b2631a859cb81379728f08fb75/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77df19583501ea288eaf15ac0fe7ad01e6d8091a91d5c41df5c718f307d8e31b", size = 6738180, upload-time = "2026-05-01T23:28:30.654Z" }, + { url = "https://files.pythonhosted.org/packages/7e/58/2e349e8d23905dc2317b80ac65f48fb6f821a4777a4e994a60da91c4850f/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:018fbed325936da502feb546642c982dcc4b9ffdea32dfef78dbf3b7f7ad4070", size = 4978828, upload-time = "2026-05-01T23:28:37.277Z" }, + { url = "https://files.pythonhosted.org/packages/45/48/57b00d03b4721878326122a1f1e6b0a90b85bcaec56b5b2f8ea6cfa45235/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:17a21953a9e5ff3a16dab692625a3676e2f101db5e40072f39dbee2250194d68", size = 4509757, upload-time = "2026-05-01T23:28:43.078Z" }, + { url = "https://files.pythonhosted.org/packages/25/37/33b47d8c007df69aec500df5889767c4d313748e8e9e27a2fef8a6dabcee/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:eb05ee1c2b817d27c537333224c9e83c7afb86fe7296ba970990068baf819b16", size = 4190546, upload-time = "2026-05-01T23:28:50.016Z" }, + { url = "https://files.pythonhosted.org/packages/ca/c6/32b0835dbc2122617902b649d76a91c1e75406e76bf3d595b0c3bb5ffad6/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:773d573e11f437ce0bdb95b7c18dc58390494f96d43f8b45b9760436114f7652", size = 3926197, upload-time = "2026-05-01T23:28:55.55Z" }, + { url = "https://files.pythonhosted.org/packages/cd/68/d190ef0c0c5b16ded07831dabc8ddd412f4cdab07ec6e30ed38d9bda0e1f/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e55ccbdfae79a2ed9c6369c3008a3025817ff9d7e27b32a2d84e2a4267e66e", size = 4236627, upload-time = "2026-05-01T23:29:05.336Z" }, + { url = "https://files.pythonhosted.org/packages/25/8f/81dcbc2e8454b74d14881275ea45f00791052dac531a9fa8be1730d1685b/psycopg_binary-3.3.4-cp312-cp312-win_amd64.whl", hash = "sha256:494ca54901be8cf9eb7e02c25b731f2317c378efa44f43e8f9bd0e1184ae7be4", size = 3560782, upload-time = "2026-05-01T23:29:11.967Z" }, + { url = "https://files.pythonhosted.org/packages/09/43/13e9c406fbbf354580476e248a16b64802a376873ebe6339e30bb655572d/psycopg_binary-3.3.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7", size = 4590377, upload-time = "2026-05-01T23:29:18.782Z" }, + { url = "https://files.pythonhosted.org/packages/22/be/2923cd7c3683e7afdecf4f10796a18de02f5c5ddc0969aa2ad0a8cdd3bbd/psycopg_binary-3.3.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:75a9067e236f9b9ae3535b66fe99bddb33d39c0de10112e49b9ab11eee53dc31", size = 4669023, upload-time = "2026-05-01T23:29:25.884Z" }, + { url = "https://files.pythonhosted.org/packages/96/a0/2c913d6fe13d6a8bd13597d36739bf47af063ad9399e402cfecab16f3c1e/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b56b603ebcea8aa10b46228b8410ba7f13e7c2ee54389d4d9be0927fd8ce2a70", size = 5467423, upload-time = "2026-05-01T23:29:33.416Z" }, + { url = "https://files.pythonhosted.org/packages/e7/38/205d10bc1ad0df4a21c5c51659126bd3ea0ef98fcad1e852f78c249bb9c3/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c677c4ad433cb7150c8cd304a0769ae3bcfbe5ea0676eb53faa7b1443b16d0d3", size = 5151137, upload-time = "2026-05-01T23:29:42.013Z" }, + { url = "https://files.pythonhosted.org/packages/36/fc/f0381ddcd45eff3bb70dbca6823a996048d7f507b2ec3fc92c6fabc0fe87/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26df2717e59c0473e4465a97dfb1b7afebaa479277870fd5784d1436470db47c", size = 6736671, upload-time = "2026-05-01T23:29:51.626Z" }, + { url = "https://files.pythonhosted.org/packages/95/40/fa545ae152c24327651e5624e4902121e808270be36c10b12e9939be09bc/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dc1f79fd16bb1f3f4421417a514607539f17804d95c7ed617265369d1981cae", size = 4979601, upload-time = "2026-05-01T23:29:56.961Z" }, + { url = "https://files.pythonhosted.org/packages/86/e4/2f8a47ee97f90cd2b933d0463081d35631ff419de2b8c984a5f369857de0/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:136f199a407b5348b9b857c504aff60c77622a28482e7195839ce1b51238c4cc", size = 4510513, upload-time = "2026-05-01T23:30:07.243Z" }, + { url = "https://files.pythonhosted.org/packages/0e/0e/94e842ff4a7f98ed162580ca2e8b8864b28c1e0350f2443f8ee47f821167/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b6f5a29e9c775b9f12a1a717aa7a2c80f9e1db6f27ba44a5b59c80ac61d2ffcf", size = 4187243, upload-time = "2026-05-01T23:30:15.352Z" }, + { url = "https://files.pythonhosted.org/packages/d0/83/fc6c174b672e29b7de996ea77b6cbddf46c891751c3355f6974292baa6b4/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:ee17a2cf4943cde261adfad1bbc5bf38d6b3776d7afff74c7cabcbeaeb08c260", size = 3927347, upload-time = "2026-05-01T23:30:21.186Z" }, + { url = "https://files.pythonhosted.org/packages/e9/65/768364d4a97a15b1a7f47ba52688c1686f22941d8332a8398cefc468e25f/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c4ab71be17bdca30cb34c34c4e1496e2f5d6f20c199c12bad226070b22ef9bf", size = 4236393, upload-time = "2026-05-01T23:30:26.211Z" }, + { url = "https://files.pythonhosted.org/packages/bd/3b/218efbc9e645becd80cdf651acda05f85cfe546b7a9c0458c7cbc8fe1f74/psycopg_binary-3.3.4-cp313-cp313-win_amd64.whl", hash = "sha256:dbfdb9b6cc79f31104a7b162a2b921b765fcc62af6c00540a167a8de47e4ed38", size = 3564592, upload-time = "2026-05-01T23:30:31.764Z" }, + { url = "https://files.pythonhosted.org/packages/48/a6/828c9185701dab71b234c2a76c38a08b098ebfec5020716b4e93807492b5/psycopg_binary-3.3.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:28b7398fdd19db3232c884fb24550bdfe951221f510e195e233299e4c9b78f97", size = 4607292, upload-time = "2026-05-01T23:30:38.962Z" }, + { url = "https://files.pythonhosted.org/packages/92/58/5b40dbc9d839045c9dae956960e4fb6d20bcabe6c59a2aa34fc3a371913f/psycopg_binary-3.3.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1fbaa292a3c8bb61b45df1ad3da1908ccee7cb889db9425e3557d9e34e2a4829", size = 4687023, upload-time = "2026-05-01T23:30:47.227Z" }, + { url = "https://files.pythonhosted.org/packages/85/a9/793f0ac107a9003b48441d0d1f9f616d96e0f37458dd8dc12528ceff55fb/psycopg_binary-3.3.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:94596f9e7633ee3f6440711d43bb70aa31cc0a46a900ab8b4201a366ace5c9e7", size = 5486985, upload-time = "2026-05-01T23:30:55.517Z" }, + { url = "https://files.pythonhosted.org/packages/8f/26/42e8533497e2592334f68ec529cf5f840f7fa4e99575a4bb61aa184dbfbf/psycopg_binary-3.3.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8c0056529e68dbe9184cd4019a1f3d8f3a4ead2f6fc7a5afcf27d3314edd1277", size = 5168745, upload-time = "2026-05-01T23:31:01.904Z" }, + { url = "https://files.pythonhosted.org/packages/15/af/b7151776cc08d5935d45c833ec818a9beb417cf7c08239af1aafbdae78ee/psycopg_binary-3.3.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c09aad7051326e7603c14e50636db9c01f78272dc54b3accff03d46370461e6", size = 6761486, upload-time = "2026-05-01T23:31:14.511Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ed/c92533b9124712d592cbf1cd6c76da933a2e0acea81dfe1fbe7e735f0cff/psycopg_binary-3.3.4-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:514404ed543efd620c85602b747df2a23cf1241b4067199e1a66f2d2757aaa41", size = 4997427, upload-time = "2026-05-01T23:31:20.901Z" }, + { url = "https://files.pythonhosted.org/packages/a2/23/ccadfd0de416aa188356daa199453af24087b042e296088706d190ae0295/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:46893c26858be12cc49ca4226ed6a60b4bfccadd946b3bebb783a60b38788228", size = 4533549, upload-time = "2026-05-01T23:31:26.204Z" }, + { url = "https://files.pythonhosted.org/packages/fd/a0/c8f43cee36386f7bc891ab41a9d31ea07cf9826038e732da79f26b1e5f34/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:df1d567fc430f6df15c9fcf67d87685fc49bdb325adc0db5af1adfb2f44eb5c9", size = 4210256, upload-time = "2026-05-01T23:31:33.884Z" }, + { url = "https://files.pythonhosted.org/packages/4e/2c/c1547871be3790676e8868b38655496422f94f0978dfb66b74bdba2f1676/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:6b9016b1714da4dd5ecaaa75b82098aa5a0b87854ce9b092e21c27c4ae23e014", size = 3946204, upload-time = "2026-05-01T23:31:39.626Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b1/f6670f00fa7ea601584623f6c11602ab92117d83eaff885e0210f6de7418/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:47c656a8a7ba6eb0cff1801a4caaa9c8bdc12d03080e273aff1c8ac39971a77e", size = 4255811, upload-time = "2026-05-01T23:31:44.986Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/08/f1ba952f1c8ae5581c70fa9c6da89f247b83e3dd8c09c035d5d7931fc23d/pydantic_core-2.46.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4", size = 2113146, upload-time = "2026-05-06T13:37:36.537Z" }, + { url = "https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5", size = 1949769, upload-time = "2026-05-06T13:37:46.365Z" }, + { url = "https://files.pythonhosted.org/packages/64/ba/bfb1d928fd5b49e1258935ff104ae356e9fd89384a55bf9f847e9193ad40/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba", size = 1974958, upload-time = "2026-05-06T13:37:28.611Z" }, + { url = "https://files.pythonhosted.org/packages/4e/74/76223bfb117b64af743c9b6670d1364516f5c0604f96b48f3272f6af6cc6/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b", size = 2042118, upload-time = "2026-05-06T13:36:55.216Z" }, + { url = "https://files.pythonhosted.org/packages/cb/7b/848732968bc8f48f3187542f08358b9d842db564147b256669426ebb1652/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c", size = 2222876, upload-time = "2026-05-06T13:38:25.455Z" }, + { url = "https://files.pythonhosted.org/packages/b5/2f/e90b63ee2e14bd8d3db8f705a6d75d64e6ee1b7c2c8833747ce706e1e0ce/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50", size = 2286703, upload-time = "2026-05-06T13:37:53.304Z" }, + { url = "https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd", size = 2092042, upload-time = "2026-05-06T13:38:46.981Z" }, + { url = "https://files.pythonhosted.org/packages/a9/da/0a422b57bf8504102bf3c4ccea9c41bab5a5cee6a54650acf8faf67f5a24/pydantic_core-2.46.4-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01", size = 2117231, upload-time = "2026-05-06T13:39:23.146Z" }, + { url = "https://files.pythonhosted.org/packages/bd/2a/2ac13c3af305843e23c5078c53d135656b3f05a2fd78cb7bbbb12e97b473/pydantic_core-2.46.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d", size = 2168388, upload-time = "2026-05-06T13:40:08.06Z" }, + { url = "https://files.pythonhosted.org/packages/72/04/2beacf7e1607e93eefe4aed1b4709f079b905fb77530179d4f7c71745f22/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4", size = 2184769, upload-time = "2026-05-06T13:38:13.901Z" }, + { url = "https://files.pythonhosted.org/packages/9e/29/d2b9fd9f539133548eaf622c06a4ce176cb46ac59f32d0359c4abc0de047/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f", size = 2319312, upload-time = "2026-05-06T13:39:08.24Z" }, + { url = "https://files.pythonhosted.org/packages/7c/af/0f7a5b85fec6075bea96e3ef9187de38fccced0de92c1e7feda8d5cc7bb9/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39", size = 2361817, upload-time = "2026-05-06T13:38:43.2Z" }, + { url = "https://files.pythonhosted.org/packages/25/a4/73363fec545fd3ec025490bdda2743c56d0dd5b6266b1a53bbe9e4265375/pydantic_core-2.46.4-cp310-cp310-win32.whl", hash = "sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d", size = 1987085, upload-time = "2026-05-06T13:39:25.497Z" }, + { url = "https://files.pythonhosted.org/packages/01/aa/62f082da2c91fac1c234bc9ee0066257ce83f0604abd72e4c9d5991f2d84/pydantic_core-2.46.4-cp310-cp310-win_amd64.whl", hash = "sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf", size = 2074311, upload-time = "2026-05-06T13:39:59.922Z" }, + { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, + { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, + { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, + { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, + { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, + { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, + { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, + { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, + { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, + { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, + { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, + { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, + { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, + { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, + { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, + { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, + { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, + { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, + { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, + { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, + { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, + { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, +] + +[[package]] +name = "pydantic-settings" +version = "2.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/ca/31c57507b13119d7d3cfa1576dad2911a4861e3be07b579395f4e9d393f9/pydantic_settings-2.15.0.tar.gz", hash = "sha256:694b793e84f766ba76a90ebdefc01d0a9a045dab0382bee70393da93712ad117", size = 261253, upload-time = "2026-08-07T09:24:57.419Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/a4/2bffa9f8e804325a09867f0e9d30795c80ea9f8d62560bd1b6ad6220eb2f/pydantic_settings-2.15.0-py3-none-any.whl", hash = "sha256:0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42", size = 69413, upload-time = "2026-08-07T09:24:55.839Z" }, +] + +[[package]] +name = "pyee" +version = "13.0.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.11.5' and python_full_version < '3.12'", + "python_full_version < '3.11.5'", +] +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" }, +] + +[[package]] +name = "pyee" +version = "14.0.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", +] +dependencies = [ + { name = "typing-extensions", marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1d/f1/fdedc2c75c3e31a330659c85e5793bb18b3397981fbf0844c6dee5b18926/pyee-14.0.0.tar.gz", hash = "sha256:76dd0f4314ecd27f02dc73589dea7fd3853f9b6176d8ef9b122860657e3602de", size = 98760, upload-time = "2026-08-13T04:26:11.021Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/12/5347938b1f9a6453f0dbdfcc3e2388a1320ef9b9ec17fbefbc4ab647ea98/pyee-14.0.0-py3-none-any.whl", hash = "sha256:3ac2d3229a9677f7de2c33d7f52fe25b638a46b19c413fea2edc8c6d0a644e4d", size = 15553, upload-time = "2026-08-13T04:26:09.916Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/53/ed9d74092561d4b01a2ef1349d52cdbc135e526c245f366b089cfca6de49/python_dotenv-1.2.3.tar.gz", hash = "sha256:a20a594dabeaa385725aa239d5244871c143ecb356add8a20fcf23773a6c3a35", size = 58945, upload-time = "2026-08-16T16:54:54.067Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/17/c5c6b53ddc18f297992099b3d9ec16c855c0ccc83263a21fe4d1c625ec6c/python_dotenv-1.2.3-py3-none-any.whl", hash = "sha256:904552145e8bfed22162c09dab1c2b9b54fefa7b23ba780f4f26ca0316b0f0d9", size = 22780, upload-time = "2026-08-16T16:54:52.473Z" }, +] + +[[package]] +name = "soupsieve" +version = "2.10" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11.5'", +] +sdist = { url = "https://files.pythonhosted.org/packages/71/c3/1b817965ac12dc002d7c9cd7dfffdd7d4fbf9b45763ef2ffe7b86ee94670/soupsieve-2.10.tar.gz", hash = "sha256:49e9380d7d2905463583bafe285e818c7366a9ed7b3aee221c1ac79c905d8bc0", size = 124337, upload-time = "2026-09-24T02:36:35.048Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/87/5ed59e1d0290564e2027ed066c52b059892c4637741337fc0967183c8d4d/soupsieve-2.10-py3-none-any.whl", hash = "sha256:8596eb8967d744174820280fa62b4542a2e955bfaccca73ed8a13c6eb8e9b502", size = 38019, upload-time = "2026-09-24T02:36:33.709Z" }, +] + +[[package]] +name = "soupsieve" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version >= '3.11.5' and python_full_version < '3.12'", +] +sdist = { url = "https://files.pythonhosted.org/packages/19/63/0b6bae54546aed98d72f12d09df57b19a287f01c84d22047fa2e23772ab9/soupsieve-3.0.4.tar.gz", hash = "sha256:924fab84ae21604c19e175e1f39815eb6fa83ba23836e09e9de180925e5bc4c0", size = 119017, upload-time = "2026-10-22T00:45:26.362Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/66/473f7ca0be0b06e38ee8c85ee2db184e3981bd769fac2e7c19c8a8865211/soupsieve-3.0.4-py3-none-any.whl", hash = "sha256:8412826614f9c632227067d0985aeb9f584bd15efc852955430e097a9fe7afbf", size = 42458, upload-time = "2026-10-22T00:45:24.843Z" }, +] + +[[package]] +name = "sqlalchemy" +version = "2.0.52" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3b/21/77b4c147963073040dc3c3a5cb7a8c3001a1893c0209432cb77f9df836aa/sqlalchemy-2.0.52.tar.gz", hash = "sha256:5e2d46356ac2ccb7d268ab6c2319ac6a2b42f1b8d5fd8bd3d46855cd82abee97", size = 9945637, upload-time = "2026-08-11T19:07:09.829Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/d7/e0354e7334d33ea2795db3ecbe2977026c05a1ecf8ba4b5953c329872453/sqlalchemy-2.0.52-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a7438774e1091192fc50a2bd8ceff5c596912d00ecd46587e88effdea7826101", size = 2172616, upload-time = "2026-08-11T20:58:21.078Z" }, + { url = "https://files.pythonhosted.org/packages/5b/64/98eef682e6946eb1b4195a9a2393db4662ebfcc89f823ae78b938765c3c0/sqlalchemy-2.0.52-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c1b7ed45bf87b214e0a9def9c2313949067efe6269db5ef18d542ee13250af7", size = 3279798, upload-time = "2026-08-11T21:00:03.535Z" }, + { url = "https://files.pythonhosted.org/packages/20/05/5b96afc1407c314347ad006b72bb251fb68ef84d05505ebf8a39bc47fcde/sqlalchemy-2.0.52-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:309cc8ba50fc5d2174189dfcd49cdf7aa711f8346afcff19f2642ae4fc449c14", size = 3277555, upload-time = "2026-08-11T21:05:47.932Z" }, + { url = "https://files.pythonhosted.org/packages/50/69/ce6776724511d1b5dd40477b08d6a5f0953a45375e092dfa852b1857732c/sqlalchemy-2.0.52-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2f9eccf8793c8c3f8dd2dfd11b9e400cb27d1d19370ef732b66017e212107822", size = 3231246, upload-time = "2026-08-11T21:00:05.184Z" }, + { url = "https://files.pythonhosted.org/packages/72/19/ab0cb9ccdafa2419c796ae62f8740aedb903f1e93bb326064b1a0147e458/sqlalchemy-2.0.52-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9255ceb65a80c1b001129060b63ee776a2e9c288be3b662be36dfbb888fffdcd", size = 3250763, upload-time = "2026-08-11T21:05:49.664Z" }, + { url = "https://files.pythonhosted.org/packages/41/f2/3c9b54b61bec4f493c0007dc9e2700c963c5b32667e21a506f1aca7a8115/sqlalchemy-2.0.52-cp310-cp310-win32.whl", hash = "sha256:2e15b1d1116a64fc399b8c2694a83f3e792fdc58df28514a81e1dc4f8cf22729", size = 2132169, upload-time = "2026-08-11T21:09:48.108Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a1/934bb6cf543a398c72784d1fc777eb530559c16ebe1549fa7611e5989ce9/sqlalchemy-2.0.52-cp310-cp310-win_amd64.whl", hash = "sha256:11560064cc4696e772298b6221ede59e646386d9f2a85d549365473b972f7850", size = 2156289, upload-time = "2026-08-11T21:09:49.429Z" }, + { url = "https://files.pythonhosted.org/packages/6b/08/cc5f7627b92f1456bc0b5fb7e98af4600248abe422a44da0d17a3fe6a448/sqlalchemy-2.0.52-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0c3ce43907374889f3352bdcc6195c970148a2cb71574cd0237a5071a37fb6c", size = 2172460, upload-time = "2026-08-11T20:58:22.429Z" }, + { url = "https://files.pythonhosted.org/packages/ed/dc/9a2abad8bfc8fdcd38c64adc056aeefab7aaa96ecd32f5e8c140e6375f17/sqlalchemy-2.0.52-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a0d48c4b80717c61385b4e966e087c839a66cfd7b780641dcb428f4dba65608", size = 3355720, upload-time = "2026-08-11T21:00:06.746Z" }, + { url = "https://files.pythonhosted.org/packages/a8/73/e75597b5841043e3c74055d00d4feb53d9a49a5c89ba2450d2d9aab53597/sqlalchemy-2.0.52-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:938325a5373267afc53bfbe72983b20fbd64ca47842aac62433c3da1137ecff1", size = 3354394, upload-time = "2026-08-11T21:05:51.454Z" }, + { url = "https://files.pythonhosted.org/packages/12/25/410fbc6c2f1fa8310f4ef1b6847d47d0ac1c042c7b4e81eaaca063d030a9/sqlalchemy-2.0.52-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5f8438a98d49424acf69d0d53c0a522951dfe49a6f2d86417fbb37ad3066ab43", size = 3306991, upload-time = "2026-08-11T21:00:08.603Z" }, + { url = "https://files.pythonhosted.org/packages/b2/ba/25ffd5c24681ea4b46e62c80ceca8200ce204de1773366321306cf3f608a/sqlalchemy-2.0.52-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4699dbb8d396d199e7e78fd4d525e3ad3d6008a9c8c0160b87e74c606c2c3736", size = 3327454, upload-time = "2026-08-11T21:05:53.368Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f1/0f1b1d4800e51218e736a06ed55a3b2a59c257600bbaca7673bf13d2dbec/sqlalchemy-2.0.52-cp311-cp311-win32.whl", hash = "sha256:cef328349452ae152637df4d11ce5a0919ecdf0a363e16c830c3518ee33bde72", size = 2131248, upload-time = "2026-08-11T21:09:50.765Z" }, + { url = "https://files.pythonhosted.org/packages/7a/f0/04d2ac5ad66f3d31278f37064ed5f5ef3fe653f7bdaa67036663f223d186/sqlalchemy-2.0.52-cp311-cp311-win_amd64.whl", hash = "sha256:f1c850792a3b25a3ad74dade3f05e4f402cdebfea27438bcadafaa1617f77bcc", size = 2156943, upload-time = "2026-08-11T21:09:51.979Z" }, + { url = "https://files.pythonhosted.org/packages/e0/d5/1b77a026d161f98a08f11af1a5f6c47b98ee7c7e2648af525a1004826c78/sqlalchemy-2.0.52-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:be8c49131665dfe2cc74c498aa1240ffb548d0fd901325dd11c2c7a18956f727", size = 2170940, upload-time = "2026-08-11T20:58:11.25Z" }, + { url = "https://files.pythonhosted.org/packages/54/bd/f444444adb37b5d53753fb1730ee7a421628e2e3b756c4da461af7e6394a/sqlalchemy-2.0.52-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b2d9e507a458832adcfbd8af6e2036ddf069b7710b799448542ebccae2dceee", size = 3383415, upload-time = "2026-08-11T21:02:38.534Z" }, + { url = "https://files.pythonhosted.org/packages/be/57/2eadf93a552568c57e8680b7e58bb5e9770d80942a1bdbaf4f2f63f0d7c8/sqlalchemy-2.0.52-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8738008376d22f30f411ea3efecf39b51110b6996d80bb73786f30bcfdd5fd3b", size = 3398577, upload-time = "2026-08-11T21:16:59.092Z" }, + { url = "https://files.pythonhosted.org/packages/15/c3/2887cf9dd111d1fbf05d22165b404c221ef43e029f7a2695e7302f27a7cc/sqlalchemy-2.0.52-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37a4d548327b6cab9c7d8cdb4e0e82feabee0110c4d150059068e2d1cfbd99ee", size = 3328225, upload-time = "2026-08-11T21:02:40.183Z" }, + { url = "https://files.pythonhosted.org/packages/02/0f/466bdf9e1feeeef5587f868c187d8687e21ff8c85b1775e9041130181132/sqlalchemy-2.0.52-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e49f51a5d59857a7a0dcaf9469febf7197d9394bd88f00d69c2c4e848112cdbf", size = 3357374, upload-time = "2026-08-11T21:17:01.076Z" }, + { url = "https://files.pythonhosted.org/packages/22/20/5c2b4583904af4173076dda1c9e53c9e2ffc7a702d2efde0216bbacbf7cb/sqlalchemy-2.0.52-cp312-cp312-win32.whl", hash = "sha256:afda3ec521d0517d0de783fc70030775841900896d832de5bbd066549290470e", size = 2129366, upload-time = "2026-08-11T21:14:50.991Z" }, + { url = "https://files.pythonhosted.org/packages/ed/06/543dab8ef62d4e9fb96fb31a30c2b8b14a8763bccf48d428294d6b3041c0/sqlalchemy-2.0.52-cp312-cp312-win_amd64.whl", hash = "sha256:2d5e53e36e37129fe0be8b9d08b6e4052c10a963ee6cda56c8c10dcc194b99ca", size = 2157344, upload-time = "2026-08-11T21:14:52.453Z" }, + { url = "https://files.pythonhosted.org/packages/7f/18/e30c6fe1eca1bf34a39fbdd6066121cc9974c850faf6f349eac563697a26/sqlalchemy-2.0.52-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2eb3c6a64b1bfe6704777cfd504e7b8ad093a5f3e03ce67663a5e6742f294e43", size = 2167724, upload-time = "2026-08-11T20:58:12.679Z" }, + { url = "https://files.pythonhosted.org/packages/d0/56/2e17d161a4f7ecc1c2ffb93e607b4e1898bb551b451b283235acb8f6ce47/sqlalchemy-2.0.52-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:923bb183c1dc64fdf7b717965e3d59938ec4f8b8710b419a21ce403e5da9a9e1", size = 3321189, upload-time = "2026-08-11T21:02:41.932Z" }, + { url = "https://files.pythonhosted.org/packages/cf/b8/8490916e893f3f8d74dc9cc54c078619364999dee37047a188e73abbc852/sqlalchemy-2.0.52-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:651d6d8782e80679e6151707c7b490834d46ada526328895abf567f25e63d29c", size = 3338185, upload-time = "2026-08-11T21:17:02.597Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f7/752cc8ee453da222829b3f5c4613614bf750d97429363b70414fa10478e4/sqlalchemy-2.0.52-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b08cddb8989775e3c88799d86704bdfc3ee6e9846118201aa5997f16f27e3a15", size = 3271698, upload-time = "2026-08-11T21:02:43.963Z" }, + { url = "https://files.pythonhosted.org/packages/51/e6/074ade0c07b9e4c8e8bca46820320ed94df9702afdb6f2af06623068d2e6/sqlalchemy-2.0.52-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ab66fa9618269390d4dfa222f2f2f88f7bc4bf5da13905131b818217db7e8057", size = 3308936, upload-time = "2026-08-11T21:17:04.172Z" }, + { url = "https://files.pythonhosted.org/packages/66/07/557c0d04716705599227945ac14e0a17ad0338e899f37d8c2ddff4dcc663/sqlalchemy-2.0.52-cp313-cp313-win32.whl", hash = "sha256:c63bda077685c85ca513286547a531ba57e7a68cf0a7ed3bafcc2bbd18896f4d", size = 2127308, upload-time = "2026-08-11T21:14:53.879Z" }, + { url = "https://files.pythonhosted.org/packages/96/4e/226eda27654318ce525d043025221f689abef883da2c7126f9065121618c/sqlalchemy-2.0.52-cp313-cp313-win_amd64.whl", hash = "sha256:9876b09b9f1ce7398b0ffece585c0a911244c53191187341f6bcae640e133751", size = 2153876, upload-time = "2026-08-11T21:14:55.527Z" }, + { url = "https://files.pythonhosted.org/packages/d5/f5/71cb30af58c9b80a4e1fac0b73bb48f86d497a774a6a2eb6d2f1e657bb73/sqlalchemy-2.0.52-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:410d52be41d17f1a236d19520fbe776257dc16516ed06bd16d433311842aefd9", size = 2169537, upload-time = "2026-08-11T20:58:13.855Z" }, + { url = "https://files.pythonhosted.org/packages/4c/93/d07ebd645d1b07b6b5ed63450a70f063a346a7e0f2c8810daf2e532400cb/sqlalchemy-2.0.52-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfe9ce533dbe4d0a2ae1486546619bd30b76bcd670539a44d910361376175f5e", size = 3319606, upload-time = "2026-08-11T21:02:45.829Z" }, + { url = "https://files.pythonhosted.org/packages/ae/5c/290c84c7c2566ecd3b65baaae0fddec9bc33b033b398a06123bb86fbfc6e/sqlalchemy-2.0.52-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:812bae5138bfc0aa46fb0686da0fc7f581f68e2bbb05bc24c3713bebaedd1437", size = 3323642, upload-time = "2026-08-11T21:17:05.675Z" }, + { url = "https://files.pythonhosted.org/packages/13/f5/2cc160590ca49173359557880b92a0572293ccb899e8f6cedf150c5a3ddf/sqlalchemy-2.0.52-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:50bff43b632a56fbf5ed9afdd76307e1512b62051bcd5afb341ae67205bbb6c8", size = 3268125, upload-time = "2026-08-11T21:02:47.649Z" }, + { url = "https://files.pythonhosted.org/packages/35/f3/ea8933fc9f7d1353e9c2ff9965eae687c4cef181120574591ed2fa0633e1/sqlalchemy-2.0.52-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:49565daf5af554f538e23aef1fc81a95a4e49658f152285e45c02f5fc44f04cd", size = 3289516, upload-time = "2026-08-11T21:17:07.267Z" }, + { url = "https://files.pythonhosted.org/packages/45/67/05cf86541c1e1716fca1e4a996954a439cd74501707cda607fb7cb02ef50/sqlalchemy-2.0.52-cp314-cp314-win32.whl", hash = "sha256:ab9da41e61b9979b910499d633b241df20c51ee5037e5405b11c2faac3cbe1a2", size = 2130249, upload-time = "2026-08-11T21:14:57.273Z" }, + { url = "https://files.pythonhosted.org/packages/96/d7/8ac6ffa1e36169e762ef65bd835046abb2251b1bc17f8f6708e14ed8d31f/sqlalchemy-2.0.52-cp314-cp314-win_amd64.whl", hash = "sha256:a593db51b3bae75db17a5738ad5f992244b3a03863f83c28117ee482c6a3f76d", size = 2156718, upload-time = "2026-08-11T21:14:58.667Z" }, + { url = "https://files.pythonhosted.org/packages/dc/4b/e01a737eef378e734cc6394a82248a6ce13b167dfa36c731075ce9fc9c64/sqlalchemy-2.0.52-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1e61d08bdf4ee2f41024569e3400de7d6734ba498144766b11260936ccfa582", size = 2190344, upload-time = "2026-08-11T19:53:21.393Z" }, + { url = "https://files.pythonhosted.org/packages/b3/3f/3582293d1e185e71d19d7c731c3e2ee20ba21981c4a1115c0806c1f62120/sqlalchemy-2.0.52-py3-none-any.whl", hash = "sha256:3b81b8363a919ce53453591cdb93702e6bd54ade6c4fa2f468fc053baee5ed89", size = 1950700, upload-time = "2026-08-11T20:47:21.603Z" }, +] + +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, +] + +[[package]] +name = "tzdata" +version = "2026.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" }, +]