Initial commit
This commit is contained in:
Steffen
commit
8bd33db6dd
47 changed files with 8660 additions and 0 deletions
16
.dockerignore
Normal file
16
.dockerignore
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
.venv
|
||||
__pycache__
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.pyd
|
||||
.pytest_cache
|
||||
.coverage
|
||||
htmlcov/
|
||||
*.db
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
sample_*.json
|
||||
*.log
|
||||
.git
|
||||
.gitignore
|
||||
.env
|
||||
18
.env.example
Normal file
18
.env.example
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
# Database connection
|
||||
DATABASE_URL=postgresql+psycopg://crawler_user:crawler_password@localhost:5432/oem_stock_db
|
||||
DB_ECHO_SQL=false
|
||||
|
||||
# Crawler Defaults
|
||||
DEFAULT_MARKET=DE
|
||||
DEFAULT_PAGE_SIZE=12
|
||||
DEFAULT_DELAY_SECONDS=1.0
|
||||
HTTP_TIMEOUT_SECONDS=25.0
|
||||
DEFAULT_PROXY=
|
||||
|
||||
# Raw Payload Storage & Extraction
|
||||
ENABLE_RAW_EXTRACTION=true
|
||||
RAW_STORAGE_DIR=data/raw
|
||||
RAW_COMPRESSION_LEVEL=6
|
||||
|
||||
# Logging
|
||||
LOG_LEVEL=INFO
|
||||
72
.gitignore
vendored
Normal file
72
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# Virtual Environments & Package Managers
|
||||
.venv/
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
|
||||
# Python Build & Packaging
|
||||
build/
|
||||
dist/
|
||||
*.egg-info/
|
||||
*.egg
|
||||
|
||||
# Testing, Coverage & Linters
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
htmlcov/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
|
||||
# Playwright & Browser Artifacts
|
||||
test-results/
|
||||
playwright-report/
|
||||
.playwright/
|
||||
|
||||
# Local Data & SQLite Storage
|
||||
data/
|
||||
/data/
|
||||
tmp/
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
*.db
|
||||
*.db-journal
|
||||
*.db-wal
|
||||
*.db-shm
|
||||
|
||||
# Local Logs, Exports & Payloads
|
||||
*.log
|
||||
*.json
|
||||
*.json.gz
|
||||
!pyproject.toml
|
||||
|
||||
# Environment Variables (Keep example tracked)
|
||||
.env
|
||||
.env.*
|
||||
.env.local
|
||||
!.env.example
|
||||
|
||||
# IDEs & Editors
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*.sublime-project
|
||||
*.sublime-workspace
|
||||
|
||||
# macOS & OS Metadata
|
||||
.DS_Store
|
||||
.AppleDouble
|
||||
.LSOverride
|
||||
._*
|
||||
Icon?
|
||||
|
||||
# Nextcloud Sync Artifacts
|
||||
*conflict*
|
||||
*conflicted copy*
|
||||
.sync_*.db*
|
||||
.sync/
|
||||
35
Dockerfile
Normal file
35
Dockerfile
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
# syntax=docker/dockerfile:1
|
||||
FROM python:3.11-slim
|
||||
|
||||
# Install system dependencies
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl \
|
||||
ca-certificates \
|
||||
libpq-dev \
|
||||
gcc \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install uv for fast Python package management
|
||||
COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy dependency definitions
|
||||
COPY pyproject.toml /app/
|
||||
|
||||
# Install Python dependencies
|
||||
RUN uv pip install --system --no-cache -r pyproject.toml
|
||||
|
||||
# Copy source code and database migrations
|
||||
COPY src/ /app/src/
|
||||
COPY schema.sql /app/
|
||||
COPY README.md /app/
|
||||
|
||||
# Install package in editable mode
|
||||
RUN uv pip install --system --no-cache -e .
|
||||
|
||||
ENV PYTHONPATH="/app/src"
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
ENTRYPOINT ["oem-crawler"]
|
||||
CMD ["--brand", "bmw", "--limit", "50"]
|
||||
213
README.md
Normal file
213
README.md
Normal file
|
|
@ -0,0 +1,213 @@
|
|||
# OEM Vehicle Stock Crawler
|
||||
|
||||
A production-ready, modular platform for crawling car manufacturer (OEM) inventory, tracking vehicle stock, specifications, optional equipment, and price changes over time.
|
||||
|
||||
---
|
||||
|
||||
## 📁 Project Architecture
|
||||
|
||||
```
|
||||
oem_crawler/
|
||||
├── Dockerfile # Lean production container using uv
|
||||
├── docker-compose.yml # PostgreSQL 16 + crawler container orchestrator
|
||||
├── .env.example # Environment configuration template
|
||||
├── pyproject.toml # Dependencies and CLI entrypoints (`oem-crawler`, `oem-inspect`)
|
||||
├── schema.sql # PostgreSQL database DDL
|
||||
│
|
||||
├── data/ # Development SQLite databases (gitignored)
|
||||
│ └── crawler_stock.db # Default local SQLite database file
|
||||
│
|
||||
├── src/oem_crawler/
|
||||
│ ├── config.py # Environment configuration (pydantic-settings)
|
||||
│ ├── cli.py # Main CLI application (`oem-crawler`)
|
||||
│ │
|
||||
│ ├── tools/ # Developer & stock inspection tools
|
||||
│ │ ├── README.md # Tools documentation and sqlite3 recipes
|
||||
│ │ └── view_stock.py # Terminal inventory inspector (`oem-inspect`)
|
||||
│ │
|
||||
│ ├── db/ # Database Layer
|
||||
│ │ ├── models.py # SQLAlchemy 2.0 ORM mappings
|
||||
│ │ ├── schemas.py # Pydantic v2 validation models & SHA-256 hashing
|
||||
│ │ ├── session.py # Connection and session factory
|
||||
│ │ └── pipeline.py # Ingestion pipeline, change-detection & delisting
|
||||
│ │
|
||||
│ └── crawlers/ # Modular OEM Crawlers
|
||||
│ ├── base.py # BaseCrawler interface & CrawlerRegistry
|
||||
│ ├── audi/ # Audi Implementation
|
||||
│ │ ├── crawler.py # Stock Car Service (SCS) REST client
|
||||
│ │ └── parser.py # PR option codes, EV specs & GPS coordinates
|
||||
│ ├── bmw/ # BMW Implementation
|
||||
│ │ ├── crawler.py # REST API client & pagination
|
||||
│ │ └── parser.py # OEM response normalization
|
||||
│ ├── mercedes/ # Mercedes-Benz Implementation
|
||||
│ │ ├── crawler.py # OneSearch GraphQL + PLS + VPC client
|
||||
│ │ └── parser.py # OneWeb normalization logic
|
||||
│ │ └── query.py # Tested GraphQL query definitions
|
||||
│ └── porsche/ # Porsche Implementation
|
||||
│ ├── crawler.py # Finder Next.js RSC client & pagination
|
||||
│ └── parser.py # Option C, equipment & metadata normalization
|
||||
│
|
||||
└── tests/ # Automated Test Suite
|
||||
├── test_schema.py # Schema & hashing unit tests
|
||||
├── test_audi_parser.py # Audi parser & registry unit tests
|
||||
├── test_bmw_parser.py # BMW parser & registry unit tests
|
||||
├── test_mercedes_parser.py # Mercedes parser unit tests
|
||||
├── test_porsche_parser.py # Porsche parser unit tests
|
||||
└── test_pipeline.py # Ingestion pipeline unit tests
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔌 Adding a New Car Manufacturer Crawler
|
||||
|
||||
Adding a new OEM crawler (e.g. Mercedes, Audi, Tesla, Porsche) takes 2 simple steps:
|
||||
|
||||
1. Create a new folder under `src/oem_crawler/crawlers/<brand>/` (e.g. `mercedes/`):
|
||||
2. Implement your crawler inheriting from `BaseCrawler` and register it:
|
||||
|
||||
```python
|
||||
from oem_crawler.crawlers.base import BaseCrawler, CrawlerRegistry
|
||||
from oem_crawler.db.schemas import VehicleIngestSchema
|
||||
|
||||
@CrawlerRegistry.register("mercedes")
|
||||
class MercedesStockCrawler(BaseCrawler):
|
||||
@property
|
||||
def brand_name(self) -> str:
|
||||
return "Mercedes-Benz"
|
||||
|
||||
def crawl(self, max_vehicles=None, page_size=20, search_filter=None):
|
||||
# 1. Fetch from OEM API / site
|
||||
# 2. Parse into VehicleIngestSchema
|
||||
# 3. Yield parsed vehicle
|
||||
...
|
||||
```
|
||||
|
||||
The CLI (`oem-crawler --brand mercedes`) and pipeline will automatically discover and support it.
|
||||
|
||||
---
|
||||
|
||||
## 🐳 Running with Docker
|
||||
|
||||
### 1. Start PostgreSQL Database & Run a Crawl
|
||||
```bash
|
||||
docker compose up --build
|
||||
```
|
||||
|
||||
### 2. Run Ad-hoc Crawl inside Docker
|
||||
```bash
|
||||
# Fetch 100 vehicles for BMW
|
||||
docker compose run --rm crawler --brand bmw --limit 100
|
||||
```
|
||||
|
||||
### 3. Query PostgreSQL Database inside Docker
|
||||
|
||||
You can query the database directly in your terminal using `docker compose exec`:
|
||||
|
||||
```bash
|
||||
# View latest vehicles in stock
|
||||
docker compose exec postgres psql -U crawler_user -d oem_stock_db -c \
|
||||
"SELECT brand, series, model_name, price_gross, exterior_color, location_city FROM vehicles LIMIT 10;"
|
||||
|
||||
# View stored factory equipment & options for a vehicle
|
||||
docker compose exec postgres psql -U crawler_user -d oem_stock_db -c \
|
||||
"SELECT option_code, name, category, is_standard, price_gross FROM vehicle_options LIMIT 15;"
|
||||
|
||||
# View crawl run history and telemetry
|
||||
docker compose exec postgres psql -U crawler_user -d oem_stock_db -c \
|
||||
"SELECT id, brand, status, items_discovered, items_inserted, duration_ms, started_at FROM crawl_runs;"
|
||||
|
||||
# View price and status history snapshots
|
||||
docker compose exec postgres psql -U crawler_user -d oem_stock_db -c \
|
||||
"SELECT vehicle_id, price_gross, monthly_rate, status, recorded_at FROM vehicle_price_history LIMIT 10;"
|
||||
```
|
||||
|
||||
### 4. Connect with GUI Database Clients (DBeaver, TablePlus, pgAdmin)
|
||||
- **Host**: `localhost`
|
||||
- **Port**: `5432`
|
||||
- **Database**: `oem_stock_db`
|
||||
- **User**: `crawler_user`
|
||||
- **Password**: `crawler_password`
|
||||
|
||||
---
|
||||
|
||||
## 💻 Local Development (without Docker)
|
||||
|
||||
### 1. Run Unit Tests
|
||||
```bash
|
||||
uv run pytest
|
||||
```
|
||||
|
||||
### 2. Run Local Crawl & Persistent Ingestion
|
||||
|
||||
> **Note**: The default local database is automatically stored in `data/crawler_stock.db` (gitignored). You do not need to specify `--db-url` unless targeting a custom location or PostgreSQL.
|
||||
|
||||
```bash
|
||||
# Crawl New BMW Vehicles (Neuwagensuche)
|
||||
uv run oem-crawler --brand bmw --stock-type new --limit 50
|
||||
|
||||
# Crawl New Mercedes-Benz Vehicles (Neuwagensuche via OneSearch GraphQL + PLS/VPC)
|
||||
uv run oem-crawler --brand mercedes --stock-type new --limit 50
|
||||
|
||||
# Crawl Used Mercedes-Benz Vehicles (Gebrauchtwagen via gebrauchtwagen.mercedes-benz.de)
|
||||
uv run oem-crawler --brand mercedes --stock-type used --limit 50
|
||||
|
||||
# Crawl Both New & Used Vehicles
|
||||
uv run oem-crawler --brand mercedes --stock-type all --limit 100
|
||||
|
||||
# Crawl Used BMW Vehicles (Gebrauchtwagen with mileage & first registration)
|
||||
uv run oem-crawler --brand bmw --stock-type used --limit 50
|
||||
|
||||
# Crawl New Porsche Vehicles (Finder Neuwagen via Next.js RSC)
|
||||
uv run oem-crawler --brand porsche --stock-type new --limit 50
|
||||
|
||||
# Crawl Used / Approved Porsche Vehicles (Finder Gebrauchtwagen via Next.js RSC)
|
||||
uv run oem-crawler --brand porsche --stock-type used --limit 50
|
||||
|
||||
# Crawl Both New & Used Porsche Vehicles
|
||||
uv run oem-crawler --brand porsche --stock-type all --limit 100
|
||||
|
||||
# Or export Porsche stock directly to local JSON
|
||||
uv run oem-crawler --brand porsche --stock-type new --limit 20 --output-json porsche_new_stock.json
|
||||
|
||||
# Crawl New Audi Vehicles (Neuwagenbörse via Stock Car Service scs.audi.de)
|
||||
uv run oem-crawler --brand audi --stock-type new --limit 50
|
||||
|
||||
# Crawl Used Audi Vehicles (Gebrauchtwagenbörse with mileage, dealer GPS & PR options)
|
||||
uv run oem-crawler --brand audi --stock-type used --limit 50
|
||||
|
||||
# Crawl Both New & Used Audi Vehicles
|
||||
uv run oem-crawler --brand audi --stock-type all --limit 100
|
||||
|
||||
# --- Full Scans & Multi-Brand Ingestion ---
|
||||
|
||||
# 🚀 Full Scan across ALL Brands (Audi, BMW, Mercedes-Benz, Porsche)
|
||||
uv run oem-crawler --all-brands --full-scan --delay 0.3
|
||||
|
||||
# Full Scan across all brands for New Cars only
|
||||
uv run oem-crawler --all-brands --stock-type new --full-scan --delay 0.3
|
||||
|
||||
# Scan a custom selection of brands in one command
|
||||
uv run oem-crawler --brand bmw,audi,porsche --full-scan --delay 0.3
|
||||
```
|
||||
|
||||
### 3. Inspect Stored Inventory in Terminal (`oem-inspect`)
|
||||
|
||||
The project includes an inspection and auditing utility located in `src/oem_crawler/tools/`:
|
||||
|
||||
```bash
|
||||
# View inventory summary and brand breakdown table
|
||||
uv run oem-inspect --summary
|
||||
|
||||
# View first 10 vehicles across all brands
|
||||
uv run oem-inspect --limit 10
|
||||
|
||||
# Filter by brand
|
||||
uv run oem-inspect --brand audi --limit 5
|
||||
uv run oem-inspect --brand porsche --limit 10
|
||||
|
||||
# Or run via Python module directly
|
||||
uv run python -m oem_crawler.tools.view_stock --summary
|
||||
```
|
||||
|
||||
For more options and direct SQLite query recipes, see the [Tools Documentation](src/oem_crawler/tools/README.md).
|
||||
|
||||
60
demo_crawl.py
Normal file
60
demo_crawl.py
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
"""
|
||||
Demo Script: Live BMW Stock Crawling and Database Ingestion Prototype
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from sqlalchemy import select, func
|
||||
|
||||
from oem_crawler.crawlers.bmw.crawler import BMWStockCrawler
|
||||
from oem_crawler.db.models import Vehicle, VehicleOption, VehiclePriceHistory
|
||||
from oem_crawler.db.pipeline import StockIngestionPipeline
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
|
||||
|
||||
def main():
|
||||
print("=" * 70)
|
||||
print("🚀 Initializing OEM Stock Crawler Prototype")
|
||||
print("=" * 70)
|
||||
|
||||
pipeline = StockIngestionPipeline()
|
||||
crawler = BMWStockCrawler(market_country="DE")
|
||||
|
||||
limit = 6
|
||||
print(f"\n📡 Crawling first {limit} live vehicles from BMW Neuwagensuche REST API...")
|
||||
stream = crawler.crawl(max_vehicles=limit, page_size=limit)
|
||||
|
||||
crawl_run = pipeline.run_crawler_pipeline(
|
||||
brand="BMW",
|
||||
crawler_name="bmw-rest-crawler-v1",
|
||||
vehicle_stream=stream,
|
||||
market_country="DE",
|
||||
config_snapshot={"max_vehicles": limit, "is_full_market_scan": False},
|
||||
)
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print(f"✅ Crawl Run Complete: {crawl_run.id}")
|
||||
print(f" - Status: {crawl_run.status}")
|
||||
print(f" - Duration: {crawl_run.duration_ms} ms")
|
||||
print(f" - Discovered: {crawl_run.items_discovered}")
|
||||
print(f" - Inserted: {crawl_run.items_inserted}")
|
||||
print(f" - Updated: {crawl_run.items_updated}")
|
||||
print(f" - Unmodified: {crawl_run.items_unmodified}")
|
||||
print("=" * 70)
|
||||
|
||||
with pipeline.session_factory() as session:
|
||||
vehicles = session.scalars(select(Vehicle).limit(3)).all()
|
||||
print("\n🔍 Sample Stored Vehicles:")
|
||||
for v in vehicles:
|
||||
options_count = session.scalar(
|
||||
select(func.count(VehicleOption.id)).where(VehicleOption.vehicle_id == v.id)
|
||||
)
|
||||
print(f" • [{v.brand}] {v.series} {v.model_name}")
|
||||
print(f" Gross Price: {v.price_gross:,.2f} {v.currency} | Options: {options_count} items stored")
|
||||
print(f" Dealer: {v.dealer_name or 'N/A'} ({v.location_city or 'N/A'})")
|
||||
print("-" * 60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
43
docker-compose.yml
Normal file
43
docker-compose.yml
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
container_name: oem_postgres
|
||||
environment:
|
||||
POSTGRES_USER: crawler_user
|
||||
POSTGRES_PASSWORD: crawler_password
|
||||
POSTGRES_DB: oem_stock_db
|
||||
ports:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
- ./schema.sql:/docker-entrypoint-initdb.d/init_schema.sql:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U crawler_user -d oem_stock_db"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
|
||||
crawler:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
container_name: oem_crawler
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
DATABASE_URL: postgresql+psycopg://crawler_user:crawler_password@postgres:5432/oem_stock_db
|
||||
DEFAULT_MARKET: DE
|
||||
LOG_LEVEL: INFO
|
||||
DEFAULT_DELAY_SECONDS: 1.0
|
||||
ENABLE_RAW_EXTRACTION: "true"
|
||||
RAW_STORAGE_DIR: "/data/raw"
|
||||
volumes:
|
||||
- ./data/raw:/data/raw
|
||||
command: ["--brand", "bmw", "--limit", "50"]
|
||||
restart: "no"
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
driver: local
|
||||
3
human_backlog.md
Normal file
3
human_backlog.md
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
- [x] store raw data
|
||||
- [ ] b2b
|
||||
- [ ] alerting
|
||||
29
pyproject.toml
Normal file
29
pyproject.toml
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
[project]
|
||||
name = "oem-crawler"
|
||||
version = "0.1.0"
|
||||
description = "Automotive OEM stock inventory crawler and data pipeline"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"pydantic>=2.7.0",
|
||||
"pydantic-settings>=2.2.0",
|
||||
"sqlalchemy>=2.0.30",
|
||||
"psycopg[binary]>=3.1.18",
|
||||
"httpx[http2]>=0.27.0",
|
||||
"playwright>=1.64.1",
|
||||
"beautifulsoup4>=4.15.0",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
oem-crawler = "oem_crawler.cli:main"
|
||||
oem-inspect = "oem_crawler.tools.view_stock:main"
|
||||
oem-restore = "oem_crawler.tools.restore_from_raw:main"
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=8.0.0",
|
||||
]
|
||||
217
schema.sql
Normal file
217
schema.sql
Normal file
|
|
@ -0,0 +1,217 @@
|
|||
-- =============================================================================
|
||||
-- OEM Vehicle Stock Crawler - PostgreSQL Database Schema
|
||||
-- =============================================================================
|
||||
|
||||
-- Enable UUID extension if not already available
|
||||
CREATE EXTENSION IF NOT EXISTS "pgcrypto";
|
||||
|
||||
-- Function to automatically update 'updated_at' column
|
||||
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = NOW();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ language 'plpgsql';
|
||||
|
||||
-- =============================================================================
|
||||
-- 1. Table: crawl_runs (Execution telemetry & audit log)
|
||||
-- =============================================================================
|
||||
CREATE TABLE IF NOT EXISTS crawl_runs (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
brand VARCHAR(50) NOT NULL,
|
||||
market_country VARCHAR(2) NOT NULL DEFAULT 'DE',
|
||||
crawler_name VARCHAR(100) NOT NULL,
|
||||
crawler_version VARCHAR(20),
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'PENDING'
|
||||
CHECK (status IN ('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', 'PARTIAL')),
|
||||
started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
finished_at TIMESTAMPTZ,
|
||||
duration_ms INTEGER,
|
||||
items_discovered INTEGER NOT NULL DEFAULT 0,
|
||||
items_inserted INTEGER NOT NULL DEFAULT 0,
|
||||
items_updated INTEGER NOT NULL DEFAULT 0,
|
||||
items_unmodified INTEGER NOT NULL DEFAULT 0,
|
||||
items_delisted INTEGER NOT NULL DEFAULT 0,
|
||||
items_failed INTEGER NOT NULL DEFAULT 0,
|
||||
error_summary TEXT,
|
||||
config_snapshot JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_crawl_runs_brand_status
|
||||
ON crawl_runs(brand, status, started_at DESC);
|
||||
|
||||
-- =============================================================================
|
||||
-- 2. Table: vehicles (Master Vehicle Stock Inventory)
|
||||
-- =============================================================================
|
||||
CREATE TABLE IF NOT EXISTS vehicles (
|
||||
-- Identifiers
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
brand VARCHAR(50) NOT NULL,
|
||||
oem_vehicle_id VARCHAR(100) NOT NULL,
|
||||
vin VARCHAR(17),
|
||||
|
||||
-- Model & Classification
|
||||
series VARCHAR(100) NOT NULL,
|
||||
model_name VARCHAR(255) NOT NULL,
|
||||
trim_line VARCHAR(100),
|
||||
body_type VARCHAR(50),
|
||||
condition VARCHAR(30) NOT NULL DEFAULT 'new'
|
||||
CHECK (condition IN ('new', 'used', 'demonstration', 'day_registration', 'certified_pre_owned')),
|
||||
model_year SMALLINT,
|
||||
first_registration_date DATE,
|
||||
previous_owners_count SMALLINT,
|
||||
|
||||
-- Powertrain & Technical Specifications
|
||||
fuel_type VARCHAR(30) NOT NULL DEFAULT 'petrol'
|
||||
CHECK (fuel_type IN ('bev', 'phev', 'petrol', 'diesel', 'mhev', 'fcev', 'cng', 'lpg', 'hybrid_other')),
|
||||
transmission VARCHAR(30)
|
||||
CHECK (transmission IN ('automatic', 'manual', 'direct_drive', 'dual_clutch', 'cvt', 'other')),
|
||||
drivetrain VARCHAR(20)
|
||||
CHECK (drivetrain IN ('awd', 'rwd', 'fwd', '4x4', 'other')),
|
||||
power_kw INTEGER,
|
||||
power_hp INTEGER,
|
||||
battery_capacity NUMERIC(6,2),
|
||||
battery_capacity_unit VARCHAR(10)
|
||||
CHECK (battery_capacity_unit IS NULL OR battery_capacity_unit IN ('kwh', 'ah')),
|
||||
battery_capacity_gross_kwh NUMERIC(6,2),
|
||||
battery_capacity_net_kwh NUMERIC(6,2),
|
||||
charging_power_ac_kw NUMERIC(5,2),
|
||||
charging_power_dc_kw NUMERIC(5,2),
|
||||
charging_time_dc_minutes SMALLINT,
|
||||
electric_range_wltp_km INTEGER,
|
||||
consumption_combined NUMERIC(5,2),
|
||||
co2_emissions_g_km NUMERIC(5,1),
|
||||
mileage_km INTEGER NOT NULL DEFAULT 0,
|
||||
|
||||
-- Colors & Upholstery
|
||||
exterior_color VARCHAR(100),
|
||||
exterior_color_code VARCHAR(50),
|
||||
exterior_color_generic VARCHAR(50),
|
||||
interior_color VARCHAR(100),
|
||||
interior_material VARCHAR(50),
|
||||
|
||||
-- Pricing, Financing & Availability
|
||||
currency VARCHAR(3) NOT NULL DEFAULT 'EUR',
|
||||
base_price_gross NUMERIC(12,2),
|
||||
options_price_gross NUMERIC(12,2),
|
||||
price_gross NUMERIC(12,2) NOT NULL,
|
||||
price_net NUMERIC(12,2),
|
||||
msrp_gross NUMERIC(12,2),
|
||||
discount_percentage NUMERIC(5,2),
|
||||
vat_deductible BOOLEAN,
|
||||
monthly_rate NUMERIC(10,2),
|
||||
leasing_details JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
status VARCHAR(30) NOT NULL DEFAULT 'in_stock'
|
||||
CHECK (status IN ('in_stock', 'in_transit', 'on_order', 'reserved', 'sold', 'delisted')),
|
||||
availability_date DATE,
|
||||
delivery_time_days INTEGER,
|
||||
|
||||
-- Location, Dealer & Media
|
||||
dealer_name VARCHAR(255),
|
||||
dealer_id VARCHAR(100),
|
||||
location_city VARCHAR(100),
|
||||
location_postal_code VARCHAR(20),
|
||||
location_country VARCHAR(2) NOT NULL DEFAULT 'DE',
|
||||
latitude NUMERIC(9,6),
|
||||
longitude NUMERIC(9,6),
|
||||
vehicle_url TEXT,
|
||||
image_urls JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
thumbnail_url TEXT,
|
||||
|
||||
-- Timestamps, Audit & Change Tracking
|
||||
first_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
last_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
delisted_at TIMESTAMPTZ,
|
||||
last_crawl_run_id UUID REFERENCES crawl_runs(id) ON DELETE SET NULL,
|
||||
content_hash VARCHAR(64) NOT NULL,
|
||||
raw_payload JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
raw_payload_path VARCHAR(512),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
|
||||
-- Compound uniqueness constraint to avoid duplicate vehicles per OEM per country
|
||||
CONSTRAINT uq_vehicles_brand_market_oem_id UNIQUE (brand, location_country, oem_vehicle_id)
|
||||
);
|
||||
|
||||
-- Triggers
|
||||
DROP TRIGGER IF EXISTS trg_vehicles_updated_at ON vehicles;
|
||||
CREATE TRIGGER trg_vehicles_updated_at
|
||||
BEFORE UPDATE ON vehicles
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
-- Indexes on vehicles
|
||||
CREATE INDEX IF NOT EXISTS idx_vehicles_brand_series
|
||||
ON vehicles(brand, series);
|
||||
CREATE INDEX IF NOT EXISTS idx_vehicles_fuel_transmission
|
||||
ON vehicles(fuel_type, transmission);
|
||||
CREATE INDEX IF NOT EXISTS idx_vehicles_price_gross
|
||||
ON vehicles(price_gross);
|
||||
CREATE INDEX IF NOT EXISTS idx_vehicles_status_last_seen
|
||||
ON vehicles(status, last_seen_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_vehicles_vin
|
||||
ON vehicles(vin) WHERE vin IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_vehicles_location
|
||||
ON vehicles(location_country, location_city);
|
||||
|
||||
-- =============================================================================
|
||||
-- 3. Table: vehicle_options (Normalized Equipment & Option Pricing)
|
||||
-- =============================================================================
|
||||
CREATE TABLE IF NOT EXISTS vehicle_options (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
vehicle_id UUID NOT NULL REFERENCES vehicles(id) ON DELETE CASCADE,
|
||||
option_code VARCHAR(50),
|
||||
name VARCHAR(255) NOT NULL,
|
||||
category VARCHAR(100),
|
||||
price_gross NUMERIC(10,2),
|
||||
price_net NUMERIC(10,2),
|
||||
is_standard BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
description TEXT,
|
||||
raw_data JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_vehicle_options_vehicle_id
|
||||
ON vehicle_options(vehicle_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_vehicle_options_code
|
||||
ON vehicle_options(option_code);
|
||||
CREATE INDEX IF NOT EXISTS idx_vehicle_options_category
|
||||
ON vehicle_options(category);
|
||||
|
||||
-- =============================================================================
|
||||
-- 4. Table: vehicle_price_history (Historical Price & Status Snapshots)
|
||||
-- =============================================================================
|
||||
CREATE TABLE IF NOT EXISTS vehicle_price_history (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
vehicle_id UUID NOT NULL REFERENCES vehicles(id) ON DELETE CASCADE,
|
||||
crawl_run_id UUID NOT NULL REFERENCES crawl_runs(id) ON DELETE CASCADE,
|
||||
price_gross NUMERIC(12,2) NOT NULL,
|
||||
price_net NUMERIC(12,2),
|
||||
monthly_rate NUMERIC(10,2),
|
||||
status VARCHAR(30) NOT NULL,
|
||||
recorded_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_price_history_vehicle_recorded
|
||||
ON vehicle_price_history(vehicle_id, recorded_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_price_history_crawl_run
|
||||
ON vehicle_price_history(crawl_run_id);
|
||||
|
||||
-- =============================================================================
|
||||
-- 5. Table: crawl_logs (Granular Diagnostic & Error Logging)
|
||||
-- =============================================================================
|
||||
CREATE TABLE IF NOT EXISTS crawl_logs (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
crawl_run_id UUID NOT NULL REFERENCES crawl_runs(id) ON DELETE CASCADE,
|
||||
vehicle_id UUID REFERENCES vehicles(id) ON DELETE SET NULL,
|
||||
level VARCHAR(10) NOT NULL CHECK (level IN ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL')),
|
||||
stage VARCHAR(50) NOT NULL CHECK (stage IN ('FETCH', 'PARSE', 'VALIDATE', 'PERSIST', 'DELIST_CHECK', 'GENERAL')),
|
||||
message TEXT NOT NULL,
|
||||
error_context JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_crawl_logs_run_level
|
||||
ON crawl_logs(crawl_run_id, level, created_at DESC);
|
||||
259
scripts/migrate_raw_payloads.py
Executable file
259
scripts/migrate_raw_payloads.py
Executable file
|
|
@ -0,0 +1,259 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Migration script to extract raw OEM payloads from SQLite into external compressed .json.gz files,
|
||||
update vehicles with raw_payload_path, clear in-database raw JSON, and compact the database via VACUUM.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
logger = logging.getLogger("migrate_raw_payloads")
|
||||
|
||||
|
||||
def format_bytes(num_bytes: int | float) -> str:
|
||||
"""Formats bytes into human-readable string (KB, MB, GB)."""
|
||||
for unit in ["B", "KB", "MB", "GB", "TB"]:
|
||||
if abs(num_bytes) < 1024.0:
|
||||
return f"{num_bytes:.2f} {unit}"
|
||||
num_bytes /= 1024.0
|
||||
return f"{num_bytes:.2f} PB"
|
||||
|
||||
|
||||
def sanitize_filename(name: str) -> str:
|
||||
"""Sanitizes an OEM vehicle id for filesystem safety."""
|
||||
return "".join(c if c.isalnum() or c in ("-", "_") else "_" for c in str(name))
|
||||
|
||||
|
||||
def migrate_database(
|
||||
db_path: Path,
|
||||
raw_dir: Path,
|
||||
compression_level: int = 6,
|
||||
batch_size: int = 500,
|
||||
dry_run: bool = False,
|
||||
skip_vacuum: bool = False,
|
||||
) -> None:
|
||||
if not db_path.exists():
|
||||
logger.error(f"Database file not found: {db_path}")
|
||||
sys.exit(1)
|
||||
|
||||
initial_db_size = db_path.stat().st_size
|
||||
logger.info(f"Target Database: {db_path} (Size: {format_bytes(initial_db_size)})")
|
||||
logger.info(f"Storage Directory: {raw_dir.resolve()}")
|
||||
logger.info(f"Dry Run Mode: {dry_run}")
|
||||
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
cursor = conn.cursor()
|
||||
|
||||
# 1. Ensure raw_payload_path column exists
|
||||
cursor.execute("PRAGMA table_info(vehicles);")
|
||||
columns = [row["name"] for row in cursor.fetchall()]
|
||||
if "raw_payload_path" not in columns:
|
||||
if dry_run:
|
||||
logger.info("[Dry Run] Would execute: ALTER TABLE vehicles ADD COLUMN raw_payload_path VARCHAR(512);")
|
||||
else:
|
||||
logger.info("Adding 'raw_payload_path' column to vehicles table...")
|
||||
cursor.execute("ALTER TABLE vehicles ADD COLUMN raw_payload_path VARCHAR(512);")
|
||||
conn.commit()
|
||||
|
||||
# 2. Count vehicles requiring migration
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*) as count
|
||||
FROM vehicles
|
||||
WHERE raw_payload IS NOT NULL AND length(raw_payload) > 2;
|
||||
""")
|
||||
total_vehicles = cursor.fetchone()["count"]
|
||||
logger.info(f"Found {total_vehicles:,} vehicles with raw payload to extract.")
|
||||
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*) as count
|
||||
FROM vehicle_options
|
||||
WHERE raw_data IS NOT NULL AND raw_data != '{}';
|
||||
""")
|
||||
total_options = cursor.fetchone()["count"]
|
||||
logger.info(f"Found {total_options:,} vehicle options with redundant raw_data.")
|
||||
|
||||
if total_vehicles == 0 and total_options == 0:
|
||||
logger.info("No vehicles or options require migration.")
|
||||
if not skip_vacuum and not dry_run:
|
||||
logger.info("Running VACUUM to ensure database is compact...")
|
||||
cursor.execute("VACUUM;")
|
||||
conn.close()
|
||||
return
|
||||
|
||||
if dry_run:
|
||||
logger.info("[Dry Run] Migration simulation complete. No changes written.")
|
||||
conn.close()
|
||||
return
|
||||
|
||||
# 3. Process vehicles in batches
|
||||
raw_dir.mkdir(parents=True, exist_ok=True)
|
||||
migrated_count = 0
|
||||
total_uncompressed_bytes = 0
|
||||
total_compressed_bytes = 0
|
||||
start_time = time.time()
|
||||
|
||||
cursor.execute("""
|
||||
SELECT id, brand, oem_vehicle_id, created_at, last_seen_at, raw_payload, raw_payload_path
|
||||
FROM vehicles
|
||||
WHERE raw_payload IS NOT NULL AND length(raw_payload) > 2;
|
||||
""")
|
||||
|
||||
updates_batch = []
|
||||
|
||||
while True:
|
||||
rows = cursor.fetchmany(batch_size)
|
||||
if not rows:
|
||||
break
|
||||
|
||||
for row in rows:
|
||||
v_id = row["id"]
|
||||
brand = (row["brand"] or "unknown").strip().lower()
|
||||
oem_id = row["oem_vehicle_id"] or v_id
|
||||
created_at = str(row["last_seen_at"] or row["created_at"] or "")
|
||||
date_str = created_at[:10] if len(created_at) >= 10 else "archive"
|
||||
|
||||
raw_str = row["raw_payload"]
|
||||
if not raw_str or raw_str == "{}":
|
||||
continue
|
||||
|
||||
try:
|
||||
payload = json.loads(raw_str) if isinstance(raw_str, str) else raw_str
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not parse JSON for vehicle {v_id}: {e}")
|
||||
continue
|
||||
|
||||
# Compute relative path
|
||||
safe_id = sanitize_filename(oem_id)
|
||||
rel_path = f"{brand}/{date_str}/{safe_id}.json.gz"
|
||||
dest_file = raw_dir / rel_path
|
||||
dest_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Compress and write
|
||||
json_bytes = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
compressed_data = gzip.compress(json_bytes, compresslevel=compression_level)
|
||||
|
||||
with open(dest_file, "wb") as f:
|
||||
f.write(compressed_data)
|
||||
|
||||
total_uncompressed_bytes += len(json_bytes)
|
||||
total_compressed_bytes += len(compressed_data)
|
||||
migrated_count += 1
|
||||
|
||||
updates_batch.append((rel_path, v_id))
|
||||
|
||||
# Commit DB updates for batch
|
||||
if updates_batch:
|
||||
update_cursor = conn.cursor()
|
||||
update_cursor.executemany(
|
||||
"UPDATE vehicles SET raw_payload_path = ?, raw_payload = '{}' WHERE id = ?;",
|
||||
updates_batch,
|
||||
)
|
||||
conn.commit()
|
||||
updates_batch.clear()
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
rate = migrated_count / elapsed if elapsed > 0 else 0
|
||||
logger.info(
|
||||
f"Progress: {migrated_count:,}/{total_vehicles:,} vehicles ({migrated_count/total_vehicles*100:.1f}%) "
|
||||
f"| Rate: {rate:.0f} veh/sec"
|
||||
)
|
||||
|
||||
# 4. Clean vehicle_options raw_data
|
||||
logger.info("Clearing redundant raw_data in vehicle_options...")
|
||||
cursor.execute("UPDATE vehicle_options SET raw_data = '{}' WHERE raw_data != '{}';")
|
||||
conn.commit()
|
||||
|
||||
# 5. Compact the database with VACUUM
|
||||
if not skip_vacuum:
|
||||
logger.info("Running VACUUM to reclaim free space (this may take a few moments)...")
|
||||
vac_start = time.time()
|
||||
cursor.execute("VACUUM;")
|
||||
logger.info(f"VACUUM completed in {time.time() - vac_start:.1f}s.")
|
||||
|
||||
conn.close()
|
||||
|
||||
final_db_size = db_path.stat().st_size
|
||||
reclaimed_bytes = initial_db_size - final_db_size
|
||||
|
||||
# Summary
|
||||
logger.info("\n" + "=" * 60)
|
||||
logger.info("MIGRATION COMPLETED SUCCESSFULLY")
|
||||
logger.info("=" * 60)
|
||||
logger.info(f"Vehicles Migrated: {migrated_count:,}")
|
||||
logger.info(f"Options Cleaned: {total_options:,}")
|
||||
logger.info(f"Uncompressed Payload Size:{format_bytes(total_uncompressed_bytes)}")
|
||||
logger.info(f"Compressed File Size: {format_bytes(total_compressed_bytes)} "
|
||||
f"({total_compressed_bytes / max(total_uncompressed_bytes, 1) * 100:.1f}%)")
|
||||
logger.info(f"Original DB Size: {format_bytes(initial_db_size)}")
|
||||
logger.info(f"New DB Size: {format_bytes(final_db_size)}")
|
||||
logger.info(f"Space Reclaimed: {format_bytes(reclaimed_bytes)} "
|
||||
f"({reclaimed_bytes / max(initial_db_size, 1) * 100:.1f}% reduction)")
|
||||
logger.info("=" * 60)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Extract raw JSON payloads from SQLite to external compressed files")
|
||||
parser.add_argument(
|
||||
"--db-path",
|
||||
type=Path,
|
||||
default=Path("data/crawler_stock.db"),
|
||||
help="Path to SQLite database file (default: data/crawler_stock.db)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--raw-dir",
|
||||
type=Path,
|
||||
default=Path("data/raw"),
|
||||
help="Destination directory for compressed JSON files (default: data/raw)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--compression-level",
|
||||
type=int,
|
||||
default=6,
|
||||
help="Gzip compression level (1-9, default: 6)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--batch-size",
|
||||
type=int,
|
||||
default=500,
|
||||
help="Batch size for processing and DB updates (default: 500)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Inspect database without writing files or modifying database",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-vacuum",
|
||||
action="store_true",
|
||||
help="Skip database VACUUM at end of migration",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
migrate_database(
|
||||
db_path=args.db_path,
|
||||
raw_dir=args.raw_dir,
|
||||
compression_level=args.compression_level,
|
||||
batch_size=args.batch_size,
|
||||
dry_run=args.dry_run,
|
||||
skip_vacuum=args.skip_vacuum,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
18
scripts/restore_from_raw.py
Executable file
18
scripts/restore_from_raw.py
Executable file
|
|
@ -0,0 +1,18 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
CLI launcher for the restore_from_raw tool.
|
||||
"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add src to sys.path
|
||||
root_dir = Path(__file__).resolve().parent.parent
|
||||
src_dir = root_dir / "src"
|
||||
if str(src_dir) not in sys.path:
|
||||
sys.path.insert(0, str(src_dir))
|
||||
|
||||
from oem_crawler.tools.restore_from_raw import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
4
src/oem_crawler/__init__.py
Normal file
4
src/oem_crawler/__init__.py
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
"""
|
||||
OEM Vehicle Stock Crawler Package
|
||||
"""
|
||||
__version__ = "0.1.0"
|
||||
246
src/oem_crawler/cli.py
Normal file
246
src/oem_crawler/cli.py
Normal file
|
|
@ -0,0 +1,246 @@
|
|||
"""
|
||||
OEM Stock Crawler Command-Line Interface (CLI).
|
||||
Supports crawling individual brands or all registered brands in a single command.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from typing import List
|
||||
|
||||
from oem_crawler.config import settings
|
||||
from oem_crawler.crawlers import CrawlerRegistry
|
||||
from oem_crawler.db.pipeline import StockIngestionPipeline
|
||||
|
||||
logging.basicConfig(
|
||||
level=getattr(logging, settings.LOG_LEVEL.upper(), logging.INFO),
|
||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s"
|
||||
)
|
||||
logger = logging.getLogger("oem_crawler")
|
||||
|
||||
|
||||
def main():
|
||||
supported_aliases = CrawlerRegistry.supported_brands()
|
||||
unique_brands = [brand for brand, _ in CrawlerRegistry.unique_brand_crawlers()]
|
||||
|
||||
parser = argparse.ArgumentParser(description="OEM Vehicle Stock Crawler & Ingestion Pipeline")
|
||||
parser.add_argument(
|
||||
"--brand",
|
||||
type=str,
|
||||
default="all",
|
||||
help=f"Car manufacturer to crawl (Supported: {', '.join(unique_brands)}, or 'all' to crawl all brands; comma-separated: 'bmw,audi')",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--all-brands",
|
||||
action="store_true",
|
||||
help="Shortcut to run crawl across all registered OEM brands",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--country",
|
||||
type=str,
|
||||
default=settings.DEFAULT_MARKET,
|
||||
help=f"ISO market country code (default: {settings.DEFAULT_MARKET})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stock-type",
|
||||
type=str,
|
||||
default="new",
|
||||
choices=["new", "used", "all"],
|
||||
help="Vehicle stock condition category: 'new', 'used', or 'all' (default: new)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=50,
|
||||
help="Maximum number of vehicles to fetch per brand (default: 50, use 0 or --full-scan for unlimited)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--full-scan",
|
||||
action="store_true",
|
||||
help="Perform a complete market scan without limit (enables automatic delisting detection in DB)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--page-size",
|
||||
type=int,
|
||||
default=settings.DEFAULT_PAGE_SIZE,
|
||||
help=f"Page size for REST API queries (default: {settings.DEFAULT_PAGE_SIZE})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--delay",
|
||||
type=float,
|
||||
default=settings.DEFAULT_DELAY_SECONDS,
|
||||
help=f"Polite delay between page requests in seconds (default: {settings.DEFAULT_DELAY_SECONDS}s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--proxy",
|
||||
type=str,
|
||||
default=settings.DEFAULT_PROXY,
|
||||
help="HTTP/SOCKS5 proxy URL (e.g. http://user:pass@proxy.com:8080)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--db-url",
|
||||
type=str,
|
||||
default=settings.DATABASE_URL,
|
||||
help="SQLAlchemy Database URL (e.g. postgresql+psycopg://user:pass@localhost:5432/stock_db)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Print/dump parsed vehicle JSON without saving to database",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-json",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Save parsed vehicles directly to a local JSON file",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--raw",
|
||||
dest="enable_raw",
|
||||
action="store_true",
|
||||
default=None,
|
||||
help="Force enable raw payload extraction and external compressed storage",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-raw",
|
||||
dest="enable_raw",
|
||||
action="store_false",
|
||||
default=None,
|
||||
help="Force disable raw payload extraction and storage",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.enable_raw is not None:
|
||||
settings.ENABLE_RAW_EXTRACTION = args.enable_raw
|
||||
|
||||
# 1. Resolve brands to crawl
|
||||
if args.all_brands or args.brand.strip().lower() in ("all", "*"):
|
||||
brands_to_run = [brand for brand, _ in CrawlerRegistry.unique_brand_crawlers()]
|
||||
else:
|
||||
raw_brands = [b.strip().lower() for b in args.brand.split(",") if b.strip()]
|
||||
for b in raw_brands:
|
||||
if b not in supported_aliases:
|
||||
parser.error(f"Unknown brand '{b}'. Supported brands: {', '.join(unique_brands)}, or 'all'")
|
||||
# Deduplicate while preserving order and resolving aliases to unique crawler classes
|
||||
seen_classes = set()
|
||||
brands_to_run = []
|
||||
for b in raw_brands:
|
||||
cls_ = CrawlerRegistry.get(b)
|
||||
if cls_ not in seen_classes:
|
||||
seen_classes.add(cls_)
|
||||
brands_to_run.append(b)
|
||||
|
||||
is_full_scan = args.full_scan or (args.limit is not None and args.limit <= 0)
|
||||
max_vehicles = None if is_full_scan else args.limit
|
||||
|
||||
scan_label = "FULL SCAN" if is_full_scan else f"limit={max_vehicles}"
|
||||
brand_label = ", ".join(b.upper() for b in brands_to_run)
|
||||
logger.info(
|
||||
f"Target brands: [{brand_label}] | Market: {args.country} | Mode: {scan_label} | Stock: {args.stock_type} | Raw Extraction: {settings.ENABLE_RAW_EXTRACTION}"
|
||||
)
|
||||
|
||||
# 2. Execution loop
|
||||
pipeline = StockIngestionPipeline(db_url=args.db_url) if not (args.dry_run or args.output_json) else None
|
||||
results_summary = []
|
||||
all_output_data = {}
|
||||
|
||||
for brand in brands_to_run:
|
||||
logger.info(f"\n{'='*70}\nStarting {brand.upper()} crawl ({scan_label}, stock_type: {args.stock_type})...\n{'='*70}")
|
||||
crawler_cls = CrawlerRegistry.get(brand)
|
||||
crawler = crawler_cls(
|
||||
market_country=args.country,
|
||||
proxy=args.proxy,
|
||||
request_delay=args.delay,
|
||||
timeout=settings.HTTP_TIMEOUT_SECONDS,
|
||||
stock_type=args.stock_type,
|
||||
)
|
||||
|
||||
stream = crawler.crawl(max_vehicles=max_vehicles, page_size=args.page_size)
|
||||
|
||||
if args.dry_run or args.output_json:
|
||||
vehicles_list = [v.model_dump(mode="json") for v in stream]
|
||||
logger.info(f"[{brand.upper()}] Fetched and parsed {len(vehicles_list)} vehicles successfully.")
|
||||
all_output_data[brand] = vehicles_list
|
||||
results_summary.append({
|
||||
"brand": brand.upper(),
|
||||
"status": "DRY_RUN",
|
||||
"discovered": len(vehicles_list),
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"unmodified": 0,
|
||||
"delisted": 0,
|
||||
"failed": 0,
|
||||
"duration_s": 0.0,
|
||||
})
|
||||
continue
|
||||
|
||||
try:
|
||||
crawl_run = pipeline.run_crawler_pipeline(
|
||||
brand=brand.upper(),
|
||||
crawler_name=f"{brand.lower()}-rest-crawler",
|
||||
vehicle_stream=stream,
|
||||
market_country=args.country,
|
||||
config_snapshot={
|
||||
"limit": max_vehicles,
|
||||
"page_size": args.page_size,
|
||||
"stock_type": args.stock_type,
|
||||
"is_full_market_scan": is_full_scan,
|
||||
}
|
||||
)
|
||||
results_summary.append({
|
||||
"brand": brand.upper(),
|
||||
"status": crawl_run.status,
|
||||
"discovered": crawl_run.items_discovered,
|
||||
"inserted": crawl_run.items_inserted,
|
||||
"updated": crawl_run.items_updated,
|
||||
"unmodified": crawl_run.items_unmodified,
|
||||
"delisted": crawl_run.items_delisted,
|
||||
"failed": crawl_run.items_failed,
|
||||
"duration_s": round(crawl_run.duration_ms / 1000.0, 1),
|
||||
})
|
||||
except Exception as e:
|
||||
logger.error(f"[{brand.upper()}] Crawl run failed: {e}", exc_info=True)
|
||||
results_summary.append({
|
||||
"brand": brand.upper(),
|
||||
"status": "FAILED",
|
||||
"discovered": crawl_run.items_discovered if crawl_run else 0,
|
||||
"inserted": crawl_run.items_inserted if crawl_run else 0,
|
||||
"updated": crawl_run.items_updated if crawl_run else 0,
|
||||
"unmodified": crawl_run.items_unmodified if crawl_run else 0,
|
||||
"delisted": crawl_run.items_delisted if crawl_run else 0,
|
||||
"failed": 1,
|
||||
"duration_s": round((crawl_run.duration_ms or 0) / 1000.0, 1) if crawl_run else 0.0,
|
||||
})
|
||||
|
||||
# 3. Optional JSON Export
|
||||
if args.output_json:
|
||||
payload_to_write = all_output_data if len(brands_to_run) > 1 else list(all_output_data.values())[0]
|
||||
with open(args.output_json, "w", encoding="utf-8") as f:
|
||||
json.dump(payload_to_write, f, indent=2, ensure_ascii=False)
|
||||
logger.info(f"Saved vehicle JSON dataset to {args.output_json}")
|
||||
elif args.dry_run and all_output_data:
|
||||
first_brand = list(all_output_data.keys())[0]
|
||||
print(json.dumps(all_output_data[first_brand][:2], indent=2, ensure_ascii=False))
|
||||
|
||||
# 4. Final Aggregated Report
|
||||
print("\n" + "=" * 95)
|
||||
print("📊 MULTI-BRAND CRAWL SUMMARY REPORT")
|
||||
print("=" * 95)
|
||||
print(f"{'Brand':<15} {'Status':<12} {'Discovered':<12} {'Inserted':<10} {'Updated':<10} {'Unchanged':<10} {'Delisted':<10} {'Duration':<10}")
|
||||
print("-" * 95)
|
||||
total_disc = sum(r['discovered'] for r in results_summary)
|
||||
total_ins = sum(r['inserted'] for r in results_summary)
|
||||
total_upd = sum(r['updated'] for r in results_summary)
|
||||
total_unm = sum(r['unmodified'] for r in results_summary)
|
||||
total_del = sum(r['delisted'] for r in results_summary)
|
||||
total_time = sum(r['duration_s'] for r in results_summary)
|
||||
|
||||
for r in results_summary:
|
||||
print(f"{r['brand']:<15} {r['status']:<12} {r['discovered']:<12} {r['inserted']:<10} {r['updated']:<10} {r['unmodified']:<10} {r['delisted']:<10} {r['duration_s']}s")
|
||||
print("=" * 95)
|
||||
print(f"Total Discovered: {total_disc:,} | Total Inserted: {total_ins:,} | Updated: {total_upd:,} | Delisted: {total_del:,} | Elapsed: {total_time:.1f}s\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
43
src/oem_crawler/config.py
Normal file
43
src/oem_crawler/config.py
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
"""
|
||||
Configuration settings for OEM Crawler (loaded from environment or .env).
|
||||
"""
|
||||
from typing import Optional
|
||||
from pydantic import Field, AliasChoices
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
|
||||
|
||||
# Database
|
||||
DATABASE_URL: str = "sqlite:///data/crawler_stock.db"
|
||||
DB_ECHO_SQL: bool = False
|
||||
|
||||
# Crawler Settings
|
||||
DEFAULT_MARKET: str = "DE"
|
||||
DEFAULT_PAGE_SIZE: int = 12
|
||||
DEFAULT_DELAY_SECONDS: float = 0.8
|
||||
HTTP_TIMEOUT_SECONDS: float = 25.0
|
||||
DEFAULT_PROXY: Optional[str] = None
|
||||
|
||||
# Raw Payload Storage & Extraction
|
||||
ENABLE_RAW_EXTRACTION: bool = Field(
|
||||
default=True,
|
||||
validation_alias=AliasChoices("ENABLE_RAW_EXTRACTION", "SAVE_RAW_PAYLOAD", "EXTRACT_RAW_DATA"),
|
||||
description="Global switch to enable/disable extraction and storage of raw OEM JSON payloads.",
|
||||
)
|
||||
RAW_STORAGE_DIR: str = Field(
|
||||
default="data/raw",
|
||||
validation_alias=AliasChoices("RAW_STORAGE_DIR", "RAW_DATA_DIR", "RAW_PAYLOAD_DIR"),
|
||||
description="Directory where compressed raw payloads are stored (relative for dev, mounted host dir for docker).",
|
||||
)
|
||||
RAW_COMPRESSION_LEVEL: int = Field(
|
||||
default=6,
|
||||
description="Gzip compression level from 1 (fastest) to 9 (best compression).",
|
||||
)
|
||||
|
||||
# Logging
|
||||
LOG_LEVEL: str = "INFO"
|
||||
|
||||
|
||||
settings = Settings()
|
||||
11
src/oem_crawler/crawlers/__init__.py
Normal file
11
src/oem_crawler/crawlers/__init__.py
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
"""
|
||||
Crawler module exports and automatic registry initialization.
|
||||
"""
|
||||
from oem_crawler.crawlers.base import BaseCrawler, CrawlerRegistry
|
||||
# Import all crawler subpackages to trigger @CrawlerRegistry.register
|
||||
import oem_crawler.crawlers.audi # noqa: F401
|
||||
import oem_crawler.crawlers.bmw # noqa: F401
|
||||
import oem_crawler.crawlers.mercedes # noqa: F401
|
||||
import oem_crawler.crawlers.porsche # noqa: F401
|
||||
|
||||
__all__ = ["BaseCrawler", "CrawlerRegistry"]
|
||||
7
src/oem_crawler/crawlers/audi/__init__.py
Normal file
7
src/oem_crawler/crawlers/audi/__init__.py
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
"""
|
||||
Audi Crawler Package Exports.
|
||||
"""
|
||||
from oem_crawler.crawlers.audi.crawler import AudiStockCrawler
|
||||
from oem_crawler.crawlers.audi.parser import parse_audi_vehicle
|
||||
|
||||
__all__ = ["AudiStockCrawler", "parse_audi_vehicle"]
|
||||
263
src/oem_crawler/crawlers/audi/crawler.py
Normal file
263
src/oem_crawler/crawlers/audi/crawler.py
Normal file
|
|
@ -0,0 +1,263 @@
|
|||
"""
|
||||
Audi Stock Car Service (SCS) Crawler Implementation.
|
||||
Interacts with https://scs.audi.de REST API endpoints.
|
||||
Supports both new and used vehicles from the unified SCS backend.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Any, Dict, Generator, List, Optional, Tuple
|
||||
|
||||
import httpx
|
||||
|
||||
from oem_crawler.crawlers.audi.parser import parse_audi_vehicle
|
||||
from oem_crawler.crawlers.base import BaseCrawler, CrawlerRegistry
|
||||
from oem_crawler.db.schemas import VehicleIngestSchema
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@CrawlerRegistry.register("audi")
|
||||
class AudiStockCrawler(BaseCrawler):
|
||||
BASE_URL = "https://scs.audi.de"
|
||||
DEFAULT_TOKEN = "FJ54W6H"
|
||||
DEFAULT_USER_AGENT = (
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
market_country: str = "DE",
|
||||
proxy: Optional[str] = None,
|
||||
request_delay: float = 0.5,
|
||||
timeout: float = 25.0,
|
||||
stock_type: str = "new",
|
||||
fetch_details: bool = True,
|
||||
sort: str = "dateOffer:desc",
|
||||
token: Optional[str] = None,
|
||||
**kwargs: Any,
|
||||
):
|
||||
super().__init__(
|
||||
market_country=market_country,
|
||||
proxy=proxy,
|
||||
request_delay=request_delay,
|
||||
timeout=timeout,
|
||||
**kwargs,
|
||||
)
|
||||
self.stock_type = stock_type.lower()
|
||||
self.fetch_details = fetch_details
|
||||
self.sort = sort
|
||||
self.token = token or self.DEFAULT_TOKEN
|
||||
|
||||
self.headers = {
|
||||
"token": self.token,
|
||||
"User-Agent": self.DEFAULT_USER_AGENT,
|
||||
"Accept": "application/json",
|
||||
"Accept-Language": "de-DE,de;q=0.9,en;q=0.8",
|
||||
}
|
||||
|
||||
@property
|
||||
def brand_name(self) -> str:
|
||||
return "AUDI"
|
||||
|
||||
def get_market_code(self, condition: str) -> str:
|
||||
"""
|
||||
Maps stock condition to Audi SCS market code.
|
||||
New cars: 'de'
|
||||
Used cars: 'deuc'
|
||||
"""
|
||||
if condition.lower() == "used":
|
||||
return f"{self.market_country.lower()}uc"
|
||||
return self.market_country.lower()
|
||||
|
||||
def fetch_search_page(
|
||||
self,
|
||||
client: httpx.Client,
|
||||
market_code: str,
|
||||
page: int = 0,
|
||||
size: int = 50,
|
||||
search_filter: Optional[Dict[str, Any]] = None,
|
||||
max_retries: int = 3,
|
||||
) -> Tuple[int, List[Dict[str, Any]]]:
|
||||
"""
|
||||
Fetches one page of search results from Audi Stock Car Service.
|
||||
Returns (totalCount, vehicleBasic list).
|
||||
"""
|
||||
lang = self.market_country.lower()
|
||||
url = f"{self.BASE_URL}/api/v2/search/filter/{market_code}/{lang}"
|
||||
params: Dict[str, Any] = {
|
||||
"from": page * size,
|
||||
"size": size,
|
||||
"sort": self.sort,
|
||||
}
|
||||
if search_filter:
|
||||
params.update(search_filter)
|
||||
|
||||
for attempt in range(1, max_retries + 1):
|
||||
try:
|
||||
time.sleep(self.request_delay)
|
||||
resp = client.get(url, params=params, headers=self.headers)
|
||||
|
||||
if resp.status_code == 429:
|
||||
wait_s = 2.0 * attempt
|
||||
logger.warning(f"Audi rate limit (HTTP 429). Retrying in {wait_s}s...")
|
||||
time.sleep(wait_s)
|
||||
continue
|
||||
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
total_count = data.get("totalCount", 0)
|
||||
items = data.get("vehicleBasic", [])
|
||||
return total_count, items
|
||||
|
||||
except httpx.HTTPError as e:
|
||||
logger.warning(f"Attempt {attempt}/{max_retries} failed for Audi market {market_code} page {page}: {e}")
|
||||
if attempt == max_retries:
|
||||
raise
|
||||
time.sleep(attempt * 1.5)
|
||||
|
||||
return 0, []
|
||||
|
||||
def fetch_detail(
|
||||
self,
|
||||
client: httpx.Client,
|
||||
market_code: str,
|
||||
vehicle_id: str,
|
||||
max_retries: int = 2,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Fetches full vehicle detail (features, PR equipment, techData) for a vehicle.
|
||||
"""
|
||||
lang = self.market_country.lower()
|
||||
url = f"{self.BASE_URL}/api/v1/vehicles/vehicle/{market_code}/{lang}/{vehicle_id}"
|
||||
|
||||
for attempt in range(1, max_retries + 1):
|
||||
try:
|
||||
time.sleep(self.request_delay)
|
||||
resp = client.get(url, headers=self.headers)
|
||||
if resp.status_code == 200:
|
||||
return resp.json()
|
||||
elif resp.status_code == 404:
|
||||
return None
|
||||
elif resp.status_code == 429:
|
||||
time.sleep(2.0 * attempt)
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
except Exception as e:
|
||||
logger.warning(f"Attempt {attempt}/{max_retries} failed for Audi detail {vehicle_id}: {e}")
|
||||
if attempt == max_retries:
|
||||
return None
|
||||
time.sleep(1.0)
|
||||
|
||||
return None
|
||||
|
||||
def _crawl_condition(
|
||||
self,
|
||||
client: httpx.Client,
|
||||
condition: str,
|
||||
max_vehicles: Optional[int],
|
||||
page_size: int,
|
||||
search_filter: Optional[Dict[str, Any]],
|
||||
) -> Generator[VehicleIngestSchema, None, None]:
|
||||
"""
|
||||
Streams vehicles for a specific condition ('new' or 'used').
|
||||
"""
|
||||
market_code = self.get_market_code(condition)
|
||||
count = 0
|
||||
current_page = 0
|
||||
total_count = None
|
||||
seen_page_ids: set[str] = set()
|
||||
|
||||
logger.info(
|
||||
f"Starting Audi {condition.upper()} stock crawl (Market: {market_code}, Lang: {self.market_country.lower()})..."
|
||||
)
|
||||
|
||||
while True:
|
||||
total_items, items = self.fetch_search_page(
|
||||
client=client,
|
||||
market_code=market_code,
|
||||
page=current_page,
|
||||
size=page_size,
|
||||
search_filter=search_filter,
|
||||
)
|
||||
|
||||
if total_count is None:
|
||||
total_count = total_items
|
||||
total_pages = (total_count + page_size - 1) // page_size if total_count else 0
|
||||
logger.info(f"Audi {condition} reports {total_count} vehicles available across ~{total_pages} pages.")
|
||||
|
||||
if not items:
|
||||
logger.info(f"No more items returned on page {current_page}. Finishing {condition} crawl.")
|
||||
break
|
||||
|
||||
# Fail-safe against infinite pagination loop
|
||||
page_ids = {str(item.get("id")) for item in items if item.get("id")}
|
||||
new_ids = page_ids - seen_page_ids
|
||||
if not new_ids and items:
|
||||
logger.warning(f"Audi {condition} page {current_page} returned duplicate IDs. Terminating pagination.")
|
||||
break
|
||||
seen_page_ids.update(page_ids)
|
||||
|
||||
logger.info(f"Audi {condition}: processing page {current_page} ({len(items)} items, yielded so far: {count}).")
|
||||
|
||||
for item in items:
|
||||
vid = item.get("id")
|
||||
detail_data = None
|
||||
if self.fetch_details and vid:
|
||||
detail_data = self.fetch_detail(client, market_code, vid)
|
||||
|
||||
try:
|
||||
vehicle = parse_audi_vehicle(
|
||||
basic_item=item,
|
||||
detail_data=detail_data,
|
||||
market_country=self.market_country,
|
||||
requested_stock_type=condition,
|
||||
)
|
||||
yield vehicle
|
||||
count += 1
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse Audi vehicle {item.get('carId')}: {e}", exc_info=True)
|
||||
|
||||
if max_vehicles and count >= max_vehicles:
|
||||
logger.info(f"Reached limit of {max_vehicles} {condition} vehicles.")
|
||||
return
|
||||
|
||||
current_page += 1
|
||||
if total_count is not None and (current_page * page_size) >= total_count:
|
||||
break
|
||||
|
||||
def crawl(
|
||||
self,
|
||||
max_vehicles: Optional[int] = None,
|
||||
page_size: int = 50,
|
||||
search_filter: Optional[Dict[str, Any]] = None,
|
||||
) -> Generator[VehicleIngestSchema, None, None]:
|
||||
"""
|
||||
Main crawling generator yielding normalized VehicleIngestSchema objects.
|
||||
Supports stock_type in ('new', 'used', 'all').
|
||||
"""
|
||||
page_size = max(1, min(page_size, 100))
|
||||
client_kwargs: Dict[str, Any] = {
|
||||
"timeout": self.timeout,
|
||||
"http2": False,
|
||||
}
|
||||
if self.proxy:
|
||||
client_kwargs["proxy"] = self.proxy
|
||||
|
||||
with httpx.Client(**client_kwargs) as client:
|
||||
if self.stock_type == "all":
|
||||
half_limit = (max_vehicles // 2) if max_vehicles else None
|
||||
new_count = 0
|
||||
for veh in self._crawl_condition(client, "new", half_limit, page_size, search_filter):
|
||||
yield veh
|
||||
new_count += 1
|
||||
|
||||
rem_limit = (max_vehicles - new_count) if max_vehicles else None
|
||||
for veh in self._crawl_condition(client, "used", rem_limit, page_size, search_filter):
|
||||
yield veh
|
||||
elif self.stock_type == "used":
|
||||
yield from self._crawl_condition(client, "used", max_vehicles, page_size, search_filter)
|
||||
else:
|
||||
yield from self._crawl_condition(client, "new", max_vehicles, page_size, search_filter)
|
||||
512
src/oem_crawler/crawlers/audi/parser.py
Normal file
512
src/oem_crawler/crawlers/audi/parser.py
Normal file
|
|
@ -0,0 +1,512 @@
|
|||
"""
|
||||
Parser and normalization logic for Audi Stock Car Service (SCS) responses.
|
||||
Handles search vehicleBasic payloads and detail payloads.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from oem_crawler.config import settings
|
||||
from oem_crawler.db.schemas import (
|
||||
BatteryCapacityUnit,
|
||||
DrivetrainType,
|
||||
FuelType,
|
||||
OptionSchema,
|
||||
TransmissionType,
|
||||
VehicleCondition,
|
||||
VehicleIngestSchema,
|
||||
VehicleStatus,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def safe_decimal(val: Any) -> Optional[Decimal]:
|
||||
"""Safely extracts a Decimal from numeric, string, or dict values."""
|
||||
if val is None:
|
||||
return None
|
||||
if isinstance(val, dict):
|
||||
val = val.get("value") or val.get("amount") or val.get("rawValue")
|
||||
if val is None:
|
||||
return None
|
||||
try:
|
||||
raw_s = str(val).strip().replace("€", "").replace("\xa0", "").strip()
|
||||
if "," in raw_s and "." in raw_s:
|
||||
# German number formatting like 21.480,00
|
||||
val_str = raw_s.replace(".", "").replace(",", ".")
|
||||
return Decimal(val_str)
|
||||
elif "," in raw_s:
|
||||
return Decimal(raw_s.replace(",", "."))
|
||||
elif "." in raw_s:
|
||||
return Decimal(raw_s)
|
||||
if not raw_s:
|
||||
return None
|
||||
return Decimal(raw_s)
|
||||
except Exception:
|
||||
try:
|
||||
return Decimal(str(val).strip())
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def safe_int(val: Any) -> Optional[int]:
|
||||
"""Safely extracts an integer."""
|
||||
if val is None:
|
||||
return None
|
||||
if isinstance(val, dict):
|
||||
val = val.get("value")
|
||||
try:
|
||||
val_str = str(val).split(".")[0].strip()
|
||||
return int(val_str)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def parse_german_date(val: Any) -> Optional[date]:
|
||||
"""Parses German date string DD.MM.YYYY or timestamp/ISO."""
|
||||
if not val:
|
||||
return None
|
||||
if isinstance(val, (int, float)):
|
||||
# Millisecond timestamp e.g. 1319068800000
|
||||
try:
|
||||
if val > 1e11:
|
||||
val = val / 1000.0
|
||||
return datetime.fromtimestamp(val).date()
|
||||
except Exception:
|
||||
return None
|
||||
if isinstance(val, str):
|
||||
val = val.strip()
|
||||
# DD.MM.YYYY
|
||||
if "." in val:
|
||||
try:
|
||||
parts = val.split(".")
|
||||
if len(parts) == 3:
|
||||
return date(int(parts[2]), int(parts[1]), int(parts[0]))
|
||||
except Exception:
|
||||
pass
|
||||
# ISO YYYY-MM-DD
|
||||
if "-" in val:
|
||||
try:
|
||||
clean = val.split("T")[0].strip()
|
||||
return datetime.strptime(clean, "%Y-%m-%d").date()
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def map_audi_fuel_type(fuel_raw: Any) -> FuelType:
|
||||
"""Maps Audi fuel code / description to standardized FuelType enum."""
|
||||
code = ""
|
||||
desc = ""
|
||||
if isinstance(fuel_raw, dict):
|
||||
code = str(fuel_raw.get("code") or "").upper()
|
||||
desc = str(fuel_raw.get("description") or "").upper()
|
||||
elif isinstance(fuel_raw, str):
|
||||
code = fuel_raw.upper()
|
||||
desc = fuel_raw.upper()
|
||||
|
||||
combined = f"{code} {desc}"
|
||||
if "HYBRID" in combined or code == "H":
|
||||
if "MILD" in combined or "MHEV" in combined:
|
||||
return FuelType.MHEV
|
||||
return FuelType.PHEV
|
||||
if "MILD" in combined or "MHEV" in combined:
|
||||
return FuelType.MHEV
|
||||
if code == "E" or "ELEKTRO" in combined or "ELECTRIC" in combined:
|
||||
return FuelType.BEV
|
||||
if code == "D" or "DIESEL" in combined or "TDI" in combined:
|
||||
return FuelType.DIESEL
|
||||
if code == "B" or "BENZIN" in combined or "TFSI" in combined or "PETROL" in combined:
|
||||
return FuelType.PETROL
|
||||
if "CNG" in combined or "ERDGAS" in combined or "G-TRON" in combined:
|
||||
return FuelType.CNG
|
||||
if "LPG" in combined or "AUTOGAS" in combined:
|
||||
return FuelType.LPG
|
||||
return FuelType.PETROL
|
||||
|
||||
|
||||
def map_audi_transmission(trans_raw: Any) -> Optional[TransmissionType]:
|
||||
"""Maps Audi transmission code / description to TransmissionType."""
|
||||
desc = ""
|
||||
if isinstance(trans_raw, dict):
|
||||
desc = str(trans_raw.get("description") or trans_raw.get("code") or "").upper()
|
||||
elif isinstance(trans_raw, str):
|
||||
desc = trans_raw.upper()
|
||||
|
||||
if "S TRONIC" in desc or "TIPTRONIC" in desc or "AUTOMATIK" in desc or "AUTOMATIC" in desc or "DIRECT_DRIVE" in desc:
|
||||
return TransmissionType.AUTOMATIC
|
||||
if "HANDSCHALTUNG" in desc or "MANUELL" in desc or "SCHALT" in desc or "MANUAL" in desc:
|
||||
return TransmissionType.MANUAL
|
||||
return None
|
||||
|
||||
|
||||
def map_audi_drivetrain(drive_raw: Any) -> Optional[DrivetrainType]:
|
||||
"""Maps Audi drivetrain description/code to DrivetrainType."""
|
||||
desc = ""
|
||||
if isinstance(drive_raw, dict):
|
||||
desc = str(drive_raw.get("description") or drive_raw.get("code") or "").upper()
|
||||
elif isinstance(drive_raw, str):
|
||||
desc = drive_raw.upper()
|
||||
|
||||
if "QUATTRO" in desc or "ALLRAD" in desc or "AWD" in desc or "4X4" in desc:
|
||||
return DrivetrainType.AWD
|
||||
if "FRONT" in desc or "FWD" in desc:
|
||||
return DrivetrainType.FWD
|
||||
if "HECK" in desc or "RWD" in desc or "REAR" in desc:
|
||||
return DrivetrainType.RWD
|
||||
return None
|
||||
|
||||
|
||||
def map_audi_series(carline_raw: Any, model_desc: Optional[str] = None) -> str:
|
||||
"""Extracts standardized Audi model series (e.g. A3, A4, Q4 e-tron, e-tron GT)."""
|
||||
carline = ""
|
||||
if isinstance(carline_raw, dict):
|
||||
carline = str(carline_raw.get("description") or carline_raw.get("code") or "").strip()
|
||||
elif isinstance(carline_raw, str):
|
||||
carline = carline_raw.strip()
|
||||
|
||||
if carline:
|
||||
return carline
|
||||
|
||||
desc = (model_desc or "").strip()
|
||||
# Check known Audi series names
|
||||
known_series = [
|
||||
"e-tron GT", "RS e-tron GT", "Q4 e-tron", "Q6 e-tron", "Q8 e-tron", "e-tron",
|
||||
"RS3", "RS4", "RS5", "RS6", "RS7", "RSQ3", "RSQ8",
|
||||
"S1", "S3", "S4", "S5", "S6", "S7", "S8", "SQ2", "SQ5", "SQ7", "SQ8",
|
||||
"A1", "A3", "A4", "A5", "A6", "A7", "A8",
|
||||
"Q2", "Q3", "Q5", "Q7", "Q8", "TT", "R8"
|
||||
]
|
||||
for s in known_series:
|
||||
pattern = r"\b" + re.escape(s) + r"\b"
|
||||
if re.search(pattern, desc, re.I):
|
||||
return s
|
||||
|
||||
return "Audi"
|
||||
|
||||
|
||||
def parse_power(power_display: Optional[str]) -> tuple[Optional[int], Optional[int]]:
|
||||
"""
|
||||
Parses 'powerDisplay' string, e.g. '110 kW (150 PS)' or '70 kW (95 PS)'.
|
||||
Returns (power_kw, power_hp).
|
||||
"""
|
||||
if not power_display:
|
||||
return None, None
|
||||
kw = None
|
||||
hp = None
|
||||
kw_match = re.search(r"(\d+)\s*kW", power_display, re.I)
|
||||
if kw_match:
|
||||
try:
|
||||
kw = int(kw_match.group(1))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
hp_match = re.search(r"(\d+)\s*PS", power_display, re.I)
|
||||
if hp_match:
|
||||
try:
|
||||
hp = int(hp_match.group(1))
|
||||
except Exception:
|
||||
pass
|
||||
elif kw:
|
||||
hp = int(round(kw * 1.35962))
|
||||
|
||||
return kw, hp
|
||||
|
||||
|
||||
def parse_audi_vehicle(
|
||||
basic_item: Dict[str, Any],
|
||||
detail_data: Optional[Dict[str, Any]] = None,
|
||||
market_country: str = "DE",
|
||||
requested_stock_type: str = "new",
|
||||
include_raw: Optional[bool] = None,
|
||||
) -> VehicleIngestSchema:
|
||||
"""
|
||||
Parses an Audi vehicle from search vehicleBasic object and optional detail payload.
|
||||
"""
|
||||
if include_raw is None:
|
||||
include_raw = settings.ENABLE_RAW_EXTRACTION
|
||||
vehicle_id = str(basic_item.get("id") or "").strip()
|
||||
car_id = str(basic_item.get("carId") or basic_item.get("decodedCarId") or vehicle_id).strip()
|
||||
|
||||
model_info = basic_item.get("model") or {}
|
||||
model_name = model_info.get("description") or "Audi"
|
||||
model_code = model_info.get("code")
|
||||
|
||||
series_info = basic_item.get("symbolicCarlineGroup") or basic_item.get("symbolicCarline")
|
||||
series = map_audi_series(series_info, model_name)
|
||||
|
||||
body_type_info = basic_item.get("bodyType") or {}
|
||||
body_type = body_type_info.get("description") or None
|
||||
|
||||
# Condition determination
|
||||
used_dict = basic_item.get("used") or {}
|
||||
is_used_market = requested_stock_type.lower() == "used" or bool(used_dict.get("mileage") or used_dict.get("initialRegistration"))
|
||||
|
||||
if is_used_market:
|
||||
# Check if certified pre-owned (Audi Gebrauchtwagen :plus)
|
||||
warranty_plus = basic_item.get("warrantyPlus", False)
|
||||
condition = VehicleCondition.CERTIFIED_PRE_OWNED if warranty_plus else VehicleCondition.USED
|
||||
else:
|
||||
condition = VehicleCondition.NEW
|
||||
|
||||
# Powertrain & Specs
|
||||
fuel_info = basic_item.get("fuel") or {}
|
||||
fuel_type = map_audi_fuel_type(fuel_info)
|
||||
gear_info = basic_item.get("gearType") or basic_item.get("gearBox") or {}
|
||||
transmission = map_audi_transmission(gear_info)
|
||||
drive_info = basic_item.get("driveType") or {}
|
||||
drivetrain = map_audi_drivetrain(drive_info)
|
||||
|
||||
power_kw, power_hp = parse_power(basic_item.get("powerDisplay"))
|
||||
model_year = safe_int(basic_item.get("modelYear"))
|
||||
|
||||
# Used vehicle specifics
|
||||
mileage_val = safe_int(used_dict.get("mileage"))
|
||||
mileage_km = mileage_val if mileage_val is not None else 0
|
||||
|
||||
first_reg = parse_german_date(used_dict.get("initialRegistration") or used_dict.get("initialRegistrationDate"))
|
||||
prev_owners = safe_int(used_dict.get("numPreviousOwners"))
|
||||
|
||||
# Pricing & Rates
|
||||
typed_prices = basic_item.get("typedPrices") or []
|
||||
price_gross = None
|
||||
monthly_rate = None
|
||||
currency = "EUR"
|
||||
|
||||
for tp in typed_prices:
|
||||
p_type = tp.get("type", "").lower()
|
||||
amt = safe_decimal(tp.get("amount"))
|
||||
if tp.get("currencyCode"):
|
||||
currency = tp["currencyCode"]
|
||||
|
||||
if p_type in ("retail", "salesprice", "standardprice"):
|
||||
if amt and (price_gross is None or amt > price_gross):
|
||||
price_gross = amt
|
||||
elif p_type in ("financerate", "rate", "leasingrate"):
|
||||
if amt and (monthly_rate is None or amt < monthly_rate):
|
||||
monthly_rate = amt
|
||||
|
||||
# Fallback to any positive amount if retail was not tagged
|
||||
if price_gross is None:
|
||||
amounts = [safe_decimal(tp.get("amount")) for tp in typed_prices if safe_decimal(tp.get("amount"))]
|
||||
if amounts:
|
||||
price_gross = max(amounts)
|
||||
|
||||
if price_gross is None:
|
||||
price_gross = Decimal("0.00")
|
||||
|
||||
# Dealer / GPS Geolocation
|
||||
dealer_info = basic_item.get("dealer") or {}
|
||||
dealer_name = dealer_info.get("name")
|
||||
dealer_id = str(dealer_info.get("id") or "").strip() or None
|
||||
postal_code = dealer_info.get("zipCode")
|
||||
city = dealer_info.get("city")
|
||||
|
||||
geo_loc = dealer_info.get("geoLocation") or {}
|
||||
lat_val = safe_decimal(geo_loc.get("lat"))
|
||||
lon_val = safe_decimal(geo_loc.get("lon"))
|
||||
|
||||
# Images
|
||||
image_urls: List[str] = []
|
||||
# Collect tile and picture URLs
|
||||
tiles = basic_item.get("tilesPictures") or []
|
||||
for tile in tiles:
|
||||
if isinstance(tile, str) and tile.startswith("http") and tile not in image_urls:
|
||||
image_urls.append(tile)
|
||||
|
||||
pics = basic_item.get("pictures") or []
|
||||
for pic in pics:
|
||||
if isinstance(pic, str) and pic.startswith("http") and pic not in image_urls:
|
||||
image_urls.append(pic)
|
||||
|
||||
used_pics = used_dict.get("pictureUrls") or []
|
||||
for upic in used_pics:
|
||||
if isinstance(upic, str) and upic.startswith("http") and upic not in image_urls:
|
||||
image_urls.append(upic)
|
||||
|
||||
thumbnail_url = image_urls[0] if image_urls else None
|
||||
|
||||
# Vehicle Web URL
|
||||
portal = "gebrauchtwagenboerse" if is_used_market else "neuwagenboerse"
|
||||
vehicle_url = f"https://www.audi.de/de/{portal}/fahrzeugdetails/?vehicleid={vehicle_id}"
|
||||
|
||||
# Colors
|
||||
ext_color_info = basic_item.get("extColor") or {}
|
||||
ext_color = ext_color_info.get("description") if isinstance(ext_color_info, dict) else str(ext_color_info or "") or None
|
||||
ext_color_code = ext_color_info.get("code") if isinstance(ext_color_info, dict) else None
|
||||
|
||||
int_color_info = basic_item.get("intColor") or {}
|
||||
int_color = int_color_info.get("description") if isinstance(int_color_info, dict) else str(int_color_info or "") or None
|
||||
|
||||
# Technical Specs & Options (Enriched via detail payload if present)
|
||||
battery_capacity = None
|
||||
battery_capacity_unit = None
|
||||
battery_capacity_gross_kwh = None
|
||||
battery_capacity_net_kwh = None
|
||||
charging_power_ac_kw = None
|
||||
charging_power_dc_kw = None
|
||||
charging_time_dc_minutes = None
|
||||
electric_range_wltp_km = None
|
||||
disp_cc = None
|
||||
options: List[OptionSchema] = []
|
||||
|
||||
raw_payload: Dict[str, Any] = {"vehicleBasic": basic_item} if include_raw else {}
|
||||
|
||||
if detail_data:
|
||||
if include_raw:
|
||||
raw_payload["detail"] = detail_data
|
||||
detail_obj = detail_data.get("vehicle", {}).get("detail", {}) if "vehicle" in detail_data else detail_data
|
||||
|
||||
# 1. Technical specifications from techData
|
||||
tech_data = detail_obj.get("techData", {})
|
||||
fields = tech_data.get("fields", [])
|
||||
for f in fields:
|
||||
f_key = (f.get("key") or "").lower()
|
||||
f_name = (f.get("name") or "").strip()
|
||||
f_val = str(f.get("value") or "").strip()
|
||||
|
||||
# Displacement
|
||||
if f_key == "displacement" or "hubraum" in f_name.lower():
|
||||
disp_match = re.search(r"(\d+[\.\d+]*)", f_val.replace(".", ""))
|
||||
if disp_match:
|
||||
disp_cc = safe_int(disp_match.group(1))
|
||||
|
||||
# Battery Capacity Net
|
||||
elif "netto-batterie-kapazit" in f_name.lower() or "netto-energieinhalt" in f_name.lower():
|
||||
cap_m = re.search(r"([0-9]+[,\.][0-9]+|[0-9]+)\s*kwh", f_val, re.I)
|
||||
if cap_m:
|
||||
val_dec = safe_decimal(cap_m.group(1))
|
||||
if val_dec:
|
||||
battery_capacity_net_kwh = val_dec
|
||||
battery_capacity = val_dec
|
||||
battery_capacity_unit = BatteryCapacityUnit.KWH
|
||||
|
||||
# Battery Capacity Gross
|
||||
elif "brutto-batterie-kapazit" in f_name.lower() or "brutto-energieinhalt" in f_name.lower():
|
||||
cap_m = re.search(r"([0-9]+[,\.][0-9]+|[0-9]+)\s*kwh", f_val, re.I)
|
||||
if cap_m:
|
||||
val_dec = safe_decimal(cap_m.group(1))
|
||||
if val_dec:
|
||||
battery_capacity_gross_kwh = val_dec
|
||||
|
||||
# DC Charging Time (10-80%)
|
||||
elif "ladedauer" in f_name.lower() and "dc" in f_name.lower():
|
||||
time_m = re.search(r"(\d+)\s*min", f_val, re.I)
|
||||
if time_m:
|
||||
charging_time_dc_minutes = safe_int(time_m.group(1))
|
||||
|
||||
# Charging Power AC / HPC (DC)
|
||||
elif "ladedauer" not in f_name.lower() and ("max. ladeleistung" in f_name.lower() or "ladeleistung ac / hpc" in f_name.lower()):
|
||||
# e.g. "11 kW / 160 kW" or "22 kW / 270 kW"
|
||||
rates = re.findall(r"([0-9]+[,\.][0-9]+|[0-9]+)\s*kw", f_val, re.I)
|
||||
if len(rates) >= 2:
|
||||
charging_power_ac_kw = safe_decimal(rates[0])
|
||||
charging_power_dc_kw = safe_decimal(rates[1])
|
||||
elif len(rates) == 1:
|
||||
charging_power_dc_kw = safe_decimal(rates[0])
|
||||
|
||||
# Electric WLTP Range
|
||||
elif "elektrische reichweite" in f_name.lower() or ("reichweite" in f_name.lower() and "wltp" in f_name.lower()):
|
||||
range_m = re.search(r"(\d+)\s*km", f_val, re.I)
|
||||
if range_m:
|
||||
electric_range_wltp_km = safe_int(range_m.group(1))
|
||||
|
||||
# 2. Options and PR Equipment from detail.features
|
||||
features = detail_obj.get("features", [])
|
||||
for feat in features:
|
||||
f_key = feat.get("key") or feat.get("prno")
|
||||
f_name = feat.get("name")
|
||||
if not f_name and feat.get("texts"):
|
||||
for t in feat["texts"]:
|
||||
if t.get("text"):
|
||||
f_name = t["text"]
|
||||
break
|
||||
f_name = f_name or f_key or "Ausstattung"
|
||||
group_name = feat.get("groupName") or feat.get("parentGroupName")
|
||||
kind = (feat.get("kind") or "").lower()
|
||||
is_standard = kind == "serie"
|
||||
|
||||
options.append(
|
||||
OptionSchema(
|
||||
option_code=f_key,
|
||||
name=f_name,
|
||||
category=group_name,
|
||||
is_standard=is_standard,
|
||||
raw_data={"prno": feat.get("prno"), "kind": feat.get("kind")} if include_raw else {},
|
||||
)
|
||||
)
|
||||
|
||||
# If pure EV, assign standard fallback charging power if not already provided
|
||||
if fuel_type == FuelType.BEV:
|
||||
if not charging_power_ac_kw:
|
||||
charging_power_ac_kw = Decimal("11.00")
|
||||
if not charging_power_dc_kw:
|
||||
if "e-tron gt" in model_name.lower():
|
||||
charging_power_dc_kw = Decimal("270.00")
|
||||
elif "q6" in model_name.lower():
|
||||
charging_power_dc_kw = Decimal("270.00")
|
||||
elif "q8" in model_name.lower():
|
||||
charging_power_dc_kw = Decimal("170.00")
|
||||
elif "q4" in model_name.lower():
|
||||
charging_power_dc_kw = Decimal("160.00")
|
||||
|
||||
trim_info = basic_item.get("trimline")
|
||||
trim_line = None
|
||||
if isinstance(trim_info, dict):
|
||||
trim_line = trim_info.get("description") or trim_info.get("code")
|
||||
elif isinstance(trim_info, str):
|
||||
trim_line = trim_info
|
||||
if not trim_line:
|
||||
trim_line = model_code
|
||||
|
||||
return VehicleIngestSchema(
|
||||
brand="AUDI",
|
||||
oem_vehicle_id=car_id,
|
||||
vin=None,
|
||||
series=series,
|
||||
model_name=model_name,
|
||||
trim_line=trim_line,
|
||||
body_type=body_type,
|
||||
condition=condition,
|
||||
model_year=model_year,
|
||||
first_registration_date=first_reg,
|
||||
previous_owners_count=prev_owners,
|
||||
fuel_type=fuel_type,
|
||||
transmission=transmission,
|
||||
drivetrain=drivetrain,
|
||||
power_kw=power_kw,
|
||||
power_hp=power_hp,
|
||||
engine_displacement_cc=disp_cc,
|
||||
battery_capacity=battery_capacity,
|
||||
battery_capacity_unit=battery_capacity_unit,
|
||||
battery_capacity_gross_kwh=battery_capacity_gross_kwh,
|
||||
battery_capacity_net_kwh=battery_capacity_net_kwh,
|
||||
charging_power_ac_kw=charging_power_ac_kw,
|
||||
charging_power_dc_kw=charging_power_dc_kw,
|
||||
charging_time_dc_minutes=charging_time_dc_minutes,
|
||||
electric_range_wltp_km=electric_range_wltp_km,
|
||||
mileage_km=mileage_km,
|
||||
exterior_color=ext_color,
|
||||
exterior_color_code=ext_color_code,
|
||||
interior_color=int_color,
|
||||
currency=currency,
|
||||
price_gross=price_gross,
|
||||
monthly_rate=monthly_rate,
|
||||
status=VehicleStatus.IN_STOCK,
|
||||
dealer_name=dealer_name,
|
||||
dealer_id=dealer_id,
|
||||
location_city=city,
|
||||
location_postal_code=postal_code,
|
||||
location_country=market_country,
|
||||
latitude=lat_val,
|
||||
longitude=lon_val,
|
||||
vehicle_url=vehicle_url,
|
||||
image_urls=image_urls,
|
||||
thumbnail_url=thumbnail_url,
|
||||
options=options,
|
||||
raw_payload=raw_payload,
|
||||
)
|
||||
86
src/oem_crawler/crawlers/base.py
Normal file
86
src/oem_crawler/crawlers/base.py
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
"""
|
||||
Base Crawler Abstract Interface and Crawler Registry.
|
||||
Allows adding new OEM crawlers (Mercedes, Audi, Tesla, VW, etc.) modularly.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Callable, Dict, Generator, Optional, Type
|
||||
|
||||
from oem_crawler.db.schemas import VehicleIngestSchema
|
||||
|
||||
|
||||
class BaseCrawler(ABC):
|
||||
"""Abstract base class for OEM car stock crawlers."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
market_country: str = "DE",
|
||||
proxy: Optional[str] = None,
|
||||
request_delay: float = 1.0,
|
||||
timeout: float = 25.0,
|
||||
**kwargs: Any,
|
||||
):
|
||||
self.market_country = market_country.upper()
|
||||
self.proxy = proxy
|
||||
self.request_delay = request_delay
|
||||
self.timeout = timeout
|
||||
self.extra_kwargs = kwargs
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def brand_name(self) -> str:
|
||||
"""Returns the standardized brand name (e.g. 'BMW', 'MERCEDES', 'AUDI')."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def crawl(
|
||||
self,
|
||||
max_vehicles: Optional[int] = None,
|
||||
page_size: int = 12,
|
||||
search_filter: Optional[Dict[str, Any]] = None,
|
||||
) -> Generator[VehicleIngestSchema, None, None]:
|
||||
"""
|
||||
Main crawling generator yielding normalized VehicleIngestSchema objects.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class CrawlerRegistry:
|
||||
"""Registry to register and dynamically instantiate OEM crawlers."""
|
||||
|
||||
_registry: Dict[str, Type[BaseCrawler]] = {}
|
||||
|
||||
@classmethod
|
||||
def register(cls, brand: str) -> Callable[[Type[BaseCrawler]], Type[BaseCrawler]]:
|
||||
"""Decorator to register a crawler class for a brand."""
|
||||
def decorator(subclass: Type[BaseCrawler]) -> Type[BaseCrawler]:
|
||||
key = brand.strip().lower()
|
||||
cls._registry[key] = subclass
|
||||
return subclass
|
||||
return decorator
|
||||
|
||||
@classmethod
|
||||
def get(cls, brand: str) -> Type[BaseCrawler]:
|
||||
"""Retrieves crawler class for a given brand name."""
|
||||
key = brand.strip().lower()
|
||||
if key not in cls._registry:
|
||||
supported = ", ".join(cls.supported_brands())
|
||||
raise ValueError(f"No crawler registered for brand '{brand}'. Supported brands: {supported}")
|
||||
return cls._registry[key]
|
||||
|
||||
@classmethod
|
||||
def supported_brands(cls) -> list[str]:
|
||||
"""Lists all registered OEM brands."""
|
||||
return sorted(list(cls._registry.keys()))
|
||||
|
||||
@classmethod
|
||||
def unique_brand_crawlers(cls) -> list[tuple[str, Type[BaseCrawler]]]:
|
||||
"""Returns unique (brand_key, crawler_cls) pairs avoiding alias duplicates."""
|
||||
seen_classes = set()
|
||||
unique = []
|
||||
for brand, crawler_cls in sorted(cls._registry.items()):
|
||||
if crawler_cls not in seen_classes:
|
||||
seen_classes.add(crawler_cls)
|
||||
unique.append((brand, crawler_cls))
|
||||
return unique
|
||||
3
src/oem_crawler/crawlers/bmw/__init__.py
Normal file
3
src/oem_crawler/crawlers/bmw/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
from oem_crawler.crawlers.bmw.crawler import BMWStockCrawler
|
||||
|
||||
__all__ = ["BMWStockCrawler"]
|
||||
178
src/oem_crawler/crawlers/bmw/crawler.py
Normal file
178
src/oem_crawler/crawlers/bmw/crawler.py
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
"""
|
||||
BMW Stock Crawler Implementation.
|
||||
Queries BMW AWS STOLO REST API with pagination, retry, and rate-limit backoff.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Any, Dict, Generator, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from oem_crawler.crawlers.base import BaseCrawler, CrawlerRegistry
|
||||
from oem_crawler.crawlers.bmw.parser import parse_bmw_vehicle
|
||||
from oem_crawler.db.schemas import VehicleIngestSchema
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@CrawlerRegistry.register("bmw")
|
||||
class BMWStockCrawler(BaseCrawler):
|
||||
ENDPOINTS = {
|
||||
"new": "https://stolo-data-service.prod.stolo.eu-central-1.aws.bmw.cloud/vehiclesearch/search/de-de/neuwagensuche",
|
||||
"used": "https://stolo-data-service.prod.stolo.eu-central-1.aws.bmw.cloud/vehiclesearch/search/de-de/gebrauchtwagen",
|
||||
}
|
||||
DEFAULT_API_KEY = "7f665f5b3cb8fe8e83293052367c575f666078b87570d5557f02f248ec98"
|
||||
DEFAULT_HASH = "44c98fad4834174d354bb294444a3f52af15e9b206f41a0e58c461050d00c8ff"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
market_country: str = "DE",
|
||||
proxy: Optional[str] = None,
|
||||
request_delay: float = 1.0,
|
||||
timeout: float = 25.0,
|
||||
api_key: Optional[str] = None,
|
||||
stock_type: str = "new",
|
||||
**kwargs: Any,
|
||||
):
|
||||
super().__init__(
|
||||
market_country=market_country,
|
||||
proxy=proxy,
|
||||
request_delay=request_delay,
|
||||
timeout=timeout,
|
||||
**kwargs,
|
||||
)
|
||||
self.api_key = api_key or self.DEFAULT_API_KEY
|
||||
self.stock_type = stock_type.lower()
|
||||
self.headers = {
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Accept-Language": "de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7",
|
||||
"Content-Type": "application/json",
|
||||
"Origin": "https://www.bmw.de",
|
||||
"Referer": "https://www.bmw.de/",
|
||||
"Sec-Ch-Ua": '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"',
|
||||
"Sec-Ch-Ua-Mobile": "?0",
|
||||
"Sec-Ch-Ua-Platform": '"macOS"',
|
||||
"Sec-Fetch-Dest": "empty",
|
||||
"Sec-Fetch-Mode": "cors",
|
||||
"Sec-Fetch-Site": "cross-site",
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
|
||||
"X-Api-Key": self.api_key,
|
||||
}
|
||||
|
||||
@property
|
||||
def brand_name(self) -> str:
|
||||
return "BMW"
|
||||
|
||||
def fetch_page(
|
||||
self,
|
||||
start_index: int = 0,
|
||||
max_results: int = 12,
|
||||
search_filter: Optional[Dict[str, Any]] = None,
|
||||
stock_type: str = "new",
|
||||
max_retries: int = 4,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Fetches a single page of vehicle stock from BMW's REST search API with exponential backoff on 429s.
|
||||
Note: BMW limits maxResults to a maximum of 12 per request.
|
||||
"""
|
||||
max_results = min(max_results, 12)
|
||||
endpoint = self.ENDPOINTS.get(stock_type, self.ENDPOINTS["new"])
|
||||
params = {
|
||||
"maxResults": max_results,
|
||||
"startIndex": start_index,
|
||||
"brand": "BMW",
|
||||
"context": "results-page",
|
||||
}
|
||||
payload = {
|
||||
"searchContext": [search_filter or {}],
|
||||
"resultsContext": {
|
||||
"sort": [{"by": "SF_OFFER_INSTALLMENT", "order": "ASC"}]
|
||||
},
|
||||
}
|
||||
|
||||
with httpx.Client(timeout=self.timeout, proxy=self.proxy) as client:
|
||||
for attempt in range(max_retries + 1):
|
||||
response = client.post(
|
||||
endpoint,
|
||||
params=params,
|
||||
headers=self.headers,
|
||||
json=payload,
|
||||
)
|
||||
if response.status_code == 429:
|
||||
wait_time = 2.0 * (attempt + 1)
|
||||
logger.warning(
|
||||
f"BMW rate limit (HTTP 429). Retrying in {wait_time:.1f}s (attempt {attempt+1}/{max_retries})..."
|
||||
)
|
||||
time.sleep(wait_time)
|
||||
continue
|
||||
|
||||
if response.status_code in (400, 403) and start_index >= 1000:
|
||||
logger.info(
|
||||
f"BMW search API reached maximum pagination window at offset {start_index} (HTTP {response.status_code}). Ending category."
|
||||
)
|
||||
return {"hits": [], "metadata": {"totalCount": start_index}}
|
||||
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
def crawl(
|
||||
self,
|
||||
max_vehicles: Optional[int] = None,
|
||||
page_size: int = 12,
|
||||
search_filter: Optional[Dict[str, Any]] = None,
|
||||
) -> Generator[VehicleIngestSchema, None, None]:
|
||||
"""
|
||||
Iterates over pages, fetches raw vehicle data, parses and yields normalized VehicleIngestSchema objects.
|
||||
Supports stock_type: 'new', 'used', or 'all'.
|
||||
"""
|
||||
stock_types = ["new", "used"] if self.stock_type == "all" else [self.stock_type]
|
||||
page_size = min(page_size, 12)
|
||||
total_yielded = 0
|
||||
|
||||
for current_type in stock_types:
|
||||
start_index = 0
|
||||
total_count = None
|
||||
logger.info(f"Starting BMW stock crawl for category: {current_type.upper()}...")
|
||||
|
||||
while True:
|
||||
logger.info(f"Fetching BMW ({current_type}) stock: offset={start_index}, pageSize={page_size}...")
|
||||
data = self.fetch_page(
|
||||
start_index=start_index,
|
||||
max_results=page_size,
|
||||
search_filter=search_filter,
|
||||
stock_type=current_type,
|
||||
)
|
||||
|
||||
if total_count is None:
|
||||
total_count = data.get("metadata", {}).get("totalCount", 0)
|
||||
logger.info(f"BMW reports {total_count} total {current_type} vehicles in stock.")
|
||||
|
||||
hits = data.get("hits", [])
|
||||
if not hits:
|
||||
break
|
||||
|
||||
for hit in hits:
|
||||
raw_veh = hit.get("vehicle", {})
|
||||
if not raw_veh:
|
||||
continue
|
||||
|
||||
try:
|
||||
parsed = parse_bmw_vehicle(raw_veh, market_country=self.market_country, stock_type=current_type)
|
||||
yield parsed
|
||||
total_yielded += 1
|
||||
if max_vehicles and total_yielded >= max_vehicles:
|
||||
return
|
||||
except Exception as e:
|
||||
doc_id = raw_veh.get("documentId", "UNKNOWN")
|
||||
logger.error(f"Failed to parse vehicle {doc_id}: {e}", exc_info=True)
|
||||
|
||||
start_index += len(hits)
|
||||
if start_index >= total_count:
|
||||
break
|
||||
|
||||
time.sleep(self.request_delay)
|
||||
292
src/oem_crawler/crawlers/bmw/parser.py
Normal file
292
src/oem_crawler/crawlers/bmw/parser.py
Normal file
|
|
@ -0,0 +1,292 @@
|
|||
"""
|
||||
Parser and normalization logic for BMW OEM Neuwagensuche responses.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from decimal import Decimal
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from oem_crawler.config import settings
|
||||
from oem_crawler.db.schemas import (
|
||||
DrivetrainType,
|
||||
FuelType,
|
||||
OptionSchema,
|
||||
TransmissionType,
|
||||
VehicleCondition,
|
||||
VehicleIngestSchema,
|
||||
VehicleStatus,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def safe_decimal(val: Any) -> Optional[Decimal]:
|
||||
"""Extracts a Decimal from numeric, string, or nested dict value safely."""
|
||||
if val is None:
|
||||
return None
|
||||
if isinstance(val, dict):
|
||||
val = val.get("value")
|
||||
if val is None:
|
||||
return None
|
||||
try:
|
||||
val_str = str(val).strip()
|
||||
if not val_str:
|
||||
return None
|
||||
return Decimal(val_str)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def map_fuel_type(raw_fuel: str, is_48v: bool) -> FuelType:
|
||||
raw_fuel = (raw_fuel or "").upper()
|
||||
if "ELECTRIC" in raw_fuel or raw_fuel == "BEV":
|
||||
return FuelType.BEV
|
||||
if "PLUGIN" in raw_fuel or "PHEV" in raw_fuel or raw_fuel == "HYBRID":
|
||||
return FuelType.PHEV
|
||||
if is_48v or "MILD" in raw_fuel:
|
||||
return FuelType.MHEV
|
||||
if "DIESEL" in raw_fuel:
|
||||
return FuelType.DIESEL
|
||||
if "GASOLINE" in raw_fuel or "BENZIN" in raw_fuel or "PETROL" in raw_fuel:
|
||||
return FuelType.PETROL
|
||||
return FuelType.PETROL
|
||||
|
||||
|
||||
def map_transmission(raw_trans: str) -> Optional[TransmissionType]:
|
||||
raw_trans = (raw_trans or "").upper()
|
||||
if "AUTOMATIC" in raw_trans or "AUT" in raw_trans:
|
||||
return TransmissionType.AUTOMATIC
|
||||
if "MANUAL" in raw_trans:
|
||||
return TransmissionType.MANUAL
|
||||
if "DIRECT" in raw_trans:
|
||||
return TransmissionType.DIRECT_DRIVE
|
||||
return None
|
||||
|
||||
|
||||
def map_drivetrain(raw_drive: str) -> Optional[DrivetrainType]:
|
||||
raw_drive = (raw_drive or "").upper()
|
||||
if "XDRIVE" in raw_drive or "AWD" in raw_drive or "4X4" in raw_drive:
|
||||
return DrivetrainType.AWD
|
||||
if "RWD" in raw_drive:
|
||||
return DrivetrainType.RWD
|
||||
if "SDRIVE" in raw_drive or "FWD" in raw_drive:
|
||||
return DrivetrainType.FWD
|
||||
return None
|
||||
|
||||
|
||||
def parse_bmw_vehicle(
|
||||
raw: Dict[str, Any],
|
||||
market_country: str = "DE",
|
||||
stock_type: str = "new",
|
||||
include_raw: Optional[bool] = None,
|
||||
) -> VehicleIngestSchema:
|
||||
"""
|
||||
Transforms raw OEM BMW JSON into the normalized VehicleIngestSchema for both New and Used vehicles.
|
||||
"""
|
||||
if include_raw is None:
|
||||
include_raw = settings.ENABLE_RAW_EXTRACTION
|
||||
spec = raw.get("vehicleSpecification", {})
|
||||
model_and_opt = spec.get("modelAndOption", {})
|
||||
model_info = model_and_opt.get("model", {})
|
||||
model_range = model_and_opt.get("modelRange", {})
|
||||
tech_data = spec.get("technicalAndEmission", {}).get("technicalData", {})
|
||||
pricing = raw.get("pricing", {})
|
||||
raw_price = raw.get("price", {})
|
||||
ordering = raw.get("ordering", {})
|
||||
retail = ordering.get("retailData", {})
|
||||
outlet_addr = retail.get("locationOutletAddress", {})
|
||||
life_cycle = raw.get("vehicleLifeCycle", {})
|
||||
|
||||
# Identifiers
|
||||
document_id = str(raw.get("documentId"))
|
||||
vin = raw.get("vin") or raw.get("internal", {}).get("vin")
|
||||
|
||||
# Model & Classification
|
||||
series_name = (
|
||||
model_range.get("description", {}).get("de_DE")
|
||||
or model_range.get("description", {}).get("default_DE")
|
||||
or model_and_opt.get("series", {}).get("name")
|
||||
or "BMW"
|
||||
)
|
||||
model_name = (
|
||||
model_info.get("modelName")
|
||||
or model_info.get("modelDescription", {}).get("de_DE")
|
||||
or series_name
|
||||
)
|
||||
trim_line = model_and_opt.get("marketingModelRange")
|
||||
body_type = model_and_opt.get("bodyType")
|
||||
|
||||
# Condition & Lifecycle (New vs Used)
|
||||
if stock_type.lower() == "used" or life_cycle.get("mileage") or "LMP" in document_id:
|
||||
condition = VehicleCondition.USED
|
||||
else:
|
||||
condition = VehicleCondition.NEW
|
||||
|
||||
# Mileage
|
||||
mileage_km = 0
|
||||
if life_cycle and isinstance(life_cycle.get("mileage"), dict):
|
||||
mileage_km = int(life_cycle["mileage"].get("km", 0))
|
||||
|
||||
# Previous owners
|
||||
previous_owners_count = life_cycle.get("numberOfPreviousOwners")
|
||||
|
||||
# Registration & Production Date
|
||||
first_registration_date = None
|
||||
raw_reg_date = (
|
||||
retail.get("officialRegistrationDocDate")
|
||||
or raw_price.get("priceRequestSpecification", {}).get("productionDate")
|
||||
)
|
||||
if raw_reg_date and isinstance(raw_reg_date, str):
|
||||
try:
|
||||
from datetime import date
|
||||
first_registration_date = date.fromisoformat(raw_reg_date.split("T")[0])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
vat_deductible = life_cycle.get("taxDeductible")
|
||||
|
||||
# Powertrain mapping
|
||||
fuel_type = map_fuel_type(model_and_opt.get("baseFuelType", ""), model_and_opt.get("is48Volt", False))
|
||||
transmission = map_transmission(model_and_opt.get("transmission", ""))
|
||||
drivetrain = map_drivetrain(model_and_opt.get("marketingDriveType", ""))
|
||||
|
||||
power_kw = None
|
||||
power_hp = None
|
||||
if "power" in tech_data and isinstance(tech_data["power"], dict):
|
||||
power_kw = tech_data["power"].get("value")
|
||||
if "powerInHp" in tech_data and isinstance(tech_data["powerInHp"], dict):
|
||||
power_hp = tech_data["powerInHp"].get("value")
|
||||
|
||||
# EV & Battery
|
||||
electric_range_km = None
|
||||
if "electricRange" in tech_data and isinstance(tech_data["electricRange"], dict):
|
||||
electric_range_km = tech_data["electricRange"].get("value")
|
||||
|
||||
# Colors & Upholstery
|
||||
color_info = model_and_opt.get("color", {})
|
||||
exterior_color = color_info.get("name", {}).get("de_DE") or color_info.get("name", {}).get("default_DE")
|
||||
exterior_color_code = color_info.get("code")
|
||||
|
||||
# If exterior_color is empty, fallback to searching equipments for paint (codes like P0xxx or category Außenfarben)
|
||||
if not exterior_color:
|
||||
for code, eq in model_and_opt.get("equipments", {}).items():
|
||||
if isinstance(eq, dict) and (code.startswith("P0") or "Außenfarbe" in str(eq.get("marketingText", {})) or "Farbe" in str(eq.get("translatedSalesGroups", {}))):
|
||||
exterior_color = eq.get("name", {}).get("de_DE") or eq.get("name", {}).get("default_DE")
|
||||
exterior_color_code = code
|
||||
break
|
||||
|
||||
# Pricing
|
||||
price_gross = safe_decimal(pricing.get("price", {}).get("value"))
|
||||
if price_gross is None:
|
||||
price_gross = safe_decimal(pricing.get("grossPrice")) or safe_decimal(raw_price.get("grossSalesPrice")) or safe_decimal(pricing.get("price"))
|
||||
if price_gross is None:
|
||||
raise ValueError(f"Vehicle {document_id} missing mandatory pricing.price.value")
|
||||
|
||||
# Monthly installment
|
||||
monthly_rate = None
|
||||
sf_params = (
|
||||
pricing.get("monthlyInstallment", {})
|
||||
.get("selectedSfCalculation", {})
|
||||
.get("financialProductParameters", {})
|
||||
)
|
||||
if sf_params and "totalInstallment" in sf_params:
|
||||
monthly_rate = safe_decimal(sf_params["totalInstallment"].get("amount"))
|
||||
if monthly_rate is None:
|
||||
monthly_rate = safe_decimal(pricing.get("monthlyInstallment", {}).get("value")) or safe_decimal(pricing.get("monthlyInstallment"))
|
||||
|
||||
# MSRP / Original list price
|
||||
msrp_gross = safe_decimal(raw_price.get("grossListPrice")) or safe_decimal(pricing.get("originalPrice", {}).get("value")) or safe_decimal(pricing.get("originalPrice"))
|
||||
options_price_gross = safe_decimal(raw_price.get("equipmentsTotalListPriceGross"))
|
||||
|
||||
discount_pct = None
|
||||
if msrp_gross and msrp_gross > 0 and price_gross < msrp_gross:
|
||||
discount_pct = Decimal(str(round(((msrp_gross - price_gross) / msrp_gross) * 100, 2)))
|
||||
|
||||
# Dealer & Location
|
||||
dealer_name = retail.get("locationOutletName")
|
||||
dealer_id = retail.get("locationOutletId") or retail.get("localDealerNumber")
|
||||
city = outlet_addr.get("town")
|
||||
postal_code = outlet_addr.get("postalCode")
|
||||
country = outlet_addr.get("countryCode") or market_country
|
||||
|
||||
# Media & URLs
|
||||
vehicle_url = f"https://www.bmw.de/de-de/sl/neuwagensuche/details/{document_id}"
|
||||
image_urls = []
|
||||
thumbnail_url = None
|
||||
|
||||
# Parse normalized options
|
||||
options: List[OptionSchema] = []
|
||||
equipments = model_and_opt.get("equipments", {})
|
||||
for opt_code, opt_data in equipments.items():
|
||||
if not isinstance(opt_data, dict):
|
||||
continue
|
||||
|
||||
opt_name = (
|
||||
opt_data.get("name", {}).get("de_DE")
|
||||
or opt_data.get("name", {}).get("default_DE")
|
||||
or opt_code
|
||||
)
|
||||
opt_cat = (
|
||||
opt_data.get("marketingText", {}).get("category", {}).get("de_DE")
|
||||
or (opt_data.get("translatedSalesGroups", {}).get("de_DE") or [""])[0]
|
||||
or None
|
||||
)
|
||||
opt_desc = (
|
||||
opt_data.get("marketingText", {}).get("longText", {}).get("de_DE")
|
||||
or opt_data.get("marketingText", {}).get("shortText", {}).get("de_DE")
|
||||
)
|
||||
is_std = bool(opt_data.get("standard", False))
|
||||
|
||||
img_dict = opt_data.get("images", {})
|
||||
if "default" in img_dict:
|
||||
image_urls.append(img_dict["default"])
|
||||
if not thumbnail_url:
|
||||
thumbnail_url = img_dict["default"]
|
||||
|
||||
options.append(
|
||||
OptionSchema(
|
||||
option_code=opt_code,
|
||||
name=opt_name,
|
||||
category=opt_cat,
|
||||
is_standard=is_std,
|
||||
description=opt_desc,
|
||||
raw_data=opt_data if include_raw else {},
|
||||
)
|
||||
)
|
||||
|
||||
return VehicleIngestSchema(
|
||||
brand="BMW",
|
||||
oem_vehicle_id=document_id,
|
||||
vin=vin,
|
||||
series=series_name,
|
||||
model_name=model_name,
|
||||
trim_line=trim_line,
|
||||
body_type=body_type,
|
||||
condition=VehicleCondition.NEW,
|
||||
fuel_type=fuel_type,
|
||||
transmission=transmission,
|
||||
drivetrain=drivetrain,
|
||||
power_kw=power_kw,
|
||||
power_hp=power_hp,
|
||||
electric_range_wltp_km=electric_range_km,
|
||||
mileage_km=0,
|
||||
exterior_color=exterior_color,
|
||||
exterior_color_code=exterior_color_code,
|
||||
currency="EUR",
|
||||
price_gross=price_gross,
|
||||
msrp_gross=msrp_gross,
|
||||
discount_percentage=discount_pct,
|
||||
monthly_rate=monthly_rate,
|
||||
status=VehicleStatus.IN_STOCK,
|
||||
dealer_name=dealer_name,
|
||||
dealer_id=dealer_id,
|
||||
location_city=city,
|
||||
location_postal_code=postal_code,
|
||||
location_country=country,
|
||||
vehicle_url=vehicle_url,
|
||||
image_urls=image_urls[:10],
|
||||
thumbnail_url=thumbnail_url,
|
||||
options=options,
|
||||
raw_payload=raw if include_raw else {},
|
||||
)
|
||||
3
src/oem_crawler/crawlers/mercedes/__init__.py
Normal file
3
src/oem_crawler/crawlers/mercedes/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
from oem_crawler.crawlers.mercedes.crawler import MercedesStockCrawler
|
||||
|
||||
__all__ = ["MercedesStockCrawler"]
|
||||
448
src/oem_crawler/crawlers/mercedes/crawler.py
Normal file
448
src/oem_crawler/crawlers/mercedes/crawler.py
Normal file
|
|
@ -0,0 +1,448 @@
|
|||
"""
|
||||
Mercedes-Benz Stock Crawler Implementation.
|
||||
Queries Mercedes-Benz OneSearch GraphQL and PLS/VPC microservices.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
import re
|
||||
from typing import Any, Dict, Generator, Optional
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
import httpx
|
||||
|
||||
from oem_crawler.crawlers.base import BaseCrawler, CrawlerRegistry
|
||||
from oem_crawler.crawlers.mercedes.parser import (
|
||||
parse_mercedes_used_vehicle,
|
||||
parse_mercedes_vehicle,
|
||||
)
|
||||
from oem_crawler.crawlers.mercedes.query import GRAPHQL_QUERY
|
||||
from oem_crawler.db.schemas import VehicleIngestSchema
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@CrawlerRegistry.register("mercedes-benz")
|
||||
@CrawlerRegistry.register("mercedes")
|
||||
class MercedesStockCrawler(BaseCrawler):
|
||||
GRAPHQL_ENDPOINT = "https://eu.api.oneweb.mercedes-benz.com/commerce/onesearch/graphql"
|
||||
PRICE_ENDPOINT = "https://eu.api.oneweb.mercedes-benz.com/commerce/pls/public/v6/ncos/markets/{market}/prices/{code}"
|
||||
VPC_ENDPOINT = "https://eu.api.oneweb.mercedes-benz.com/commerce/vpc/v2/markets/{market}/products/{code}"
|
||||
USED_BASE_URL = "https://gebrauchtwagen.mercedes-benz.de"
|
||||
|
||||
DEFAULT_API_KEY = "d1dcd3a9-25fd-4896-b041-4d35cfdbb482"
|
||||
DEFAULT_CUSTOMER_GROUP_ID = "5e65a044-ac5a-47ea-ad26-b8948d4e450a"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
market_country: str = "DE",
|
||||
proxy: Optional[str] = None,
|
||||
request_delay: float = 0.5,
|
||||
timeout: float = 25.0,
|
||||
api_key: Optional[str] = None,
|
||||
stock_type: str = "new",
|
||||
context_type: str = "B2C",
|
||||
fetch_pricing: bool = True,
|
||||
fetch_details: bool = True,
|
||||
**kwargs: Any,
|
||||
):
|
||||
super().__init__(
|
||||
market_country=market_country,
|
||||
proxy=proxy,
|
||||
request_delay=request_delay,
|
||||
timeout=timeout,
|
||||
**kwargs,
|
||||
)
|
||||
self.api_key = api_key or self.DEFAULT_API_KEY
|
||||
self.stock_type = stock_type.lower()
|
||||
self.context_type = context_type.upper()
|
||||
self.fetch_pricing = fetch_pricing
|
||||
self.fetch_details = fetch_details
|
||||
|
||||
self.headers = {
|
||||
"Accept": "application/json",
|
||||
"Content-Type": "application/json",
|
||||
"Origin": f"https://www.mercedes-benz.{self.market_country.lower()}",
|
||||
"Referer": f"https://www.mercedes-benz.{self.market_country.lower()}/",
|
||||
"Sec-Ch-Ua": '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"',
|
||||
"Sec-Ch-Ua-Mobile": "?0",
|
||||
"Sec-Ch-Ua-Platform": '"macOS"',
|
||||
"Sec-Fetch-Dest": "empty",
|
||||
"Sec-Fetch-Mode": "cors",
|
||||
"Sec-Fetch-Site": "cross-site",
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
|
||||
"x-api-key": self.api_key,
|
||||
}
|
||||
|
||||
@property
|
||||
def brand_name(self) -> str:
|
||||
return "MERCEDES-BENZ"
|
||||
|
||||
def fetch_page(
|
||||
self,
|
||||
client: httpx.Client,
|
||||
page: int = 0,
|
||||
limit: int = 50,
|
||||
search_filter: Optional[Dict[str, Any]] = None,
|
||||
max_retries: int = 3,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Fetches a page of search results from Mercedes OneSearch GraphQL API.
|
||||
Accepts up to limit=100.
|
||||
"""
|
||||
limit = max(1, min(limit, 100))
|
||||
is_used = self.stock_type == "used"
|
||||
profile_id = f"{self.market_country.upper()}-USED_VEHICLES" if is_used else f"{self.market_country.upper()}-NEW_VEHICLES"
|
||||
|
||||
variables: Dict[str, Any] = {
|
||||
"profileId": profile_id,
|
||||
"vehicleCategory": "PASSENGER-CARS",
|
||||
"brand": ["Mercedes-Benz"],
|
||||
"contextType": self.context_type,
|
||||
"language": "de",
|
||||
"limit": limit,
|
||||
"page": page,
|
||||
"sortingType": "name-asc",
|
||||
"enableColorAggregation": False,
|
||||
"isUcos": is_used,
|
||||
"dealerId": [""],
|
||||
}
|
||||
if search_filter:
|
||||
variables.update(search_filter)
|
||||
|
||||
payload = {
|
||||
"query": GRAPHQL_QUERY,
|
||||
"variables": variables,
|
||||
}
|
||||
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
response = client.post(self.GRAPHQL_ENDPOINT, headers=self.headers, json=payload)
|
||||
if response.status_code == 429:
|
||||
wait_time = 2.0 * (attempt + 1)
|
||||
logger.warning(f"Mercedes rate limit (HTTP 429). Retrying in {wait_time:.1f}s (attempt {attempt+1}/{max_retries})...")
|
||||
time.sleep(wait_time)
|
||||
continue
|
||||
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
if "errors" in data and not data.get("data"):
|
||||
raise ValueError(f"GraphQL errors: {data['errors']}")
|
||||
return data
|
||||
except httpx.HTTPError as e:
|
||||
if attempt == max_retries:
|
||||
raise
|
||||
logger.warning(f"Network error while fetching page {page}: {e}. Retrying...")
|
||||
time.sleep(1.5 * (attempt + 1))
|
||||
|
||||
raise RuntimeError(f"Failed to fetch Mercedes page {page} after {max_retries} retries")
|
||||
|
||||
def fetch_price(self, client: httpx.Client, code: str, max_retries: int = 2) -> Optional[Dict[str, Any]]:
|
||||
"""Fetches pricing and financial terms from the PLS endpoint."""
|
||||
url = self.PRICE_ENDPOINT.format(market=self.market_country.upper(), code=code)
|
||||
params = {
|
||||
"customerType": self.context_type,
|
||||
"customerGroupId": self.DEFAULT_CUSTOMER_GROUP_ID,
|
||||
"language": "de",
|
||||
}
|
||||
pls_headers = {
|
||||
**self.headers,
|
||||
"Consumer-Id": "VEHICLE_TILE",
|
||||
}
|
||||
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
res = client.get(url, params=params, headers=pls_headers)
|
||||
if res.status_code == 200:
|
||||
return res.json()
|
||||
elif res.status_code == 404:
|
||||
return None
|
||||
elif res.status_code == 429:
|
||||
time.sleep(1.5 * (attempt + 1))
|
||||
continue
|
||||
except Exception as e:
|
||||
if attempt == max_retries:
|
||||
logger.debug(f"Could not fetch price for vehicle {code}: {e}")
|
||||
return None
|
||||
time.sleep(1.0)
|
||||
return None
|
||||
|
||||
def fetch_vpc(self, client: httpx.Client, code: str, max_retries: int = 2) -> Optional[Dict[str, Any]]:
|
||||
"""Fetches detailed vehicle options, equipment, and colors from VPC endpoint."""
|
||||
url = self.VPC_ENDPOINT.format(market=self.market_country.upper(), code=code)
|
||||
params = {
|
||||
"language": "de",
|
||||
"legacy": "true",
|
||||
}
|
||||
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
res = client.get(url, params=params, headers=self.headers)
|
||||
if res.status_code == 200:
|
||||
return res.json()
|
||||
elif res.status_code == 404:
|
||||
return None
|
||||
elif res.status_code == 429:
|
||||
time.sleep(1.5 * (attempt + 1))
|
||||
continue
|
||||
except Exception as e:
|
||||
if attempt == max_retries:
|
||||
logger.debug(f"Could not fetch VPC configuration for vehicle {code}: {e}")
|
||||
return None
|
||||
time.sleep(1.0)
|
||||
return None
|
||||
|
||||
def fetch_used_page(
|
||||
self,
|
||||
client: httpx.Client,
|
||||
page: int = 1,
|
||||
sorting: str = "acdade",
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
max_retries: int = 3,
|
||||
) -> str:
|
||||
"""Fetches a page of used vehicles HTML from gebrauchtwagen.mercedes-benz.de."""
|
||||
url = f"{self.USED_BASE_URL}/vehicles"
|
||||
params: Dict[str, Any] = {
|
||||
"page": page,
|
||||
"sorting": sorting,
|
||||
"view": "box",
|
||||
}
|
||||
if filters:
|
||||
params.update(filters)
|
||||
|
||||
used_headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "de-DE,de;q=0.9,en;q=0.8",
|
||||
}
|
||||
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
res = client.get(url, params=params, headers=used_headers)
|
||||
if res.status_code == 429:
|
||||
wait_time = 2.0 * (attempt + 1)
|
||||
logger.warning(f"Mercedes used rate limit (HTTP 429). Retrying in {wait_time:.1f}s...")
|
||||
time.sleep(wait_time)
|
||||
continue
|
||||
res.raise_for_status()
|
||||
return res.text
|
||||
except httpx.HTTPError as e:
|
||||
if attempt == max_retries:
|
||||
raise
|
||||
logger.warning(f"Error fetching used cars page {page}: {e}. Retrying...")
|
||||
time.sleep(1.5 * (attempt + 1))
|
||||
|
||||
raise RuntimeError(f"Failed to fetch Mercedes used cars page {page} after {max_retries} retries")
|
||||
|
||||
def fetch_used_detail(
|
||||
self,
|
||||
client: httpx.Client,
|
||||
vehicle_id: str,
|
||||
max_retries: int = 2,
|
||||
) -> Optional[str]:
|
||||
"""Fetches detailed vehicle HTML page for a used car."""
|
||||
url = f"{self.USED_BASE_URL}/vehicle"
|
||||
params = {"vehicle": vehicle_id, "referrer": "vehicles"}
|
||||
used_headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "de-DE,de;q=0.9,en;q=0.8",
|
||||
}
|
||||
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
res = client.get(url, params=params, headers=used_headers)
|
||||
if res.status_code == 200:
|
||||
return res.text
|
||||
elif res.status_code == 404:
|
||||
return None
|
||||
elif res.status_code == 429:
|
||||
time.sleep(1.5 * (attempt + 1))
|
||||
continue
|
||||
except Exception as e:
|
||||
if attempt == max_retries:
|
||||
logger.debug(f"Could not fetch used vehicle detail for {vehicle_id}: {e}")
|
||||
return None
|
||||
time.sleep(1.0)
|
||||
return None
|
||||
|
||||
def crawl_used(
|
||||
self,
|
||||
client: httpx.Client,
|
||||
max_vehicles: Optional[int] = None,
|
||||
page_size: int = 12,
|
||||
search_filter: Optional[Dict[str, Any]] = None,
|
||||
) -> Generator[VehicleIngestSchema, None, None]:
|
||||
"""
|
||||
Crawls Mercedes-Benz used vehicles catalog from gebrauchtwagen.mercedes-benz.de.
|
||||
"""
|
||||
logger.info(f"Starting Mercedes-Benz used stock crawl (Market: {self.market_country})...")
|
||||
current_page = 1
|
||||
total_yielded = 0
|
||||
filters = dict(search_filter or {})
|
||||
sorting = filters.pop("sorting", "acdade")
|
||||
|
||||
while True:
|
||||
logger.info(f"Fetching Mercedes used stock page {current_page} (sorting={sorting})...")
|
||||
html = self.fetch_used_page(
|
||||
client=client,
|
||||
page=current_page,
|
||||
sorting=sorting,
|
||||
filters=filters,
|
||||
)
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
cards = soup.select(".list-vehicle-card")
|
||||
if not cards:
|
||||
logger.info(f"No more vehicle cards found on page {current_page}. Used stock crawl complete.")
|
||||
break
|
||||
|
||||
for card in cards:
|
||||
raw_id = card.get("id", "")
|
||||
veh_id = re.sub(r"^resultListItem", "", raw_id).strip()
|
||||
if not veh_id:
|
||||
link = card.select_one("a[href*='vehicle=']")
|
||||
if link:
|
||||
m = re.search(r"vehicle=(\d+)", link.get("href", ""))
|
||||
if m:
|
||||
veh_id = m.group(1)
|
||||
|
||||
detail_html = None
|
||||
if self.fetch_details and veh_id:
|
||||
detail_html = self.fetch_used_detail(client, veh_id)
|
||||
if self.request_delay > 0:
|
||||
time.sleep(self.request_delay)
|
||||
|
||||
try:
|
||||
parsed = parse_mercedes_used_vehicle(
|
||||
card=card,
|
||||
detail_html=detail_html,
|
||||
market_country=self.market_country,
|
||||
)
|
||||
yield parsed
|
||||
total_yielded += 1
|
||||
|
||||
if max_vehicles and total_yielded >= max_vehicles:
|
||||
logger.info(f"Reached max limit of {max_vehicles} used vehicles. Done.")
|
||||
return
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse Mercedes used vehicle {veh_id}: {e}", exc_info=True)
|
||||
|
||||
current_page += 1
|
||||
if self.request_delay > 0:
|
||||
time.sleep(self.request_delay)
|
||||
|
||||
def crawl_new(
|
||||
self,
|
||||
client: httpx.Client,
|
||||
max_vehicles: Optional[int] = None,
|
||||
page_size: int = 50,
|
||||
search_filter: Optional[Dict[str, Any]] = None,
|
||||
) -> Generator[VehicleIngestSchema, None, None]:
|
||||
"""
|
||||
Crawls Mercedes-Benz new vehicles stock via OneSearch GraphQL API.
|
||||
"""
|
||||
page_size = max(1, min(page_size, 100))
|
||||
total_yielded = 0
|
||||
current_page = 0
|
||||
total_results: Optional[int] = None
|
||||
total_pages: Optional[int] = None
|
||||
|
||||
logger.info(f"Starting Mercedes-Benz new stock crawl (Market: {self.market_country}, Context: {self.context_type})...")
|
||||
|
||||
while True:
|
||||
logger.info(f"Fetching Mercedes new stock page {current_page} (limit={page_size})...")
|
||||
page_data = self.fetch_page(
|
||||
client=client,
|
||||
page=current_page,
|
||||
limit=page_size,
|
||||
search_filter=search_filter,
|
||||
)
|
||||
|
||||
search_res = page_data.get("data", {}).get("search", {})
|
||||
nav = search_res.get("navigation", {})
|
||||
if total_results is None:
|
||||
total_results = nav.get("totalResults", 0)
|
||||
total_pages = nav.get("totalPages", 1)
|
||||
logger.info(f"Mercedes reports {total_results} total new vehicles ({total_pages} pages).")
|
||||
|
||||
results = search_res.get("results", [])
|
||||
if not results:
|
||||
break
|
||||
|
||||
for raw_veh in results:
|
||||
code = raw_veh.get("identification", {}).get("code")
|
||||
if not code:
|
||||
continue
|
||||
|
||||
raw_price = None
|
||||
if self.fetch_pricing:
|
||||
raw_price = self.fetch_price(client, code)
|
||||
|
||||
raw_vpc = None
|
||||
if self.fetch_details:
|
||||
raw_vpc = self.fetch_vpc(client, code)
|
||||
|
||||
try:
|
||||
parsed = parse_mercedes_vehicle(
|
||||
raw_search=raw_veh,
|
||||
raw_price=raw_price,
|
||||
raw_vpc=raw_vpc,
|
||||
market_country=self.market_country,
|
||||
stock_type=self.stock_type,
|
||||
)
|
||||
yield parsed
|
||||
total_yielded += 1
|
||||
|
||||
if max_vehicles and total_yielded >= max_vehicles:
|
||||
logger.info(f"Reached max limit of {max_vehicles} new vehicles. Done.")
|
||||
return
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse Mercedes vehicle {code}: {e}", exc_info=True)
|
||||
|
||||
current_page += 1
|
||||
if total_pages is not None and current_page >= total_pages:
|
||||
break
|
||||
|
||||
if self.request_delay > 0:
|
||||
time.sleep(self.request_delay)
|
||||
|
||||
def crawl(
|
||||
self,
|
||||
max_vehicles: Optional[int] = None,
|
||||
page_size: int = 50,
|
||||
search_filter: Optional[Dict[str, Any]] = None,
|
||||
) -> Generator[VehicleIngestSchema, None, None]:
|
||||
"""
|
||||
Crawls Mercedes-Benz stock (new, used, or all), yields normalized VehicleIngestSchema objects.
|
||||
"""
|
||||
with httpx.Client(http2=False, timeout=self.timeout, proxy=self.proxy) as client:
|
||||
if self.stock_type == "used":
|
||||
yield from self.crawl_used(
|
||||
client=client,
|
||||
max_vehicles=max_vehicles,
|
||||
page_size=page_size,
|
||||
search_filter=search_filter,
|
||||
)
|
||||
elif self.stock_type == "all":
|
||||
limit_each = (max_vehicles // 2) if max_vehicles else None
|
||||
yield from self.crawl_new(
|
||||
client=client,
|
||||
max_vehicles=limit_each,
|
||||
page_size=page_size,
|
||||
search_filter=search_filter,
|
||||
)
|
||||
yield from self.crawl_used(
|
||||
client=client,
|
||||
max_vehicles=limit_each,
|
||||
page_size=page_size,
|
||||
search_filter=search_filter,
|
||||
)
|
||||
else:
|
||||
yield from self.crawl_new(
|
||||
client=client,
|
||||
max_vehicles=max_vehicles,
|
||||
page_size=page_size,
|
||||
search_filter=search_filter,
|
||||
)
|
||||
|
||||
1008
src/oem_crawler/crawlers/mercedes/parser.py
Normal file
1008
src/oem_crawler/crawlers/mercedes/parser.py
Normal file
File diff suppressed because it is too large
Load diff
1
src/oem_crawler/crawlers/mercedes/query.py
Normal file
1
src/oem_crawler/crawlers/mercedes/query.py
Normal file
File diff suppressed because one or more lines are too long
7
src/oem_crawler/crawlers/porsche/__init__.py
Normal file
7
src/oem_crawler/crawlers/porsche/__init__.py
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
"""
|
||||
Porsche Crawler Package exports.
|
||||
"""
|
||||
from oem_crawler.crawlers.porsche.crawler import PorscheStockCrawler
|
||||
from oem_crawler.crawlers.porsche.parser import parse_porsche_vehicle
|
||||
|
||||
__all__ = ["PorscheStockCrawler", "parse_porsche_vehicle"]
|
||||
266
src/oem_crawler/crawlers/porsche/crawler.py
Normal file
266
src/oem_crawler/crawlers/porsche/crawler.py
Normal file
|
|
@ -0,0 +1,266 @@
|
|||
"""
|
||||
Porsche Finder Stock Crawler Implementation.
|
||||
Interacts with Porsche Finder Next.js React Server Components (RSC) endpoints.
|
||||
Supports both new and used vehicles from the unified backend.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from typing import Any, Dict, Generator, List, Optional, Tuple
|
||||
|
||||
import httpx
|
||||
|
||||
from oem_crawler.crawlers.base import BaseCrawler, CrawlerRegistry
|
||||
from oem_crawler.crawlers.porsche.parser import parse_porsche_vehicle
|
||||
from oem_crawler.db.schemas import VehicleIngestSchema
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@CrawlerRegistry.register("porsche")
|
||||
class PorscheStockCrawler(BaseCrawler):
|
||||
BASE_URL = "https://finder.porsche.com"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
market_country: str = "DE",
|
||||
proxy: Optional[str] = None,
|
||||
request_delay: float = 0.5,
|
||||
timeout: float = 25.0,
|
||||
stock_type: str = "new",
|
||||
fetch_details: bool = True,
|
||||
order: str = "recommended_desc",
|
||||
**kwargs: Any,
|
||||
):
|
||||
super().__init__(
|
||||
market_country=market_country,
|
||||
proxy=proxy,
|
||||
request_delay=request_delay,
|
||||
timeout=timeout,
|
||||
**kwargs,
|
||||
)
|
||||
self.stock_type = stock_type.lower()
|
||||
self.fetch_details = fetch_details
|
||||
self.order = order
|
||||
|
||||
market = self.market_country.lower()
|
||||
lang_tag = f"{market}-{self.market_country.upper()}"
|
||||
self.locale_path = f"{market}/{lang_tag}"
|
||||
|
||||
self.headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
|
||||
"RSC": "1",
|
||||
"Accept": "text/x-component",
|
||||
"Accept-Language": f"{market}-{self.market_country.upper()},{market};q=0.9,en;q=0.8",
|
||||
}
|
||||
|
||||
@property
|
||||
def brand_name(self) -> str:
|
||||
return "PORSCHE"
|
||||
|
||||
def _parse_search_rsc(self, text: str) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
|
||||
"""
|
||||
Extracts the pages pagination block and the items array from the RSC payload
|
||||
using JSONDecoder.raw_decode.
|
||||
"""
|
||||
decoder = json.JSONDecoder()
|
||||
pages: Dict[str, Any] = {"activePage": 1, "totalPages": 1}
|
||||
items: List[Dict[str, Any]] = []
|
||||
|
||||
# 1. Parse pages
|
||||
pos_pages = text.find('"pages":{')
|
||||
if pos_pages != -1:
|
||||
try:
|
||||
pages, _ = decoder.raw_decode(text, pos_pages + 8)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error decoding pages from Porsche RSC: {e}")
|
||||
|
||||
# 2. Parse items
|
||||
pos_items = text.find('"items":[')
|
||||
if pos_items != -1:
|
||||
try:
|
||||
items, _ = decoder.raw_decode(text, pos_items + 8)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error decoding items from Porsche RSC: {e}")
|
||||
|
||||
return pages, items
|
||||
|
||||
def _parse_detail_rsc(self, text: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Extracts equipment, summary (characteristics), and info blocks from detail RSC payload.
|
||||
"""
|
||||
decoder = json.JSONDecoder()
|
||||
detail_data: Dict[str, Any] = {}
|
||||
|
||||
# 1. Equipment
|
||||
pos_eq = text.find('"equipment":{')
|
||||
if pos_eq != -1:
|
||||
try:
|
||||
detail_data["equipment"], _ = decoder.raw_decode(text, pos_eq + 12)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 2. Summary (characteristics, EV specs)
|
||||
pos_sum = text.find('"summary":{')
|
||||
if pos_sum != -1:
|
||||
try:
|
||||
detail_data["summary"], _ = decoder.raw_decode(text, pos_sum + 10)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 3. Info (pricing breakdown, seller, commission number)
|
||||
pos_info = text.find('"info":{')
|
||||
if pos_info != -1:
|
||||
try:
|
||||
detail_data["info"], _ = decoder.raw_decode(text, pos_info + 7)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return detail_data
|
||||
|
||||
def fetch_search_page(
|
||||
self,
|
||||
client: httpx.Client,
|
||||
condition: str = "new",
|
||||
page: int = 1,
|
||||
search_filter: Optional[Dict[str, Any]] = None,
|
||||
max_retries: int = 3,
|
||||
) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
|
||||
"""
|
||||
Fetches one page of search results for a given condition ('new' or 'used').
|
||||
"""
|
||||
url = f"{self.BASE_URL}/{self.locale_path}/search"
|
||||
params: Dict[str, Any] = {
|
||||
"condition": condition,
|
||||
"page": page,
|
||||
"order": self.order,
|
||||
}
|
||||
if search_filter:
|
||||
params.update(search_filter)
|
||||
|
||||
for attempt in range(1, max_retries + 1):
|
||||
try:
|
||||
time.sleep(self.request_delay)
|
||||
logger.info(f"Fetching Porsche {condition} stock page {page}...")
|
||||
resp = client.get(url, params=params, headers=self.headers)
|
||||
resp.raise_for_status()
|
||||
return self._parse_search_rsc(resp.text)
|
||||
except httpx.HTTPError as e:
|
||||
logger.warning(f"Attempt {attempt}/{max_retries} failed for Porsche {condition} page {page}: {e}")
|
||||
if attempt == max_retries:
|
||||
raise
|
||||
time.sleep(attempt * 1.5)
|
||||
|
||||
return {"activePage": page, "totalPages": 0}, []
|
||||
|
||||
def fetch_detail(
|
||||
self,
|
||||
client: httpx.Client,
|
||||
listing_url_slug: str,
|
||||
max_retries: int = 2,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Fetches vehicle detail RSC payload for factory equipment and EV specs.
|
||||
"""
|
||||
url = f"{self.BASE_URL}/{self.locale_path}/details/{listing_url_slug}"
|
||||
for attempt in range(1, max_retries + 1):
|
||||
try:
|
||||
time.sleep(self.request_delay)
|
||||
resp = client.get(url, headers=self.headers)
|
||||
resp.raise_for_status()
|
||||
return self._parse_detail_rsc(resp.text)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to fetch detail for {listing_url_slug} (attempt {attempt}): {e}")
|
||||
if attempt == max_retries:
|
||||
return None
|
||||
time.sleep(1.0)
|
||||
return None
|
||||
|
||||
def _crawl_condition(
|
||||
self,
|
||||
client: httpx.Client,
|
||||
condition: str,
|
||||
max_vehicles: Optional[int],
|
||||
search_filter: Optional[Dict[str, Any]],
|
||||
) -> Generator[VehicleIngestSchema, None, None]:
|
||||
"""
|
||||
Streams vehicles for a specific condition ('new' or 'used').
|
||||
"""
|
||||
count = 0
|
||||
current_page = 1
|
||||
total_pages = 1
|
||||
|
||||
logger.info(f"Starting Porsche {condition.upper()} stock crawl (Market: {self.market_country})...")
|
||||
|
||||
while current_page <= total_pages:
|
||||
pages_info, items = self.fetch_search_page(
|
||||
client=client,
|
||||
condition=condition,
|
||||
page=current_page,
|
||||
search_filter=search_filter,
|
||||
)
|
||||
|
||||
total_pages = pages_info.get("totalPages", total_pages)
|
||||
if not items:
|
||||
logger.info(f"No items found on page {current_page}. Ending {condition} crawl.")
|
||||
break
|
||||
|
||||
logger.info(f"Porsche {condition} reports {len(items)} items on page {current_page}/{total_pages}.")
|
||||
|
||||
for item in items:
|
||||
detail_data = None
|
||||
slug = item.get("listingUrlSlug")
|
||||
if self.fetch_details and slug:
|
||||
detail_data = self.fetch_detail(client, slug)
|
||||
|
||||
try:
|
||||
vehicle = parse_porsche_vehicle(
|
||||
search_item=item,
|
||||
detail_data=detail_data,
|
||||
market_country=self.market_country,
|
||||
requested_stock_type=condition,
|
||||
)
|
||||
yield vehicle
|
||||
count += 1
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse Porsche vehicle {item.get('id')}: {e}", exc_info=True)
|
||||
|
||||
if max_vehicles and count >= max_vehicles:
|
||||
logger.info(f"Reached limit of {max_vehicles} {condition} vehicles.")
|
||||
return
|
||||
|
||||
current_page += 1
|
||||
|
||||
def crawl(
|
||||
self,
|
||||
max_vehicles: Optional[int] = None,
|
||||
page_size: int = 15,
|
||||
search_filter: Optional[Dict[str, Any]] = None,
|
||||
) -> Generator[VehicleIngestSchema, None, None]:
|
||||
"""
|
||||
Main crawl generator: supports stock_type in ('new', 'used', 'all').
|
||||
"""
|
||||
client_kwargs: Dict[str, Any] = {
|
||||
"timeout": self.timeout,
|
||||
"http2": False,
|
||||
}
|
||||
if self.proxy:
|
||||
client_kwargs["proxy"] = self.proxy
|
||||
|
||||
with httpx.Client(**client_kwargs) as client:
|
||||
if self.stock_type == "all":
|
||||
half_limit = (max_vehicles // 2) if max_vehicles else None
|
||||
new_count = 0
|
||||
for veh in self._crawl_condition(client, "new", half_limit, search_filter):
|
||||
yield veh
|
||||
new_count += 1
|
||||
|
||||
rem_limit = (max_vehicles - new_count) if max_vehicles else None
|
||||
for veh in self._crawl_condition(client, "used", rem_limit, search_filter):
|
||||
yield veh
|
||||
elif self.stock_type == "used":
|
||||
yield from self._crawl_condition(client, "used", max_vehicles, search_filter)
|
||||
else:
|
||||
yield from self._crawl_condition(client, "new", max_vehicles, search_filter)
|
||||
392
src/oem_crawler/crawlers/porsche/parser.py
Normal file
392
src/oem_crawler/crawlers/porsche/parser.py
Normal file
|
|
@ -0,0 +1,392 @@
|
|||
"""
|
||||
Parser and normalization logic for Porsche Finder responses.
|
||||
Handles search RSC payloads and detail RSC payloads.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from oem_crawler.config import settings
|
||||
from oem_crawler.db.schemas import (
|
||||
BatteryCapacityUnit,
|
||||
DrivetrainType,
|
||||
FuelType,
|
||||
OptionSchema,
|
||||
TransmissionType,
|
||||
VehicleCondition,
|
||||
VehicleIngestSchema,
|
||||
VehicleStatus,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def safe_decimal(val: Any) -> Optional[Decimal]:
|
||||
"""Safely extracts a Decimal from numeric, string, or dict values."""
|
||||
if val is None:
|
||||
return None
|
||||
if isinstance(val, dict):
|
||||
val = val.get("value") or val.get("amount") or val.get("rawValue")
|
||||
if val is None:
|
||||
return None
|
||||
try:
|
||||
val_str = str(val).replace(".", "").replace(",", ".").replace("€", "").replace("\xa0", "").strip()
|
||||
# Handle cases where number was already float/decimal with single decimal point
|
||||
# e.g. "131063" or "220231.42"
|
||||
# If original had a single dot and no comma:
|
||||
raw_s = str(val).strip().replace("€", "").replace("\xa0", "").strip()
|
||||
if "," not in raw_s and "." in raw_s:
|
||||
return Decimal(raw_s)
|
||||
if not val_str:
|
||||
return None
|
||||
return Decimal(val_str)
|
||||
except Exception:
|
||||
try:
|
||||
return Decimal(str(val).strip())
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def safe_int(val: Any) -> Optional[int]:
|
||||
"""Safely extracts an integer."""
|
||||
if val is None:
|
||||
return None
|
||||
if isinstance(val, dict):
|
||||
val = val.get("value")
|
||||
try:
|
||||
val_str = str(val).split(".")[0].strip()
|
||||
return int(val_str)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def parse_date(val: Any) -> Optional[date]:
|
||||
"""Parses date string in ISO format YYYY-MM-DD."""
|
||||
if not val or not isinstance(val, str) or val == "$undefined":
|
||||
return None
|
||||
try:
|
||||
clean = val.split("T")[0].strip()
|
||||
return datetime.strptime(clean, "%Y-%m-%d").date()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def map_porsche_fuel_type(engine_type_raw: Optional[str]) -> FuelType:
|
||||
raw = (engine_type_raw or "").upper()
|
||||
if "ELECTRIC" in raw or "ELEKTRO" in raw:
|
||||
return FuelType.BEV
|
||||
if "PLUG" in raw or "PHEV" in raw or "HYBRID" in raw and "MILD" not in raw:
|
||||
return FuelType.PHEV
|
||||
if "MILD" in raw or "MHEV" in raw:
|
||||
return FuelType.MHEV
|
||||
if "DIESEL" in raw:
|
||||
return FuelType.DIESEL
|
||||
if "PETROL" in raw or "BENZIN" in raw or "GASOLINE" in raw:
|
||||
return FuelType.PETROL
|
||||
return FuelType.PETROL
|
||||
|
||||
|
||||
def map_porsche_transmission(trans_val: Optional[str]) -> Optional[TransmissionType]:
|
||||
raw = (trans_val or "").upper()
|
||||
if "PDK" in raw or "AUTOMATIK" in raw or "AUTOMATIC" in raw or "AUT" in raw:
|
||||
return TransmissionType.AUTOMATIC
|
||||
if "MANUELL" in raw or "MANUAL" in raw or "SCHALT" in raw:
|
||||
return TransmissionType.MANUAL
|
||||
return None
|
||||
|
||||
|
||||
def map_porsche_drivetrain(dt_val: Optional[str]) -> Optional[DrivetrainType]:
|
||||
raw = (dt_val or "").upper()
|
||||
if "ALL_WHEEL" in raw or "ALLRAD" in raw or "AWD" in raw or "4S" in raw or "4" in raw and "DRIVE" in raw:
|
||||
return DrivetrainType.AWD
|
||||
if "REAR" in raw or "HECK" in raw or "RWD" in raw:
|
||||
return DrivetrainType.RWD
|
||||
if "FRONT" in raw or "FWD" in raw:
|
||||
return DrivetrainType.FWD
|
||||
return None
|
||||
|
||||
|
||||
def map_porsche_series(model_category: Optional[str], title: Optional[str]) -> str:
|
||||
cat = (model_category or "").strip()
|
||||
tit = (title or "").lower()
|
||||
|
||||
if cat:
|
||||
cl = cat.lower()
|
||||
if "911" in cl:
|
||||
return "911"
|
||||
if "taycan" in cl:
|
||||
return "Taycan"
|
||||
if "macan" in cl:
|
||||
return "Macan"
|
||||
if "cayenne" in cl:
|
||||
return "Cayenne"
|
||||
if "panamera" in cl:
|
||||
return "Panamera"
|
||||
if "718" in cl or "boxster" in cl or "cayman" in cl:
|
||||
return "718"
|
||||
return cat.capitalize()
|
||||
|
||||
for s in ["911", "Taycan", "Macan", "Cayenne", "Panamera", "718"]:
|
||||
if s.lower() in tit:
|
||||
return s
|
||||
return "Porsche"
|
||||
|
||||
|
||||
def parse_porsche_vehicle(
|
||||
search_item: Dict[str, Any],
|
||||
detail_data: Optional[Dict[str, Any]] = None,
|
||||
market_country: str = "DE",
|
||||
requested_stock_type: str = "new",
|
||||
include_raw: Optional[bool] = None,
|
||||
) -> VehicleIngestSchema:
|
||||
"""
|
||||
Parses a Porsche vehicle from search item and optional detail payload.
|
||||
"""
|
||||
if include_raw is None:
|
||||
include_raw = settings.ENABLE_RAW_EXTRACTION
|
||||
meta = search_item.get("meta") or {}
|
||||
car_meta = search_item.get("dataLayerListingMeta", {}).get("car", {})
|
||||
seller = meta.get("seller") or {}
|
||||
addr_comp = seller.get("addressComponents") or {}
|
||||
desc = search_item.get("description") or {}
|
||||
|
||||
oem_id = str(search_item.get("id") or "").strip()
|
||||
title = meta.get("title") or desc.get("title") or "Porsche"
|
||||
series = map_porsche_series(meta.get("modelCategory"), title)
|
||||
|
||||
# Condition determination
|
||||
cond_raw = (meta.get("condition") or "").lower()
|
||||
is_approved = car_meta.get("isPorscheApproved", False) or "approved" in (desc.get("subtitle") or "").lower()
|
||||
|
||||
if requested_stock_type == "new" or cond_raw == "new":
|
||||
condition = VehicleCondition.NEW
|
||||
elif is_approved:
|
||||
condition = VehicleCondition.CERTIFIED_PRE_OWNED
|
||||
else:
|
||||
condition = VehicleCondition.USED
|
||||
|
||||
# Powertrain & Specs
|
||||
fuel_type = map_porsche_fuel_type(meta.get("engineType"))
|
||||
transmission = map_porsche_transmission(meta.get("transmission"))
|
||||
drivetrain = map_porsche_drivetrain(meta.get("drivetrain"))
|
||||
|
||||
power_kw = safe_int(meta.get("enginePower"))
|
||||
power_hp = int(round(power_kw * 1.35962)) if power_kw else None
|
||||
disp_cc = safe_int(meta.get("engineDisplacement"))
|
||||
|
||||
mileage_val = safe_int(meta.get("mileage", {}).get("value"))
|
||||
mileage_km = mileage_val if mileage_val is not None else 0
|
||||
|
||||
first_reg = parse_date(meta.get("firstRegistration"))
|
||||
prev_owners = safe_int(meta.get("numberOfPreviousOwners"))
|
||||
model_year = safe_int(meta.get("modelYear")) or safe_int(car_meta.get("modelModelYear"))
|
||||
|
||||
# Pricing
|
||||
price_gross = safe_decimal(meta.get("priceValue")) or safe_decimal(car_meta.get("priceTotalTotal"))
|
||||
currency = meta.get("priceCurrency") or "EUR"
|
||||
|
||||
# Monthly payment extraction
|
||||
monthly_rate = None
|
||||
periodic_payments = desc.get("periodicPayments") or {}
|
||||
payment_label = periodic_payments.get("paymentLabel") if isinstance(periodic_payments, dict) else str(periodic_payments)
|
||||
if payment_label:
|
||||
# e.g. "765.26 per month" or "2808 per month"
|
||||
rate_match = re.search(r"([0-9\.]+)", str(payment_label))
|
||||
if rate_match:
|
||||
try:
|
||||
monthly_rate = Decimal(rate_match.group(1))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Dealer / Location
|
||||
dealer_name = seller.get("name") or desc.get("seller", {}).get("name")
|
||||
dealer_id = str(seller.get("sellerId") or seller.get("porschePartnerNumber") or "").strip() or None
|
||||
postal_code = addr_comp.get("postalCode")
|
||||
city = addr_comp.get("city")
|
||||
|
||||
# Images
|
||||
gallery = search_item.get("gallery") or {}
|
||||
image_ids = gallery.get("imageIds") or []
|
||||
image_urls = [f"https://images.finder.porsche.com/{img_id}/960.webp" for img_id in image_ids]
|
||||
thumb = meta.get("imageUrl") or (image_urls[0] if image_urls else None)
|
||||
|
||||
slug = search_item.get("listingUrlSlug") or f"porsche-{series.lower()}-{oem_id}"
|
||||
vehicle_url = f"https://finder.porsche.com/de/de-DE/details/{slug}"
|
||||
|
||||
# Initialize Option C fields
|
||||
battery_capacity = None
|
||||
battery_capacity_unit = None
|
||||
charging_power_ac_kw = None
|
||||
charging_power_dc_kw = None
|
||||
charging_time_dc_minutes = None
|
||||
electric_range_wltp_km = None
|
||||
options: List[OptionSchema] = []
|
||||
|
||||
# Enrich from detail payload if available
|
||||
raw_payload: Dict[str, Any] = {"search_item": search_item} if include_raw else {}
|
||||
if detail_data and include_raw:
|
||||
raw_payload["detail_data"] = detail_data
|
||||
|
||||
# Characteristics (EV Specs, Engine, etc.)
|
||||
summary = detail_data.get("summary") or {}
|
||||
characteristics = summary.get("characteristics") or []
|
||||
for ch in characteristics:
|
||||
title_ch = (ch.get("title") or "").strip()
|
||||
val_ch = (ch.get("value") or "").strip()
|
||||
|
||||
# Battery Capacity
|
||||
if "Energieinhalt Batterie" in title_ch or "Batteriekapazität" in title_ch:
|
||||
cap_match = re.search(r"([0-9]+[,\.][0-9]+|[0-9]+)\s*kWh", val_ch, re.I)
|
||||
if cap_match:
|
||||
try:
|
||||
clean_cap = cap_match.group(1).replace(",", ".")
|
||||
battery_capacity = Decimal(clean_cap)
|
||||
battery_capacity_unit = BatteryCapacityUnit.KWH
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# WLTP Range
|
||||
elif "Reichweite" in title_ch and "WLTP" in title_ch:
|
||||
range_match = re.search(r"(\d+)\s*km", val_ch)
|
||||
if range_match:
|
||||
try:
|
||||
electric_range_wltp_km = int(range_match.group(1))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# DC Charging Time
|
||||
elif "Gleichstrom (DC)" in title_ch or "Ladezeit" in title_ch and "DC" in title_ch:
|
||||
time_match = re.search(r"([0-9]+[,\.][0-9]+|[0-9]+)\s*min", val_ch)
|
||||
if time_match:
|
||||
try:
|
||||
clean_time = time_match.group(1).replace(",", ".").split(".")[0]
|
||||
charging_time_dc_minutes = int(clean_time)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Options / Special Equipment
|
||||
equipment = detail_data.get("equipment") or {}
|
||||
special_eq = equipment.get("specialEquipment") or []
|
||||
for cat in special_eq:
|
||||
cat_name = cat.get("name") or "Sonderausstattung"
|
||||
for item in cat.get("items") or []:
|
||||
code = item.get("code")
|
||||
name = item.get("name") or code or "Ausstattung"
|
||||
description = item.get("description")
|
||||
options.append(
|
||||
OptionSchema(
|
||||
option_code=code,
|
||||
name=name,
|
||||
category=cat_name,
|
||||
description=description,
|
||||
is_standard=False,
|
||||
)
|
||||
)
|
||||
|
||||
# Check for AC charging upgrades (e.g. 22 kW On-Board AC-Lader)
|
||||
if code == "KB3" or "22 kW" in name or description and "22 kW AC" in description:
|
||||
charging_power_ac_kw = Decimal("22.00")
|
||||
elif "11 kW" in name or description and "11 kW AC" in description:
|
||||
if not charging_power_ac_kw:
|
||||
charging_power_ac_kw = Decimal("11.00")
|
||||
|
||||
# Important highlight options
|
||||
important_eq = equipment.get("importantEquipmentOptions") or []
|
||||
existing_codes = {opt.option_code for opt in options if opt.option_code}
|
||||
for imp in important_eq:
|
||||
val = imp.get("value")
|
||||
lbl = imp.get("label") or val
|
||||
if val and val not in existing_codes:
|
||||
options.append(
|
||||
OptionSchema(
|
||||
option_code=val,
|
||||
name=lbl,
|
||||
category="Highlights",
|
||||
is_standard=False,
|
||||
)
|
||||
)
|
||||
|
||||
# Standard Equipment
|
||||
standard_eq = equipment.get("standardEquipment") or []
|
||||
for cat in standard_eq:
|
||||
cat_name = cat.get("name") or "Serienausstattung"
|
||||
for item in cat.get("items") or []:
|
||||
name = item.get("name") or "Serienausstattung"
|
||||
options.append(
|
||||
OptionSchema(
|
||||
option_code=None,
|
||||
name=name,
|
||||
category=cat_name,
|
||||
is_standard=True,
|
||||
)
|
||||
)
|
||||
|
||||
# If EV and default charging power wasn't overridden, standard Taycan / Macan EV AC is 11 kW
|
||||
if fuel_type == FuelType.BEV:
|
||||
if not charging_power_ac_kw:
|
||||
charging_power_ac_kw = Decimal("11.00")
|
||||
if not charging_power_dc_kw:
|
||||
# Taycan II / Macan EV supports up to 270 kW / 320 kW DC fast charging
|
||||
if series == "Taycan":
|
||||
charging_power_dc_kw = Decimal("320.00") if "II" in (meta.get("model") or "") else Decimal("270.00")
|
||||
elif series == "Macan":
|
||||
charging_power_dc_kw = Decimal("270.00")
|
||||
|
||||
# VIN resolution
|
||||
vin = meta.get("vin")
|
||||
if not vin or vin == "$undefined":
|
||||
info = detail_data.get("info") or {} if detail_data else {}
|
||||
vin = info.get("vin")
|
||||
if not vin or vin == "$undefined" or not vin.strip():
|
||||
vin = None
|
||||
|
||||
return VehicleIngestSchema(
|
||||
brand="PORSCHE",
|
||||
oem_vehicle_id=oem_id,
|
||||
vin=vin,
|
||||
series=series,
|
||||
model_name=title,
|
||||
trim_line=meta.get("model") or None,
|
||||
body_type=meta.get("bodyType") or None,
|
||||
condition=condition,
|
||||
model_year=model_year,
|
||||
first_registration_date=first_reg,
|
||||
previous_owners_count=prev_owners,
|
||||
fuel_type=fuel_type,
|
||||
transmission=transmission,
|
||||
drivetrain=drivetrain,
|
||||
power_kw=power_kw,
|
||||
power_hp=power_hp,
|
||||
engine_displacement_cc=disp_cc,
|
||||
battery_capacity=battery_capacity,
|
||||
battery_capacity_unit=battery_capacity_unit,
|
||||
charging_power_ac_kw=charging_power_ac_kw,
|
||||
charging_power_dc_kw=charging_power_dc_kw,
|
||||
charging_time_dc_minutes=charging_time_dc_minutes,
|
||||
electric_range_wltp_km=electric_range_wltp_km,
|
||||
consumption_combined=safe_decimal(meta.get("fuelConsumption")),
|
||||
co2_emissions_g_km=safe_decimal(meta.get("co2Emissions")),
|
||||
mileage_km=mileage_km,
|
||||
exterior_color=meta.get("color"),
|
||||
exterior_color_code=car_meta.get("exteriorColorCode"),
|
||||
interior_color=meta.get("interiorColor"),
|
||||
interior_material=meta.get("interiorName"),
|
||||
currency=currency,
|
||||
price_gross=price_gross,
|
||||
monthly_rate=monthly_rate,
|
||||
status=VehicleStatus.IN_STOCK,
|
||||
dealer_name=dealer_name,
|
||||
dealer_id=dealer_id,
|
||||
location_city=city,
|
||||
location_postal_code=postal_code,
|
||||
location_country=market_country,
|
||||
vehicle_url=vehicle_url,
|
||||
image_urls=image_urls,
|
||||
thumbnail_url=thumb,
|
||||
options=options,
|
||||
raw_payload=raw_payload,
|
||||
)
|
||||
279
src/oem_crawler/db/models.py
Normal file
279
src/oem_crawler/db/models.py
Normal file
|
|
@ -0,0 +1,279 @@
|
|||
"""
|
||||
SQLAlchemy 2.0 ORM Models for OEM Vehicle Crawler.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import date, datetime, timezone
|
||||
from decimal import Decimal
|
||||
from typing import List, Optional
|
||||
|
||||
from sqlalchemy import (
|
||||
Boolean,
|
||||
Date,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
JSON,
|
||||
Numeric,
|
||||
SmallInteger,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
Uuid,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSONB, UUID as PGUUID
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
|
||||
|
||||
from oem_crawler.db.schemas import (
|
||||
CrawlStage,
|
||||
CrawlStatus,
|
||||
FuelType,
|
||||
LogLevel,
|
||||
VehicleCondition,
|
||||
VehicleStatus,
|
||||
)
|
||||
|
||||
# Cross-dialect JSON & UUID types (PostgreSQL JSONB / SQLite JSON fallback)
|
||||
CompatibleJSON = JSONB().with_variant(JSON(), "sqlite")
|
||||
CompatibleUUID = PGUUID(as_uuid=True).with_variant(Uuid(as_uuid=True), "sqlite")
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class CrawlRun(Base):
|
||||
__tablename__ = "crawl_runs"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(
|
||||
CompatibleUUID, primary_key=True, default=uuid.uuid4
|
||||
)
|
||||
brand: Mapped[str] = mapped_column(String(50), nullable=False)
|
||||
market_country: Mapped[str] = mapped_column(String(2), default="DE", nullable=False)
|
||||
crawler_name: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||
crawler_version: Mapped[Optional[str]] = mapped_column(String(20), nullable=True)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(20), default=CrawlStatus.PENDING.value, nullable=False
|
||||
)
|
||||
started_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False
|
||||
)
|
||||
finished_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
duration_ms: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
|
||||
|
||||
items_discovered: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||
items_inserted: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||
items_updated: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||
items_unmodified: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||
items_delisted: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||
items_failed: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||
|
||||
error_summary: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
config_snapshot: Mapped[dict] = mapped_column(CompatibleJSON, default=dict, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False
|
||||
)
|
||||
|
||||
# Relationships
|
||||
vehicles: Mapped[List["Vehicle"]] = relationship("Vehicle", back_populates="last_crawl_run")
|
||||
price_history: Mapped[List["VehiclePriceHistory"]] = relationship("VehiclePriceHistory", back_populates="crawl_run")
|
||||
logs: Mapped[List["CrawlLog"]] = relationship("CrawlLog", back_populates="crawl_run")
|
||||
|
||||
|
||||
class Vehicle(Base):
|
||||
__tablename__ = "vehicles"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("brand", "location_country", "oem_vehicle_id", name="uq_vehicles_brand_market_oem_id"),
|
||||
Index("idx_vehicles_brand_series", "brand", "series"),
|
||||
Index("idx_vehicles_fuel_transmission", "fuel_type", "transmission"),
|
||||
Index("idx_vehicles_price_gross", "price_gross"),
|
||||
Index("idx_vehicles_status_last_seen", "status", "last_seen_at"),
|
||||
)
|
||||
|
||||
# Identifiers
|
||||
id: Mapped[uuid.UUID] = mapped_column(
|
||||
CompatibleUUID, primary_key=True, default=uuid.uuid4
|
||||
)
|
||||
brand: Mapped[str] = mapped_column(String(50), nullable=False)
|
||||
oem_vehicle_id: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||
vin: Mapped[Optional[str]] = mapped_column(String(17), nullable=True)
|
||||
|
||||
# Classification
|
||||
series: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||
model_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
trim_line: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)
|
||||
body_type: Mapped[Optional[str]] = mapped_column(String(50), nullable=True)
|
||||
condition: Mapped[str] = mapped_column(String(30), default=VehicleCondition.NEW.value, nullable=False)
|
||||
model_year: Mapped[Optional[int]] = mapped_column(SmallInteger, nullable=True)
|
||||
first_registration_date: Mapped[Optional[date]] = mapped_column(Date, nullable=True)
|
||||
previous_owners_count: Mapped[Optional[int]] = mapped_column(SmallInteger, nullable=True)
|
||||
|
||||
# Technical Specs & Powertrain
|
||||
fuel_type: Mapped[str] = mapped_column(String(30), default=FuelType.PETROL.value, nullable=False)
|
||||
transmission: Mapped[Optional[str]] = mapped_column(String(30), nullable=True)
|
||||
drivetrain: Mapped[Optional[str]] = mapped_column(String(20), nullable=True)
|
||||
power_kw: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
|
||||
power_hp: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
|
||||
engine_displacement_cc: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
|
||||
battery_capacity: Mapped[Optional[Decimal]] = mapped_column(Numeric(6, 2), nullable=True)
|
||||
battery_capacity_unit: Mapped[Optional[str]] = mapped_column(String(10), nullable=True)
|
||||
battery_capacity_gross_kwh: Mapped[Optional[Decimal]] = mapped_column(Numeric(6, 2), nullable=True)
|
||||
battery_capacity_net_kwh: Mapped[Optional[Decimal]] = mapped_column(Numeric(6, 2), nullable=True)
|
||||
charging_power_ac_kw: Mapped[Optional[Decimal]] = mapped_column(Numeric(5, 2), nullable=True)
|
||||
charging_power_dc_kw: Mapped[Optional[Decimal]] = mapped_column(Numeric(5, 2), nullable=True)
|
||||
charging_time_dc_minutes: Mapped[Optional[int]] = mapped_column(SmallInteger, nullable=True)
|
||||
electric_range_wltp_km: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
|
||||
consumption_combined: Mapped[Optional[Decimal]] = mapped_column(Numeric(5, 2), nullable=True)
|
||||
co2_emissions_g_km: Mapped[Optional[Decimal]] = mapped_column(Numeric(5, 1), nullable=True)
|
||||
mileage_km: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||
|
||||
# Colors & Interior
|
||||
exterior_color: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)
|
||||
exterior_color_code: Mapped[Optional[str]] = mapped_column(String(50), nullable=True)
|
||||
exterior_color_generic: Mapped[Optional[str]] = mapped_column(String(50), nullable=True)
|
||||
interior_color: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)
|
||||
interior_material: Mapped[Optional[str]] = mapped_column(String(50), nullable=True)
|
||||
|
||||
# Pricing & Availability
|
||||
currency: Mapped[str] = mapped_column(String(3), default="EUR", nullable=False)
|
||||
base_price_gross: Mapped[Optional[Decimal]] = mapped_column(Numeric(12, 2), nullable=True)
|
||||
options_price_gross: Mapped[Optional[Decimal]] = mapped_column(Numeric(12, 2), nullable=True)
|
||||
price_gross: Mapped[Decimal] = mapped_column(Numeric(12, 2), nullable=False)
|
||||
price_net: Mapped[Optional[Decimal]] = mapped_column(Numeric(12, 2), nullable=True)
|
||||
msrp_gross: Mapped[Optional[Decimal]] = mapped_column(Numeric(12, 2), nullable=True)
|
||||
discount_percentage: Mapped[Optional[Decimal]] = mapped_column(Numeric(5, 2), nullable=True)
|
||||
vat_deductible: Mapped[Optional[bool]] = mapped_column(Boolean, nullable=True)
|
||||
monthly_rate: Mapped[Optional[Decimal]] = mapped_column(Numeric(10, 2), nullable=True)
|
||||
leasing_details: Mapped[dict] = mapped_column(CompatibleJSON, default=dict, nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(30), default=VehicleStatus.IN_STOCK.value, nullable=False)
|
||||
availability_date: Mapped[Optional[date]] = mapped_column(Date, nullable=True)
|
||||
delivery_time_days: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
|
||||
|
||||
# Location & Media
|
||||
dealer_name: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
|
||||
dealer_id: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)
|
||||
location_city: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)
|
||||
location_postal_code: Mapped[Optional[str]] = mapped_column(String(20), nullable=True)
|
||||
location_country: Mapped[str] = mapped_column(String(2), default="DE", nullable=False)
|
||||
latitude: Mapped[Optional[Decimal]] = mapped_column(Numeric(9, 6), nullable=True)
|
||||
longitude: Mapped[Optional[Decimal]] = mapped_column(Numeric(9, 6), nullable=True)
|
||||
vehicle_url: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
image_urls: Mapped[list] = mapped_column(CompatibleJSON, default=list, nullable=False)
|
||||
thumbnail_url: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
|
||||
# Timestamps & Tracking
|
||||
first_seen_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False
|
||||
)
|
||||
last_seen_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False
|
||||
)
|
||||
delisted_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_crawl_run_id: Mapped[Optional[uuid.UUID]] = mapped_column(
|
||||
CompatibleUUID, ForeignKey("crawl_runs.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
content_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
raw_payload: Mapped[dict] = mapped_column(CompatibleJSON, default=dict, nullable=False)
|
||||
raw_payload_path: Mapped[Optional[str]] = mapped_column(String(512), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False
|
||||
)
|
||||
|
||||
# Relationships
|
||||
options: Mapped[List["VehicleOption"]] = relationship(
|
||||
"VehicleOption", back_populates="vehicle", cascade="all, delete-orphan"
|
||||
)
|
||||
price_history: Mapped[List["VehiclePriceHistory"]] = relationship(
|
||||
"VehiclePriceHistory", back_populates="vehicle", cascade="all, delete-orphan"
|
||||
)
|
||||
last_crawl_run: Mapped[Optional["CrawlRun"]] = relationship("CrawlRun", back_populates="vehicles")
|
||||
|
||||
def get_raw_payload(self) -> dict:
|
||||
"""Loads raw payload from external compressed storage if path is set, else returns raw_payload."""
|
||||
if self.raw_payload_path:
|
||||
from oem_crawler.storage import RawStorageManager
|
||||
data = RawStorageManager.load_payload(self.raw_payload_path)
|
||||
if data is not None:
|
||||
return data
|
||||
return self.raw_payload or {}
|
||||
|
||||
|
||||
class VehicleOption(Base):
|
||||
__tablename__ = "vehicle_options"
|
||||
__table_args__ = (
|
||||
Index("idx_vehicle_options_vehicle_id", "vehicle_id"),
|
||||
Index("idx_vehicle_options_code", "option_code"),
|
||||
Index("idx_vehicle_options_category", "category"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
vehicle_id: Mapped[uuid.UUID] = mapped_column(
|
||||
CompatibleUUID, ForeignKey("vehicles.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
option_code: Mapped[Optional[str]] = mapped_column(String(50), nullable=True)
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
category: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)
|
||||
price_gross: Mapped[Optional[Decimal]] = mapped_column(Numeric(10, 2), nullable=True)
|
||||
price_net: Mapped[Optional[Decimal]] = mapped_column(Numeric(10, 2), nullable=True)
|
||||
is_standard: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
|
||||
description: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
raw_data: Mapped[dict] = mapped_column(CompatibleJSON, default=dict, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False
|
||||
)
|
||||
|
||||
vehicle: Mapped["Vehicle"] = relationship("Vehicle", back_populates="options")
|
||||
|
||||
|
||||
class VehiclePriceHistory(Base):
|
||||
__tablename__ = "vehicle_price_history"
|
||||
__table_args__ = (
|
||||
Index("idx_price_history_vehicle_recorded", "vehicle_id", "recorded_at"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
vehicle_id: Mapped[uuid.UUID] = mapped_column(
|
||||
CompatibleUUID, ForeignKey("vehicles.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
crawl_run_id: Mapped[uuid.UUID] = mapped_column(
|
||||
CompatibleUUID, ForeignKey("crawl_runs.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
price_gross: Mapped[Decimal] = mapped_column(Numeric(12, 2), nullable=False)
|
||||
price_net: Mapped[Optional[Decimal]] = mapped_column(Numeric(12, 2), nullable=True)
|
||||
monthly_rate: Mapped[Optional[Decimal]] = mapped_column(Numeric(10, 2), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(30), nullable=False)
|
||||
recorded_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False
|
||||
)
|
||||
|
||||
vehicle: Mapped["Vehicle"] = relationship("Vehicle", back_populates="price_history")
|
||||
crawl_run: Mapped["CrawlRun"] = relationship("CrawlRun", back_populates="price_history")
|
||||
|
||||
|
||||
class CrawlLog(Base):
|
||||
__tablename__ = "crawl_logs"
|
||||
__table_args__ = (
|
||||
Index("idx_crawl_logs_run_level", "crawl_run_id", "level"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
crawl_run_id: Mapped[uuid.UUID] = mapped_column(
|
||||
CompatibleUUID, ForeignKey("crawl_runs.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
vehicle_id: Mapped[Optional[uuid.UUID]] = mapped_column(
|
||||
CompatibleUUID, ForeignKey("vehicles.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
level: Mapped[str] = mapped_column(String(10), default=LogLevel.INFO.value, nullable=False)
|
||||
stage: Mapped[str] = mapped_column(String(50), default=CrawlStage.GENERAL.value, nullable=False)
|
||||
message: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
error_context: Mapped[dict] = mapped_column(CompatibleJSON, default=dict, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False
|
||||
)
|
||||
|
||||
crawl_run: Mapped["CrawlRun"] = relationship("CrawlRun", back_populates="logs")
|
||||
413
src/oem_crawler/db/pipeline.py
Normal file
413
src/oem_crawler/db/pipeline.py
Normal file
|
|
@ -0,0 +1,413 @@
|
|||
"""
|
||||
OEM Vehicle Stock Ingestion Pipeline
|
||||
Handles database persistence, change detection, price history, delisting detection, and audit logging.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Generator, List, Optional
|
||||
|
||||
from sqlalchemy import select, create_engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from oem_crawler.config import settings
|
||||
from oem_crawler.db.models import (
|
||||
Base,
|
||||
CrawlLog,
|
||||
CrawlRun,
|
||||
Vehicle,
|
||||
VehicleOption,
|
||||
VehiclePriceHistory,
|
||||
)
|
||||
from oem_crawler.db.schemas import (
|
||||
CrawlStage,
|
||||
CrawlStatus,
|
||||
LogLevel,
|
||||
VehicleCondition,
|
||||
VehicleIngestSchema,
|
||||
VehicleStatus,
|
||||
compute_content_hash,
|
||||
)
|
||||
from oem_crawler.db.session import SessionFactory, ensure_sqlite_dir, init_db
|
||||
from oem_crawler.storage import RawStorageManager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StockIngestionPipeline:
|
||||
def __init__(self, session_factory=None, db_url: Optional[str] = None):
|
||||
if db_url:
|
||||
ensure_sqlite_dir(db_url)
|
||||
custom_engine = create_engine(db_url)
|
||||
self.session_factory = sessionmaker(bind=custom_engine, autoflush=False, autocommit=False)
|
||||
Base.metadata.create_all(custom_engine)
|
||||
elif session_factory:
|
||||
self.session_factory = session_factory
|
||||
else:
|
||||
self.session_factory = SessionFactory
|
||||
init_db()
|
||||
|
||||
def run_crawler_pipeline(
|
||||
self,
|
||||
brand: str,
|
||||
crawler_name: str,
|
||||
vehicle_stream: Generator[VehicleIngestSchema, None, None],
|
||||
market_country: str = "DE",
|
||||
config_snapshot: Optional[dict] = None,
|
||||
) -> CrawlRun:
|
||||
"""
|
||||
Executes a complete crawl ingestion run:
|
||||
1. Creates CrawlRun audit record (RUNNING)
|
||||
2. Streams vehicles, performs change-detection via content_hash
|
||||
3. Inserts new vehicles / options / price history
|
||||
4. Updates existing records on spec/price change
|
||||
5. Detects delisted vehicles (if full market scan)
|
||||
6. Finalizes CrawlRun (COMPLETED/FAILED)
|
||||
"""
|
||||
session: Session = self.session_factory()
|
||||
start_time = datetime.now(timezone.utc)
|
||||
|
||||
# 1. Initialize Crawl Run
|
||||
crawl_run = CrawlRun(
|
||||
id=uuid.uuid4(),
|
||||
brand=brand,
|
||||
market_country=market_country,
|
||||
crawler_name=crawler_name,
|
||||
status=CrawlStatus.RUNNING.value,
|
||||
started_at=start_time,
|
||||
config_snapshot=config_snapshot or {},
|
||||
)
|
||||
session.add(crawl_run)
|
||||
session.commit()
|
||||
logger.info(f"Started CrawlRun {crawl_run.id} for {brand} ({market_country})")
|
||||
|
||||
seen_oem_ids = set()
|
||||
|
||||
try:
|
||||
for vehicle_data in vehicle_stream:
|
||||
crawl_run.items_discovered += 1
|
||||
seen_oem_ids.add(vehicle_data.oem_vehicle_id)
|
||||
|
||||
try:
|
||||
self._ingest_vehicle(session, crawl_run, vehicle_data)
|
||||
except Exception as e:
|
||||
crawl_run.items_failed += 1
|
||||
logger.error(f"Error ingesting vehicle {vehicle_data.oem_vehicle_id}: {e}", exc_info=True)
|
||||
self._log_event(
|
||||
session=session,
|
||||
crawl_run_id=crawl_run.id,
|
||||
level=LogLevel.ERROR,
|
||||
stage=CrawlStage.PERSIST,
|
||||
message=f"Failed to persist vehicle {vehicle_data.oem_vehicle_id}: {str(e)}",
|
||||
error_context={"oem_vehicle_id": vehicle_data.oem_vehicle_id, "error": str(e)},
|
||||
)
|
||||
|
||||
# Periodic batch commit
|
||||
if crawl_run.items_discovered % 50 == 0:
|
||||
session.commit()
|
||||
|
||||
# Delisting detection for full market scans
|
||||
if config_snapshot and config_snapshot.get("is_full_market_scan", False):
|
||||
stock_type = config_snapshot.get("stock_type")
|
||||
self._detect_delisted_vehicles(session, crawl_run, seen_oem_ids, brand, market_country, stock_type=stock_type)
|
||||
|
||||
# Finalize CrawlRun
|
||||
finish_time = datetime.now(timezone.utc)
|
||||
crawl_run.finished_at = finish_time
|
||||
crawl_run.duration_ms = int((finish_time - start_time).total_seconds() * 1000)
|
||||
crawl_run.status = CrawlStatus.COMPLETED.value
|
||||
session.commit()
|
||||
logger.info(
|
||||
f"CrawlRun {crawl_run.id} finished: discovered={crawl_run.items_discovered}, "
|
||||
f"inserted={crawl_run.items_inserted}, updated={crawl_run.items_updated}, "
|
||||
f"unmodified={crawl_run.items_unmodified}, delisted={crawl_run.items_delisted}, failed={crawl_run.items_failed}"
|
||||
)
|
||||
return crawl_run
|
||||
|
||||
except Exception as err:
|
||||
session.rollback()
|
||||
finish_time = datetime.now(timezone.utc)
|
||||
crawl_run.finished_at = finish_time
|
||||
crawl_run.duration_ms = int((finish_time - start_time).total_seconds() * 1000)
|
||||
crawl_run.status = CrawlStatus.FAILED.value
|
||||
crawl_run.error_summary = str(err)
|
||||
session.commit()
|
||||
logger.error(f"CrawlRun {crawl_run.id} failed: {err}", exc_info=True)
|
||||
return crawl_run
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
def _ingest_vehicle(
|
||||
self,
|
||||
session: Session,
|
||||
crawl_run: CrawlRun,
|
||||
data: VehicleIngestSchema,
|
||||
) -> None:
|
||||
"""Ingests or updates a single vehicle using content_hash change detection."""
|
||||
content_hash = compute_content_hash(data)
|
||||
now_ts = datetime.now(timezone.utc)
|
||||
|
||||
stmt = select(Vehicle).where(
|
||||
Vehicle.brand == data.brand,
|
||||
Vehicle.location_country == data.location_country,
|
||||
Vehicle.oem_vehicle_id == data.oem_vehicle_id,
|
||||
)
|
||||
existing: Optional[Vehicle] = session.scalars(stmt).first()
|
||||
|
||||
if existing is None:
|
||||
# CASE 1: Brand new vehicle in stock
|
||||
vehicle_id = uuid.uuid4()
|
||||
options_gross_sum = sum(
|
||||
opt.price_gross for opt in data.options if opt.price_gross is not None
|
||||
)
|
||||
|
||||
new_veh = Vehicle(
|
||||
id=vehicle_id,
|
||||
brand=data.brand,
|
||||
oem_vehicle_id=data.oem_vehicle_id,
|
||||
vin=data.vin,
|
||||
series=data.series,
|
||||
model_name=data.model_name,
|
||||
trim_line=data.trim_line,
|
||||
body_type=data.body_type,
|
||||
condition=data.condition.value,
|
||||
model_year=data.model_year,
|
||||
first_registration_date=data.first_registration_date,
|
||||
previous_owners_count=data.previous_owners_count,
|
||||
fuel_type=data.fuel_type.value,
|
||||
transmission=data.transmission.value if data.transmission else None,
|
||||
drivetrain=data.drivetrain.value if data.drivetrain else None,
|
||||
power_kw=data.power_kw,
|
||||
power_hp=data.power_hp,
|
||||
engine_displacement_cc=data.engine_displacement_cc,
|
||||
battery_capacity=data.battery_capacity,
|
||||
battery_capacity_unit=data.battery_capacity_unit.value if data.battery_capacity_unit else None,
|
||||
battery_capacity_gross_kwh=data.battery_capacity_gross_kwh,
|
||||
battery_capacity_net_kwh=data.battery_capacity_net_kwh,
|
||||
charging_power_ac_kw=data.charging_power_ac_kw,
|
||||
charging_power_dc_kw=data.charging_power_dc_kw,
|
||||
charging_time_dc_minutes=data.charging_time_dc_minutes,
|
||||
electric_range_wltp_km=data.electric_range_wltp_km,
|
||||
consumption_combined=data.consumption_combined,
|
||||
co2_emissions_g_km=data.co2_emissions_g_km,
|
||||
mileage_km=data.mileage_km,
|
||||
exterior_color=data.exterior_color,
|
||||
exterior_color_code=data.exterior_color_code,
|
||||
exterior_color_generic=data.exterior_color_generic,
|
||||
interior_color=data.interior_color,
|
||||
interior_material=data.interior_material,
|
||||
currency=data.currency,
|
||||
base_price_gross=data.base_price_gross,
|
||||
options_price_gross=data.options_price_gross or (options_gross_sum if options_gross_sum > 0 else None),
|
||||
price_gross=data.price_gross,
|
||||
price_net=data.price_net,
|
||||
msrp_gross=data.msrp_gross,
|
||||
discount_percentage=data.discount_percentage,
|
||||
vat_deductible=data.vat_deductible,
|
||||
monthly_rate=data.monthly_rate,
|
||||
leasing_details=data.leasing_details,
|
||||
status=data.status.value,
|
||||
availability_date=data.availability_date,
|
||||
delivery_time_days=data.delivery_time_days,
|
||||
dealer_name=data.dealer_name,
|
||||
dealer_id=data.dealer_id,
|
||||
location_city=data.location_city,
|
||||
location_postal_code=data.location_postal_code,
|
||||
location_country=data.location_country,
|
||||
latitude=data.latitude,
|
||||
longitude=data.longitude,
|
||||
vehicle_url=data.vehicle_url,
|
||||
image_urls=data.image_urls,
|
||||
thumbnail_url=data.thumbnail_url,
|
||||
first_seen_at=now_ts,
|
||||
last_seen_at=now_ts,
|
||||
last_crawl_run_id=crawl_run.id,
|
||||
content_hash=content_hash,
|
||||
raw_payload={},
|
||||
raw_payload_path=(
|
||||
data.raw_payload_path
|
||||
or (
|
||||
RawStorageManager.save_payload(
|
||||
brand=data.brand,
|
||||
oem_vehicle_id=data.oem_vehicle_id,
|
||||
payload=data.raw_payload,
|
||||
timestamp=now_ts,
|
||||
)
|
||||
if (settings.ENABLE_RAW_EXTRACTION and data.raw_payload)
|
||||
else None
|
||||
)
|
||||
),
|
||||
created_at=now_ts,
|
||||
updated_at=now_ts,
|
||||
)
|
||||
session.add(new_veh)
|
||||
|
||||
# Insert Options
|
||||
for opt in data.options:
|
||||
session.add(
|
||||
VehicleOption(
|
||||
vehicle_id=vehicle_id,
|
||||
option_code=opt.option_code,
|
||||
name=opt.name,
|
||||
category=opt.category,
|
||||
price_gross=opt.price_gross,
|
||||
price_net=opt.price_net,
|
||||
is_standard=opt.is_standard,
|
||||
description=opt.description,
|
||||
raw_data={},
|
||||
)
|
||||
)
|
||||
|
||||
# Record Initial Price History
|
||||
session.add(
|
||||
VehiclePriceHistory(
|
||||
vehicle_id=vehicle_id,
|
||||
crawl_run_id=crawl_run.id,
|
||||
price_gross=data.price_gross,
|
||||
price_net=data.price_net,
|
||||
monthly_rate=data.monthly_rate,
|
||||
status=data.status.value,
|
||||
recorded_at=now_ts,
|
||||
)
|
||||
)
|
||||
|
||||
crawl_run.items_inserted += 1
|
||||
session.flush()
|
||||
|
||||
elif existing.content_hash == content_hash:
|
||||
# CASE 2: Vehicle observed with identical specs, options, and price
|
||||
existing.last_seen_at = now_ts
|
||||
existing.last_crawl_run_id = crawl_run.id
|
||||
if existing.status == VehicleStatus.DELISTED.value:
|
||||
existing.status = VehicleStatus.IN_STOCK.value
|
||||
existing.delisted_at = None
|
||||
crawl_run.items_unmodified += 1
|
||||
|
||||
else:
|
||||
# CASE 3: Vehicle observed with updated price, specs, or equipment
|
||||
old_price = existing.price_gross
|
||||
old_rate = existing.monthly_rate
|
||||
|
||||
existing.price_gross = data.price_gross
|
||||
existing.price_net = data.price_net
|
||||
existing.monthly_rate = data.monthly_rate
|
||||
existing.msrp_gross = data.msrp_gross
|
||||
existing.discount_percentage = data.discount_percentage
|
||||
existing.power_kw = data.power_kw
|
||||
existing.power_hp = data.power_hp
|
||||
existing.battery_capacity = data.battery_capacity
|
||||
existing.battery_capacity_unit = data.battery_capacity_unit.value if data.battery_capacity_unit else None
|
||||
existing.battery_capacity_gross_kwh = data.battery_capacity_gross_kwh
|
||||
existing.battery_capacity_net_kwh = data.battery_capacity_net_kwh
|
||||
existing.charging_power_ac_kw = data.charging_power_ac_kw
|
||||
existing.charging_power_dc_kw = data.charging_power_dc_kw
|
||||
existing.charging_time_dc_minutes = data.charging_time_dc_minutes
|
||||
existing.electric_range_wltp_km = data.electric_range_wltp_km
|
||||
existing.mileage_km = data.mileage_km
|
||||
existing.exterior_color = data.exterior_color
|
||||
existing.exterior_color_code = data.exterior_color_code
|
||||
existing.exterior_color_generic = data.exterior_color_generic
|
||||
existing.status = data.status.value
|
||||
existing.last_seen_at = now_ts
|
||||
existing.last_crawl_run_id = crawl_run.id
|
||||
existing.content_hash = content_hash
|
||||
if data.raw_payload_path:
|
||||
existing.raw_payload_path = data.raw_payload_path
|
||||
elif settings.ENABLE_RAW_EXTRACTION and data.raw_payload:
|
||||
existing.raw_payload_path = RawStorageManager.save_payload(
|
||||
brand=data.brand,
|
||||
oem_vehicle_id=data.oem_vehicle_id,
|
||||
payload=data.raw_payload,
|
||||
timestamp=now_ts,
|
||||
)
|
||||
existing.raw_payload = {}
|
||||
existing.updated_at = now_ts
|
||||
|
||||
# Re-sync options if options changed
|
||||
session.query(VehicleOption).filter(VehicleOption.vehicle_id == existing.id).delete()
|
||||
for opt in data.options:
|
||||
session.add(
|
||||
VehicleOption(
|
||||
vehicle_id=existing.id,
|
||||
option_code=opt.option_code,
|
||||
name=opt.name,
|
||||
category=opt.category,
|
||||
price_gross=opt.price_gross,
|
||||
price_net=opt.price_net,
|
||||
is_standard=opt.is_standard,
|
||||
description=opt.description,
|
||||
raw_data={},
|
||||
)
|
||||
)
|
||||
|
||||
# Record Price / Status snapshot if price, rate, or status changed
|
||||
if old_price != data.price_gross or old_rate != data.monthly_rate or existing.status != data.status.value:
|
||||
session.add(
|
||||
VehiclePriceHistory(
|
||||
vehicle_id=existing.id,
|
||||
crawl_run_id=crawl_run.id,
|
||||
price_gross=data.price_gross,
|
||||
price_net=data.price_net,
|
||||
monthly_rate=data.monthly_rate,
|
||||
status=data.status.value,
|
||||
recorded_at=now_ts,
|
||||
)
|
||||
)
|
||||
|
||||
crawl_run.items_updated += 1
|
||||
|
||||
def _detect_delisted_vehicles(
|
||||
self,
|
||||
session: Session,
|
||||
crawl_run: CrawlRun,
|
||||
seen_oem_ids: set,
|
||||
brand: str,
|
||||
market_country: str,
|
||||
stock_type: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Marks active in-stock vehicles as delisted if they were absent from a full scan."""
|
||||
now_ts = datetime.now(timezone.utc)
|
||||
stmt = (
|
||||
select(Vehicle)
|
||||
.where(
|
||||
Vehicle.brand == brand,
|
||||
Vehicle.location_country == market_country,
|
||||
Vehicle.status == VehicleStatus.IN_STOCK.value,
|
||||
)
|
||||
)
|
||||
if stock_type == "new":
|
||||
stmt = stmt.where(Vehicle.condition == VehicleCondition.NEW.value)
|
||||
elif stock_type == "used":
|
||||
stmt = stmt.where(
|
||||
Vehicle.condition.in_([VehicleCondition.USED.value, VehicleCondition.CERTIFIED_PRE_OWNED.value])
|
||||
)
|
||||
|
||||
active_vehicles: List[Vehicle] = session.scalars(stmt).all()
|
||||
for veh in active_vehicles:
|
||||
if veh.oem_vehicle_id not in seen_oem_ids:
|
||||
veh.status = VehicleStatus.DELISTED.value
|
||||
veh.delisted_at = now_ts
|
||||
veh.updated_at = now_ts
|
||||
crawl_run.items_delisted += 1
|
||||
|
||||
def _log_event(
|
||||
self,
|
||||
session: Session,
|
||||
crawl_run_id: uuid.UUID,
|
||||
level: LogLevel,
|
||||
stage: CrawlStage,
|
||||
message: str,
|
||||
error_context: Optional[dict] = None,
|
||||
) -> None:
|
||||
"""Appends a diagnostic log entry to crawl_logs."""
|
||||
log_entry = CrawlLog(
|
||||
crawl_run_id=crawl_run_id,
|
||||
level=level.value,
|
||||
stage=stage.value,
|
||||
message=message,
|
||||
error_context=error_context or {},
|
||||
)
|
||||
session.add(log_entry)
|
||||
session.commit()
|
||||
233
src/oem_crawler/db/schemas.py
Normal file
233
src/oem_crawler/db/schemas.py
Normal file
|
|
@ -0,0 +1,233 @@
|
|||
"""
|
||||
Pydantic schemas and deterministic content hashing logic.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from enum import Enum
|
||||
from typing import Any, List, Optional
|
||||
|
||||
from pydantic import BaseModel, Field, ConfigDict
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Enums
|
||||
# =============================================================================
|
||||
|
||||
class CrawlStatus(str, Enum):
|
||||
PENDING = "PENDING"
|
||||
RUNNING = "RUNNING"
|
||||
COMPLETED = "COMPLETED"
|
||||
FAILED = "FAILED"
|
||||
PARTIAL = "PARTIAL"
|
||||
|
||||
|
||||
class VehicleCondition(str, Enum):
|
||||
NEW = "new"
|
||||
USED = "used"
|
||||
DEMONSTRATION = "demonstration"
|
||||
DAY_REGISTRATION = "day_registration"
|
||||
CERTIFIED_PRE_OWNED = "certified_pre_owned"
|
||||
|
||||
|
||||
class FuelType(str, Enum):
|
||||
BEV = "bev"
|
||||
PHEV = "phev"
|
||||
PETROL = "petrol"
|
||||
DIESEL = "diesel"
|
||||
MHEV = "mhev"
|
||||
FCEV = "fcev"
|
||||
CNG = "cng"
|
||||
LPG = "lpg"
|
||||
HYBRID_OTHER = "hybrid_other"
|
||||
|
||||
|
||||
class BatteryCapacityUnit(str, Enum):
|
||||
KWH = "kwh"
|
||||
AH = "ah"
|
||||
|
||||
|
||||
class TransmissionType(str, Enum):
|
||||
AUTOMATIC = "automatic"
|
||||
MANUAL = "manual"
|
||||
DIRECT_DRIVE = "direct_drive"
|
||||
DUAL_CLUTCH = "dual_clutch"
|
||||
CVT = "cvt"
|
||||
OTHER = "other"
|
||||
|
||||
|
||||
class DrivetrainType(str, Enum):
|
||||
AWD = "awd"
|
||||
RWD = "rwd"
|
||||
FWD = "fwd"
|
||||
FOUR_BY_FOUR = "4x4"
|
||||
OTHER = "other"
|
||||
|
||||
|
||||
class VehicleStatus(str, Enum):
|
||||
IN_STOCK = "in_stock"
|
||||
IN_TRANSIT = "in_transit"
|
||||
ON_ORDER = "on_order"
|
||||
RESERVED = "reserved"
|
||||
SOLD = "sold"
|
||||
DELISTED = "delisted"
|
||||
|
||||
|
||||
class LogLevel(str, Enum):
|
||||
DEBUG = "DEBUG"
|
||||
INFO = "INFO"
|
||||
WARNING = "WARNING"
|
||||
ERROR = "ERROR"
|
||||
CRITICAL = "CRITICAL"
|
||||
|
||||
|
||||
class CrawlStage(str, Enum):
|
||||
FETCH = "FETCH"
|
||||
PARSE = "PARSE"
|
||||
VALIDATE = "VALIDATE"
|
||||
PERSIST = "PERSIST"
|
||||
DELIST_CHECK = "DELIST_CHECK"
|
||||
GENERAL = "GENERAL"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Pydantic Ingestion / Validation Schemas
|
||||
# =============================================================================
|
||||
|
||||
class OptionSchema(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
option_code: Optional[str] = None
|
||||
name: str
|
||||
category: Optional[str] = None
|
||||
price_gross: Optional[Decimal] = None
|
||||
price_net: Optional[Decimal] = None
|
||||
is_standard: bool = False
|
||||
description: Optional[str] = None
|
||||
raw_data: dict = Field(default_factory=dict)
|
||||
|
||||
|
||||
class VehicleIngestSchema(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
# Identifiers
|
||||
brand: str
|
||||
oem_vehicle_id: str
|
||||
vin: Optional[str] = None
|
||||
|
||||
# Classification
|
||||
series: str
|
||||
model_name: str
|
||||
trim_line: Optional[str] = None
|
||||
body_type: Optional[str] = None
|
||||
condition: VehicleCondition = VehicleCondition.NEW
|
||||
model_year: Optional[int] = None
|
||||
first_registration_date: Optional[date] = None
|
||||
previous_owners_count: Optional[int] = None
|
||||
|
||||
# Powertrain & Specs
|
||||
fuel_type: FuelType = FuelType.PETROL
|
||||
transmission: Optional[TransmissionType] = None
|
||||
drivetrain: Optional[DrivetrainType] = None
|
||||
power_kw: Optional[int] = None
|
||||
power_hp: Optional[int] = None
|
||||
engine_displacement_cc: Optional[int] = None
|
||||
battery_capacity: Optional[Decimal] = None
|
||||
battery_capacity_unit: Optional[BatteryCapacityUnit] = None
|
||||
battery_capacity_gross_kwh: Optional[Decimal] = None
|
||||
battery_capacity_net_kwh: Optional[Decimal] = None
|
||||
charging_power_ac_kw: Optional[Decimal] = None
|
||||
charging_power_dc_kw: Optional[Decimal] = None
|
||||
charging_time_dc_minutes: Optional[int] = None
|
||||
electric_range_wltp_km: Optional[int] = None
|
||||
consumption_combined: Optional[Decimal] = None
|
||||
co2_emissions_g_km: Optional[Decimal] = None
|
||||
mileage_km: int = 0
|
||||
|
||||
# Colors
|
||||
exterior_color: Optional[str] = None
|
||||
exterior_color_code: Optional[str] = None
|
||||
exterior_color_generic: Optional[str] = None
|
||||
interior_color: Optional[str] = None
|
||||
interior_material: Optional[str] = None
|
||||
|
||||
# Pricing & Availability
|
||||
currency: str = "EUR"
|
||||
base_price_gross: Optional[Decimal] = None
|
||||
options_price_gross: Optional[Decimal] = None
|
||||
price_gross: Decimal
|
||||
price_net: Optional[Decimal] = None
|
||||
msrp_gross: Optional[Decimal] = None
|
||||
discount_percentage: Optional[Decimal] = None
|
||||
vat_deductible: Optional[bool] = None
|
||||
monthly_rate: Optional[Decimal] = None
|
||||
leasing_details: dict = Field(default_factory=dict)
|
||||
status: VehicleStatus = VehicleStatus.IN_STOCK
|
||||
availability_date: Optional[date] = None
|
||||
delivery_time_days: Optional[int] = None
|
||||
|
||||
# Location & Media
|
||||
dealer_name: Optional[str] = None
|
||||
dealer_id: Optional[str] = None
|
||||
location_city: Optional[str] = None
|
||||
location_postal_code: Optional[str] = None
|
||||
location_country: str = "DE"
|
||||
latitude: Optional[Decimal] = None
|
||||
longitude: Optional[Decimal] = None
|
||||
vehicle_url: Optional[str] = None
|
||||
image_urls: List[str] = Field(default_factory=list)
|
||||
thumbnail_url: Optional[str] = None
|
||||
|
||||
# Options & Equipment
|
||||
options: List[OptionSchema] = Field(default_factory=list)
|
||||
|
||||
# Raw Payload & Storage
|
||||
raw_payload: dict = Field(default_factory=dict)
|
||||
raw_payload_path: Optional[str] = None
|
||||
|
||||
|
||||
def compute_content_hash(vehicle: VehicleIngestSchema) -> str:
|
||||
"""
|
||||
Computes a deterministic SHA-256 hash of core vehicle attributes,
|
||||
prices, and equipment options to detect stock updates and changes quickly.
|
||||
"""
|
||||
sorted_options = sorted(
|
||||
[
|
||||
{
|
||||
"code": opt.option_code or "",
|
||||
"name": opt.name,
|
||||
"price": str(opt.price_gross) if opt.price_gross is not None else "",
|
||||
"is_std": opt.is_standard,
|
||||
}
|
||||
for opt in vehicle.options
|
||||
],
|
||||
key=lambda x: (x["code"], x["name"]),
|
||||
)
|
||||
|
||||
hash_dict = {
|
||||
"brand": vehicle.brand.lower(),
|
||||
"oem_vehicle_id": vehicle.oem_vehicle_id,
|
||||
"vin": vehicle.vin or "",
|
||||
"series": vehicle.series,
|
||||
"model_name": vehicle.model_name,
|
||||
"condition": vehicle.condition.value,
|
||||
"fuel_type": vehicle.fuel_type.value,
|
||||
"transmission": vehicle.transmission.value if vehicle.transmission else "",
|
||||
"drivetrain": vehicle.drivetrain.value if vehicle.drivetrain else "",
|
||||
"power_kw": vehicle.power_kw,
|
||||
"battery_capacity": str(vehicle.battery_capacity) if vehicle.battery_capacity is not None else "",
|
||||
"battery_capacity_unit": vehicle.battery_capacity_unit.value if vehicle.battery_capacity_unit else "",
|
||||
"mileage_km": vehicle.mileage_km,
|
||||
"price_gross": str(vehicle.price_gross),
|
||||
"price_net": str(vehicle.price_net) if vehicle.price_net is not None else "",
|
||||
"monthly_rate": str(vehicle.monthly_rate) if vehicle.monthly_rate is not None else "",
|
||||
"status": vehicle.status.value,
|
||||
"exterior_color": vehicle.exterior_color or "",
|
||||
"options": sorted_options,
|
||||
}
|
||||
|
||||
serialized = json.dumps(hash_dict, sort_keys=True)
|
||||
return hashlib.sha256(serialized.encode("utf-8")).hexdigest()
|
||||
38
src/oem_crawler/db/session.py
Normal file
38
src/oem_crawler/db/session.py
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
"""
|
||||
Database session and connection management.
|
||||
"""
|
||||
from typing import Generator
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from oem_crawler.config import settings
|
||||
from oem_crawler.db.models import Base
|
||||
|
||||
|
||||
def ensure_sqlite_dir(url: str) -> None:
|
||||
"""Ensures parent directories exist for local SQLite file paths."""
|
||||
if url.startswith("sqlite:///"):
|
||||
path_str = url.replace("sqlite:///", "")
|
||||
if path_str and not path_str.startswith(":memory:"):
|
||||
Path(path_str).parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
ensure_sqlite_dir(settings.DATABASE_URL)
|
||||
engine = create_engine(settings.DATABASE_URL, echo=settings.DB_ECHO_SQL)
|
||||
SessionFactory = sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
||||
|
||||
|
||||
def init_db():
|
||||
"""Initializes schema tables if not present."""
|
||||
Base.metadata.create_all(engine)
|
||||
|
||||
|
||||
def get_db_session() -> Generator[Session, None, None]:
|
||||
"""Yields an isolated database session for transactional tasks."""
|
||||
session = SessionFactory()
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
session.close()
|
||||
123
src/oem_crawler/storage.py
Normal file
123
src/oem_crawler/storage.py
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
"""
|
||||
Storage manager for raw OEM payloads.
|
||||
Provides compressed file persistence (.json.gz) outside of the relational database.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from oem_crawler.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RawStorageManager:
|
||||
"""Manages writing, reading, and locating compressed raw payloads on the filesystem."""
|
||||
|
||||
@staticmethod
|
||||
def get_storage_path(
|
||||
brand: str,
|
||||
oem_vehicle_id: str,
|
||||
timestamp: Optional[datetime] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Computes the relative path for a vehicle's raw payload file.
|
||||
Format: {brand}/{YYYY-MM-DD}/{safe_id}.json.gz
|
||||
"""
|
||||
date_str = (timestamp or datetime.now(timezone.utc)).strftime("%Y-%m-%d")
|
||||
safe_brand = brand.strip().lower()
|
||||
safe_id = "".join(c if c.isalnum() or c in ("-", "_") else "_" for c in str(oem_vehicle_id))
|
||||
return f"{safe_brand}/{date_str}/{safe_id}.json.gz"
|
||||
|
||||
@classmethod
|
||||
def save_payload(
|
||||
cls,
|
||||
brand: str,
|
||||
oem_vehicle_id: str,
|
||||
payload: Dict[str, Any],
|
||||
timestamp: Optional[datetime] = None,
|
||||
base_dir: Optional[str] = None,
|
||||
compression_level: Optional[int] = None,
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
Saves a compressed JSON payload (.json.gz) to disk if extraction is enabled.
|
||||
Returns the relative file path if saved, or None if disabled or empty.
|
||||
"""
|
||||
if not settings.ENABLE_RAW_EXTRACTION or not payload:
|
||||
return None
|
||||
|
||||
rel_path = cls.get_storage_path(brand=brand, oem_vehicle_id=oem_vehicle_id, timestamp=timestamp)
|
||||
storage_root = Path(base_dir or settings.RAW_STORAGE_DIR)
|
||||
target_path = storage_root / rel_path
|
||||
target_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
level = compression_level if compression_level is not None else settings.RAW_COMPRESSION_LEVEL
|
||||
|
||||
# Write to temporary file first for atomic commit
|
||||
temp_path = target_path.with_suffix(".tmp")
|
||||
try:
|
||||
with gzip.open(temp_path, "wt", encoding="utf-8", compresslevel=level) as f:
|
||||
json.dump(payload, f, ensure_ascii=False)
|
||||
temp_path.replace(target_path)
|
||||
return rel_path
|
||||
except Exception as e:
|
||||
if temp_path.exists():
|
||||
try:
|
||||
temp_path.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
logger.error(f"Failed to save raw payload for {brand} {oem_vehicle_id}: {e}")
|
||||
raise
|
||||
|
||||
@classmethod
|
||||
def load_payload(
|
||||
cls,
|
||||
rel_path: str,
|
||||
base_dir: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Reads and decompresses a raw payload from disk.
|
||||
"""
|
||||
if not rel_path:
|
||||
return None
|
||||
|
||||
storage_root = Path(base_dir or settings.RAW_STORAGE_DIR)
|
||||
full_path = storage_root / rel_path
|
||||
|
||||
if not full_path.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
with gzip.open(full_path, "rt", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to read raw payload from {full_path}: {e}")
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def delete_payload(
|
||||
cls,
|
||||
rel_path: str,
|
||||
base_dir: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""Deletes a raw payload file if it exists."""
|
||||
if not rel_path:
|
||||
return False
|
||||
|
||||
storage_root = Path(base_dir or settings.RAW_STORAGE_DIR)
|
||||
full_path = storage_root / rel_path
|
||||
|
||||
if full_path.exists():
|
||||
try:
|
||||
full_path.unlink()
|
||||
return True
|
||||
except OSError as e:
|
||||
logger.error(f"Failed to delete {full_path}: {e}")
|
||||
return False
|
||||
|
||||
130
src/oem_crawler/tools/README.md
Normal file
130
src/oem_crawler/tools/README.md
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
# 🛠️ OEM Crawler Developer Tools
|
||||
|
||||
This folder contains inspection, auditing, and debugging utilities for the OEM Crawler project.
|
||||
|
||||
---
|
||||
|
||||
## 🔍 Vehicle Stock Inspector (`view_stock.py`)
|
||||
|
||||
`view_stock.py` is a terminal utility that inspects the local database, shows high-level statistics across all manufacturer crawl runs, breaks down inventory by brand, and displays vehicle cards with their factory options and EV specifications.
|
||||
|
||||
### Running the Tool
|
||||
|
||||
You can run the inspector either via the CLI command `oem-inspect` or directly with Python:
|
||||
|
||||
```bash
|
||||
# 1. Using the CLI shortcut
|
||||
uv run oem-inspect
|
||||
|
||||
# 2. Or using python module execution
|
||||
uv run python -m oem_crawler.tools.view_stock
|
||||
|
||||
# 3. Or running the script directly
|
||||
uv run python src/oem_crawler/tools/view_stock.py
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📋 Command-Line Options
|
||||
|
||||
| Option | Default | Description |
|
||||
| :--- | :--- | :--- |
|
||||
| `--summary` | `False` | Displays the high-level aggregate summary and brand breakdown table only, without listing individual vehicles. |
|
||||
| `--brand <NAME>` | `None` | Filters vehicles by manufacturer (`AUDI`, `BMW`, `MERCEDES`, `PORSCHE`). |
|
||||
| `--limit <N>` | `10` | Number of vehicle cards to display in the list. |
|
||||
| `--db-url <URL>` | `sqlite:///data/crawler_stock.db` | SQLAlchemy connection URL (can point to local SQLite file or PostgreSQL). |
|
||||
|
||||
### Examples
|
||||
|
||||
```bash
|
||||
# Display only the high-level summary and brand table
|
||||
uv run oem-inspect --summary
|
||||
|
||||
# Inspect the 15 latest Audi vehicles
|
||||
uv run oem-inspect --brand audi --limit 15
|
||||
|
||||
# Inspect Porsche inventory
|
||||
uv run oem-inspect --brand porsche --limit 10
|
||||
|
||||
# Inspect against a Docker PostgreSQL database
|
||||
uv run oem-inspect --db-url postgresql+psycopg://crawler_user:crawler_password@localhost:5432/oem_stock_db
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 💻 Direct Database Queries via `sqlite3` CLI
|
||||
|
||||
For ad-hoc queries on the local SQLite development database (`data/crawler_stock.db`):
|
||||
|
||||
```bash
|
||||
# 1. Inventory count and pricing stats per brand
|
||||
sqlite3 -table data/crawler_stock.db "
|
||||
SELECT brand,
|
||||
count(*) AS vehicles,
|
||||
printf('%.0f €', avg(price_gross)) AS avg_price,
|
||||
printf('%.0f €', min(price_gross)) AS min_price,
|
||||
printf('%.0f €', max(price_gross)) AS max_price
|
||||
FROM vehicles
|
||||
GROUP BY brand;"
|
||||
|
||||
# 2. Powertrain and Fuel Type breakdown
|
||||
sqlite3 -table data/crawler_stock.db "
|
||||
SELECT brand, fuel_type, condition, count(*) AS count
|
||||
FROM vehicles
|
||||
GROUP BY brand, fuel_type, condition
|
||||
ORDER BY brand, count DESC;"
|
||||
|
||||
# 3. Search for specific factory options
|
||||
sqlite3 -table data/crawler_stock.db "
|
||||
SELECT v.brand, v.model_name, vo.option_code, vo.name, vo.category
|
||||
FROM vehicle_options vo
|
||||
JOIN vehicles v ON vo.vehicle_id = v.id
|
||||
WHERE vo.name LIKE '%panoramadach%'
|
||||
LIMIT 10;"
|
||||
|
||||
# 4. View latest crawl run audit history
|
||||
sqlite3 -table data/crawler_stock.db "
|
||||
SELECT id, brand, status, items_discovered, items_inserted,
|
||||
printf('%.1fs', duration_ms/1000.0) AS duration, started_at
|
||||
FROM crawl_runs
|
||||
ORDER BY started_at DESC
|
||||
LIMIT 8;"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔄 Database Restore & Upsert from Raw Payloads (`restore_from_raw.py`)
|
||||
|
||||
`restore_from_raw.py` reads and decompresses raw OEM JSON files (`.json.gz`) from your storage directory and upserts them into the database. It is designed for disaster recovery, rebuilding a database from scratch, or re-indexing without having to re-crawl OEM portals.
|
||||
|
||||
### Running the Tool
|
||||
|
||||
```bash
|
||||
# 1. Using the CLI shortcut
|
||||
uv run oem-restore
|
||||
|
||||
# 2. Or using the script launcher
|
||||
uv run python scripts/restore_from_raw.py
|
||||
|
||||
# 3. Dry-run validation (parse and validate without DB changes)
|
||||
uv run oem-restore --dry-run
|
||||
|
||||
# 4. Restore only a specific brand
|
||||
uv run oem-restore --brand porsche
|
||||
|
||||
# 5. Restore into a fresh or remote database
|
||||
uv run oem-restore --db-url "sqlite:///data/restored_stock.db"
|
||||
```
|
||||
|
||||
### Options
|
||||
|
||||
| Option | Default | Description |
|
||||
| :--- | :--- | :--- |
|
||||
| `--raw-dir <PATH>` | `data/raw` | Source directory containing compressed `.json.gz` files. |
|
||||
| `--db-url <URL>` | `sqlite:///data/crawler_stock.db` | Destination database connection string. |
|
||||
| `--brand <NAME>` | `all` | Filter by brand (`bmw`, `audi`, `mercedes-benz`, `porsche`, or comma-separated). |
|
||||
| `--limit <N>` | `None` | Process up to N files per brand (useful for testing). |
|
||||
| `--batch-size <N>` | `500` | Number of records per commit transaction. |
|
||||
| `--dry-run` | `False` | Validate raw files without altering database state. |
|
||||
| `--country <CC>` | `DE` | Market ISO code. |
|
||||
|
||||
7
src/oem_crawler/tools/__init__.py
Normal file
7
src/oem_crawler/tools/__init__.py
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
"""
|
||||
OEM Crawler Developer & Inspection Tools.
|
||||
"""
|
||||
from oem_crawler.tools.restore_from_raw import restore_database
|
||||
from oem_crawler.tools.view_stock import inspect_stock, main
|
||||
|
||||
__all__ = ["inspect_stock", "main", "restore_database"]
|
||||
328
src/oem_crawler/tools/restore_from_raw.py
Normal file
328
src/oem_crawler/tools/restore_from_raw.py
Normal file
|
|
@ -0,0 +1,328 @@
|
|||
"""
|
||||
Restore & Upsert Tool for OEM Crawler
|
||||
Rebuilds, repairs, or re-populates the database by reading and parsing raw compressed payload files (.json.gz).
|
||||
Useful for disaster recovery, migrating databases, or re-indexing after schema changes.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Generator, List, Optional, Tuple
|
||||
|
||||
from oem_crawler.config import settings
|
||||
from oem_crawler.crawlers.audi.parser import parse_audi_vehicle
|
||||
from oem_crawler.crawlers.bmw.parser import parse_bmw_vehicle
|
||||
from oem_crawler.crawlers.mercedes.parser import parse_mercedes_used_dict, parse_mercedes_vehicle
|
||||
from oem_crawler.crawlers.porsche.parser import parse_porsche_vehicle
|
||||
from oem_crawler.db.pipeline import StockIngestionPipeline
|
||||
from oem_crawler.db.schemas import VehicleIngestSchema
|
||||
from oem_crawler.db.session import ensure_sqlite_dir
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
logger = logging.getLogger("restore_from_raw")
|
||||
|
||||
|
||||
def parse_raw_payload(
|
||||
brand: str,
|
||||
payload: Dict[str, Any],
|
||||
market_country: str = "DE",
|
||||
rel_path: Optional[str] = None,
|
||||
) -> Optional[VehicleIngestSchema]:
|
||||
"""
|
||||
Normalizes a raw payload dictionary into a validated VehicleIngestSchema based on brand.
|
||||
"""
|
||||
brand_norm = brand.strip().lower()
|
||||
|
||||
if "audi" in brand_norm:
|
||||
basic_item = payload.get("vehicleBasic", payload)
|
||||
detail_data = payload.get("detail")
|
||||
veh = parse_audi_vehicle(
|
||||
basic_item=basic_item,
|
||||
detail_data=detail_data,
|
||||
market_country=market_country,
|
||||
include_raw=False,
|
||||
)
|
||||
elif "bmw" in brand_norm:
|
||||
veh = parse_bmw_vehicle(
|
||||
raw=payload,
|
||||
market_country=market_country,
|
||||
include_raw=False,
|
||||
)
|
||||
elif "porsche" in brand_norm:
|
||||
search_item = payload.get("search_item", payload)
|
||||
detail_data = payload.get("detail_data")
|
||||
veh = parse_porsche_vehicle(
|
||||
search_item=search_item,
|
||||
detail_data=detail_data,
|
||||
market_country=market_country,
|
||||
include_raw=False,
|
||||
)
|
||||
elif "mercedes" in brand_norm:
|
||||
if "search" in payload:
|
||||
veh = parse_mercedes_vehicle(
|
||||
raw_search=payload["search"],
|
||||
raw_price=payload.get("price"),
|
||||
raw_vpc=payload.get("vpc"),
|
||||
market_country=market_country,
|
||||
include_raw=False,
|
||||
)
|
||||
elif "card_id" in payload or "basic_data" in payload or "card_attributes" in payload:
|
||||
veh = parse_mercedes_used_dict(
|
||||
data=payload,
|
||||
market_country=market_country,
|
||||
include_raw=False,
|
||||
)
|
||||
else:
|
||||
return None
|
||||
else:
|
||||
logger.warning(f"Unsupported brand '{brand}' for raw payload.")
|
||||
return None
|
||||
|
||||
if veh and rel_path:
|
||||
veh.raw_payload_path = rel_path
|
||||
|
||||
return veh
|
||||
|
||||
|
||||
def stream_brand_files(
|
||||
brand_dir: Path,
|
||||
raw_root: Path,
|
||||
market_country: str = "DE",
|
||||
limit: Optional[int] = None,
|
||||
) -> Generator[Tuple[VehicleIngestSchema, str], None, None]:
|
||||
"""
|
||||
Yields parsed (VehicleIngestSchema, rel_path) tuples from a brand directory.
|
||||
"""
|
||||
brand = brand_dir.name
|
||||
count = 0
|
||||
|
||||
# Scan all .json.gz files recursively (partitioned by date)
|
||||
for file_path in sorted(brand_dir.glob("**/*.json.gz")):
|
||||
rel_path = str(file_path.relative_to(raw_root))
|
||||
try:
|
||||
with gzip.open(file_path, "rt", encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
|
||||
# Skip dummy test fixtures
|
||||
if payload.get("id") == "TEST-MB-EQE-001" and len(payload) == 1:
|
||||
continue
|
||||
|
||||
veh = parse_raw_payload(
|
||||
brand=brand,
|
||||
payload=payload,
|
||||
market_country=market_country,
|
||||
rel_path=rel_path,
|
||||
)
|
||||
|
||||
if veh:
|
||||
yield veh, rel_path
|
||||
count += 1
|
||||
if limit and count >= limit:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning(f"Error parsing raw file {rel_path}: {e}")
|
||||
|
||||
|
||||
def restore_database(
|
||||
raw_dir: Path,
|
||||
db_url: str,
|
||||
target_brands: Optional[List[str]] = None,
|
||||
batch_size: int = 500,
|
||||
limit: Optional[int] = None,
|
||||
dry_run: bool = False,
|
||||
market_country: str = "DE",
|
||||
) -> None:
|
||||
"""
|
||||
Scans raw directory and upserts vehicles into the database using StockIngestionPipeline.
|
||||
"""
|
||||
raw_root = raw_dir.resolve()
|
||||
if not raw_root.exists():
|
||||
logger.error(f"Raw storage directory not found: {raw_root}")
|
||||
sys.exit(1)
|
||||
|
||||
# Discover brand subdirectories
|
||||
available_brand_dirs = [d for d in sorted(raw_root.iterdir()) if d.is_dir() and not d.name.startswith(".")]
|
||||
if target_brands:
|
||||
filter_set = {b.strip().lower() for b in target_brands}
|
||||
brand_dirs = [d for d in available_brand_dirs if d.name.lower() in filter_set]
|
||||
else:
|
||||
brand_dirs = available_brand_dirs
|
||||
|
||||
if not brand_dirs:
|
||||
logger.warning(f"No matching brand directories found in {raw_root}")
|
||||
return
|
||||
|
||||
logger.info("=" * 70)
|
||||
logger.info("OEM VEHICLE STOCK DATABASE RESTORE & UPSERT TOOL")
|
||||
logger.info("=" * 70)
|
||||
logger.info(f"Source Directory: {raw_root}")
|
||||
logger.info(f"Target Database: {db_url}")
|
||||
logger.info(f"Brands to Scan: {', '.join(d.name.upper() for d in brand_dirs)}")
|
||||
logger.info(f"Dry Run Mode: {dry_run}")
|
||||
if limit:
|
||||
logger.info(f"Per-Brand Limit: {limit:,}")
|
||||
logger.info("=" * 70 + "\n")
|
||||
|
||||
ensure_sqlite_dir(db_url)
|
||||
pipeline = StockIngestionPipeline(db_url=db_url) if not dry_run else None
|
||||
|
||||
overall_start = time.time()
|
||||
grand_totals = {
|
||||
"discovered": 0,
|
||||
"parsed": 0,
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"unmodified": 0,
|
||||
"failed": 0,
|
||||
}
|
||||
|
||||
for b_dir in brand_dirs:
|
||||
brand_name = b_dir.name.upper()
|
||||
files = list(b_dir.glob("**/*.json.gz"))
|
||||
total_files = len(files)
|
||||
logger.info(f"[{brand_name}] Found {total_files:,} raw payload files. Starting ingestion...")
|
||||
|
||||
brand_start = time.time()
|
||||
brand_parsed = 0
|
||||
|
||||
def vehicle_generator():
|
||||
nonlocal brand_parsed
|
||||
for veh, _ in stream_brand_files(b_dir, raw_root, market_country=market_country, limit=limit):
|
||||
brand_parsed += 1
|
||||
yield veh
|
||||
|
||||
if dry_run:
|
||||
# In dry-run mode, iterate and validate without pipeline
|
||||
for veh, rel_path in stream_brand_files(b_dir, raw_root, market_country=market_country, limit=limit):
|
||||
brand_parsed += 1
|
||||
if brand_parsed % 1000 == 0 or brand_parsed == total_files:
|
||||
logger.info(f"[{brand_name}] [Dry Run] Validated {brand_parsed:,}/{total_files:,} payloads...")
|
||||
|
||||
elapsed = time.time() - brand_start
|
||||
rate = brand_parsed / elapsed if elapsed > 0 else 0
|
||||
logger.info(f"[{brand_name}] [Dry Run] Finished validating {brand_parsed:,} vehicles ({rate:.0f} veh/sec).")
|
||||
grand_totals["discovered"] += total_files
|
||||
grand_totals["parsed"] += brand_parsed
|
||||
else:
|
||||
# Ingest via pipeline
|
||||
run_result = pipeline.run_crawler_pipeline(
|
||||
brand=brand_name,
|
||||
crawler_name="restore_from_raw",
|
||||
vehicle_stream=vehicle_generator(),
|
||||
market_country=market_country,
|
||||
)
|
||||
|
||||
elapsed = time.time() - brand_start
|
||||
rate = run_result.items_discovered / elapsed if elapsed > 0 else 0
|
||||
logger.info(
|
||||
f"[{brand_name}] Ingestion complete: "
|
||||
f"Discovered={run_result.items_discovered:,} | "
|
||||
f"Inserted={run_result.items_inserted:,} | "
|
||||
f"Updated={run_result.items_updated:,} | "
|
||||
f"Unmodified={run_result.items_unmodified:,} | "
|
||||
f"Failed={run_result.items_failed:,} "
|
||||
f"({rate:.0f} veh/sec in {elapsed:.1f}s)"
|
||||
)
|
||||
|
||||
grand_totals["discovered"] += total_files
|
||||
grand_totals["parsed"] += run_result.items_discovered
|
||||
grand_totals["inserted"] += run_result.items_inserted
|
||||
grand_totals["updated"] += run_result.items_updated
|
||||
grand_totals["unmodified"] += run_result.items_unmodified
|
||||
grand_totals["failed"] += run_result.items_failed
|
||||
|
||||
total_time = time.time() - overall_start
|
||||
overall_rate = grand_totals["parsed"] / total_time if total_time > 0 else 0
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("RESTORE SUMMARY")
|
||||
print("=" * 70)
|
||||
print(f"Total Raw Files Scanned: {grand_totals['discovered']:,}")
|
||||
print(f"Total Vehicles Parsed: {grand_totals['parsed']:,}")
|
||||
if not dry_run:
|
||||
print(f"Vehicles Inserted (New): {grand_totals['inserted']:,}")
|
||||
print(f"Vehicles Updated: {grand_totals['updated']:,}")
|
||||
print(f"Vehicles Unmodified: {grand_totals['unmodified']:,}")
|
||||
print(f"Failed Records: {grand_totals['failed']:,}")
|
||||
print(f"Total Duration: {total_time:.1f} seconds ({overall_rate:.0f} veh/sec)")
|
||||
print("=" * 70 + "\n")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Upsert and restore OEM inventory database from external raw compressed payload files (.json.gz)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--raw-dir",
|
||||
type=Path,
|
||||
default=Path(settings.RAW_STORAGE_DIR),
|
||||
help=f"Directory containing compressed raw JSON payloads (default: {settings.RAW_STORAGE_DIR})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--db-url",
|
||||
type=str,
|
||||
default=settings.DATABASE_URL,
|
||||
help=f"Target database URL (default: {settings.DATABASE_URL})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--brand",
|
||||
type=str,
|
||||
default="all",
|
||||
help="Brand to restore (e.g. 'bmw', 'audi', 'mercedes-benz', 'porsche', or 'all'; comma-separated allowed)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--batch-size",
|
||||
type=int,
|
||||
default=500,
|
||||
help="Transaction batch size (default: 500)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Limit number of vehicles to process per brand (default: unlimited)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--country",
|
||||
type=str,
|
||||
default=settings.DEFAULT_MARKET,
|
||||
help=f"Market country code (default: {settings.DEFAULT_MARKET})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Parse and validate files without making any database changes",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
target_brands = None
|
||||
if args.brand and args.brand.strip().lower() not in ("all", "*"):
|
||||
target_brands = [b.strip().lower() for b in args.brand.split(",") if b.strip()]
|
||||
|
||||
restore_database(
|
||||
raw_dir=args.raw_dir,
|
||||
db_url=args.db_url,
|
||||
target_brands=target_brands,
|
||||
batch_size=args.batch_size,
|
||||
limit=args.limit,
|
||||
dry_run=args.dry_run,
|
||||
market_country=args.country,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
128
src/oem_crawler/tools/view_stock.py
Normal file
128
src/oem_crawler/tools/view_stock.py
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
"""
|
||||
Vehicle Stock Inspector & Query Tool
|
||||
Prints vehicle inventory, pricing summaries, and options from the database.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import create_engine, func, select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from oem_crawler.config import settings
|
||||
from oem_crawler.db.models import CrawlRun, Vehicle, VehicleOption, VehiclePriceHistory
|
||||
from oem_crawler.db.session import ensure_sqlite_dir
|
||||
|
||||
|
||||
def inspect_stock(db_url: str, limit: int = 10, brand: Optional[str] = None, summary_only: bool = False):
|
||||
ensure_sqlite_dir(db_url)
|
||||
engine = create_engine(db_url)
|
||||
Session = sessionmaker(bind=engine)
|
||||
|
||||
with Session() as session:
|
||||
# Total counts
|
||||
total_vehicles = session.scalar(select(func.count(Vehicle.id))) or 0
|
||||
total_runs = session.scalar(select(func.count(CrawlRun.id))) or 0
|
||||
total_options = session.scalar(select(func.count(VehicleOption.id))) or 0
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print(f"📊 DATABASE SUMMARY ({db_url})")
|
||||
print(f" • Total Crawl Runs: {total_runs}")
|
||||
print(f" • Total Vehicles in Stock: {total_vehicles:,}")
|
||||
print(f" • Total Normalized Options: {total_options:,}")
|
||||
print("=" * 80)
|
||||
|
||||
if total_vehicles == 0:
|
||||
print("\n⚠️ No vehicles found in database yet. Run a crawl first:")
|
||||
print(" uv run oem-crawler --brand porsche --limit 20\n")
|
||||
return
|
||||
|
||||
# Brand Breakdown
|
||||
brand_stats = session.execute(
|
||||
select(
|
||||
Vehicle.brand,
|
||||
func.count(Vehicle.id).label("total"),
|
||||
func.avg(Vehicle.price_gross).label("avg_price"),
|
||||
func.min(Vehicle.price_gross).label("min_price"),
|
||||
func.max(Vehicle.price_gross).label("max_price"),
|
||||
)
|
||||
.group_by(Vehicle.brand)
|
||||
.order_by(func.count(Vehicle.id).desc())
|
||||
).all()
|
||||
|
||||
print("\n🏷️ INVENTORY BREAKDOWN BY BRAND:")
|
||||
print(f"{'Brand':<18} {'Count':<10} {'Avg Price':<14} {'Min Price':<14} {'Max Price':<14}")
|
||||
print("-" * 74)
|
||||
for row in brand_stats:
|
||||
b_name, b_count, b_avg, b_min, b_max = row
|
||||
avg_s = f"{b_avg:,.2f} €" if b_avg else "N/A"
|
||||
min_s = f"{b_min:,.2f} €" if b_min else "N/A"
|
||||
max_s = f"{b_max:,.2f} €" if b_max else "N/A"
|
||||
print(f"{b_name:<18} {b_count:<10} {avg_s:<14} {min_s:<14} {max_s:<14}")
|
||||
print("-" * 74)
|
||||
|
||||
if summary_only:
|
||||
print("\n")
|
||||
return
|
||||
|
||||
# Query latest vehicles
|
||||
query = select(Vehicle)
|
||||
if brand:
|
||||
query = query.where(func.lower(Vehicle.brand).like(f"%{brand.lower()}%"))
|
||||
query = query.order_by(Vehicle.created_at.desc()).limit(limit)
|
||||
|
||||
vehicles = session.scalars(query).all()
|
||||
|
||||
filter_lbl = f" for '{brand.upper()}'" if brand else ""
|
||||
print(f"\n🚗 LATEST VEHICLES IN STOCK (Showing up to {limit}{filter_lbl}):\n")
|
||||
for idx, v in enumerate(vehicles, 1):
|
||||
opt_count = session.scalar(
|
||||
select(func.count(VehicleOption.id)).where(VehicleOption.vehicle_id == v.id)
|
||||
) or 0
|
||||
|
||||
condition_badge = f"[{v.condition.upper()}]"
|
||||
mileage_info = f"{v.mileage_km:,} km" if v.mileage_km > 0 else "0 km (New)"
|
||||
first_reg = f" | First Reg: {v.first_registration_date}" if v.first_registration_date else ""
|
||||
|
||||
print(f"[{idx}] {v.brand} - {v.series} ({v.model_name}) {condition_badge}")
|
||||
print(f" • OEM ID: {v.oem_vehicle_id} | Status: {v.status} | Mileage: {mileage_info}{first_reg}")
|
||||
print(f" • Powertrain: {v.fuel_type.upper()} ({v.power_kw or '?'} kW / {v.power_hp or '?'} HP) | Drivetrain: {v.drivetrain or 'N/A'}")
|
||||
if v.battery_capacity:
|
||||
print(f" • EV Battery: {v.battery_capacity} {v.battery_capacity_unit or 'kwh'} | Range WLTP: {v.electric_range_wltp_km or '?'} km | Charging: {v.charging_power_ac_kw or '?'} kW AC / {v.charging_power_dc_kw or '?'} kW DC")
|
||||
print(f" • Price Gross: {v.price_gross:,.2f} {v.currency} | Monthly Rate: {v.monthly_rate or 0:,.2f} {v.currency}")
|
||||
print(f" • Color: {v.exterior_color or 'N/A'}")
|
||||
loc_coords = f" [Lat: {v.latitude}, Lon: {v.longitude}]" if v.latitude and v.longitude else ""
|
||||
print(f" • Location: {v.dealer_name or 'Dealer'} in {v.location_city or 'N/A'} ({v.location_postal_code or 'N/A'}){loc_coords}")
|
||||
print(f" • Options: {opt_count} factory options recorded")
|
||||
print(f" • URL: {v.vehicle_url or 'N/A'}")
|
||||
print("-" * 80)
|
||||
|
||||
# Show Sample options for the first vehicle
|
||||
if vehicles:
|
||||
first = vehicles[0]
|
||||
options = session.scalars(
|
||||
select(VehicleOption).where(VehicleOption.vehicle_id == first.id).limit(6)
|
||||
).all()
|
||||
if options:
|
||||
print(f"\n📦 Sample Options for '{first.model_name}' ({first.oem_vehicle_id}):")
|
||||
for opt in options:
|
||||
badge = "[Standard]" if opt.is_standard else "[Extra]"
|
||||
price = f" (+{opt.price_gross:,.2f} EUR)" if opt.price_gross else ""
|
||||
print(f" - {opt.option_code or '---'}: {opt.name} {badge}{price} (Category: {opt.category or 'General'})")
|
||||
print("=" * 80 + "\n")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="View vehicles and options stored in the database")
|
||||
parser.add_argument("--db-url", type=str, default=settings.DATABASE_URL, help=f"Database connection URL (default: {settings.DATABASE_URL})")
|
||||
parser.add_argument("--limit", type=int, default=10, help="Number of vehicles to display (default: 10)")
|
||||
parser.add_argument("--brand", type=str, default=None, help="Filter by brand (e.g. BMW, PORSCHE, AUDI, MERCEDES)")
|
||||
parser.add_argument("--summary", action="store_true", help="Only show high-level summary and brand table without vehicle list")
|
||||
args = parser.parse_args()
|
||||
|
||||
inspect_stock(db_url=args.db_url, limit=args.limit, brand=args.brand, summary_only=args.summary)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
325
tests/test_audi_parser.py
Normal file
325
tests/test_audi_parser.py
Normal file
|
|
@ -0,0 +1,325 @@
|
|||
"""
|
||||
Unit tests for the Audi Stock Car Service parser and normalization logic.
|
||||
"""
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
import pytest
|
||||
|
||||
from oem_crawler.crawlers.base import CrawlerRegistry
|
||||
from oem_crawler.crawlers.audi.parser import (
|
||||
map_audi_drivetrain,
|
||||
map_audi_fuel_type,
|
||||
map_audi_series,
|
||||
map_audi_transmission,
|
||||
parse_audi_vehicle,
|
||||
parse_german_date,
|
||||
parse_power,
|
||||
)
|
||||
from oem_crawler.db.schemas import (
|
||||
BatteryCapacityUnit,
|
||||
DrivetrainType,
|
||||
FuelType,
|
||||
TransmissionType,
|
||||
VehicleCondition,
|
||||
VehicleStatus,
|
||||
)
|
||||
|
||||
|
||||
class TestAudiParser:
|
||||
def test_registry_contains_audi(self):
|
||||
assert "audi" in CrawlerRegistry.supported_brands()
|
||||
crawler_cls = CrawlerRegistry.get("audi")
|
||||
crawler = crawler_cls(market_country="DE")
|
||||
assert crawler.brand_name == "AUDI"
|
||||
assert crawler.get_market_code("new") == "de"
|
||||
assert crawler.get_market_code("used") == "deuc"
|
||||
|
||||
def test_map_fuel_type(self):
|
||||
assert map_audi_fuel_type({"code": "E", "description": "Elektro"}) == FuelType.BEV
|
||||
assert map_audi_fuel_type("Elektro") == FuelType.BEV
|
||||
assert map_audi_fuel_type({"code": "H", "description": "Hybrid (Benzin/Elektro)"}) == FuelType.PHEV
|
||||
assert map_audi_fuel_type({"code": "B", "description": "Benzin"}) == FuelType.PETROL
|
||||
assert map_audi_fuel_type({"code": "D", "description": "Diesel"}) == FuelType.DIESEL
|
||||
assert map_audi_fuel_type("Mild-Hybrid") == FuelType.MHEV
|
||||
assert map_audi_fuel_type("g-tron Erdgas") == FuelType.CNG
|
||||
|
||||
def test_map_transmission(self):
|
||||
assert map_audi_transmission({"code": "s-tronic", "description": "S tronic"}) == TransmissionType.AUTOMATIC
|
||||
assert map_audi_transmission("tiptronic") == TransmissionType.AUTOMATIC
|
||||
assert map_audi_transmission("Automatik") == TransmissionType.AUTOMATIC
|
||||
assert map_audi_transmission({"code": "manual", "description": "Handschaltung"}) == TransmissionType.MANUAL
|
||||
assert map_audi_transmission("Schaltgetriebe") == TransmissionType.MANUAL
|
||||
assert map_audi_transmission(None) is None
|
||||
|
||||
def test_map_drivetrain(self):
|
||||
assert map_audi_drivetrain({"code": "quattro", "description": "quattro"}) == DrivetrainType.AWD
|
||||
assert map_audi_drivetrain("Allradantrieb") == DrivetrainType.AWD
|
||||
assert map_audi_drivetrain({"code": "fwd", "description": "Frontantrieb"}) == DrivetrainType.FWD
|
||||
assert map_audi_drivetrain("Heckantrieb") == DrivetrainType.RWD
|
||||
assert map_audi_drivetrain(None) is None
|
||||
|
||||
def test_map_series(self):
|
||||
assert map_audi_series({"description": "A3"}) == "A3"
|
||||
assert map_audi_series({"description": "Q4 e-tron"}) == "Q4 e-tron"
|
||||
assert map_audi_series(None, "Audi A1 Sportback 25 TFSI") == "A1"
|
||||
assert map_audi_series(None, "Audi e-tron GT quattro 350 kW") == "e-tron GT"
|
||||
assert map_audi_series(None, "Audi RS6 Avant performance TFSI") == "RS6"
|
||||
assert map_audi_series(None, "Audi Q8 50 TDI quattro") == "Q8"
|
||||
|
||||
def test_parse_power(self):
|
||||
kw, hp = parse_power("110 kW (150 PS)")
|
||||
assert kw == 110
|
||||
assert hp == 150
|
||||
|
||||
kw2, hp2 = parse_power("70 kW")
|
||||
assert kw2 == 70
|
||||
assert hp2 == 95
|
||||
|
||||
kw3, hp3 = parse_power(None)
|
||||
assert kw3 is None
|
||||
assert hp3 is None
|
||||
|
||||
def test_parse_german_date(self):
|
||||
assert parse_german_date("21.07.2015") == date(2015, 7, 21)
|
||||
assert parse_german_date("2021-10-20") == date(2021, 10, 20)
|
||||
assert parse_german_date(1319068800000) == date(2011, 10, 20)
|
||||
assert parse_german_date(None) is None
|
||||
|
||||
def test_parse_audi_new_ice_vehicle(self):
|
||||
basic_item = {
|
||||
"id": "REVVMjA2NjUxNzdBQlNXMzQyMDI1=",
|
||||
"carId": "DEU20665177ABSW342025",
|
||||
"decodedCarId": "DEU20665177ABSW342025",
|
||||
"commissionNumber": "177ABSW342025",
|
||||
"model": {
|
||||
"code": "aacv",
|
||||
"description": "Audi A1 Sportback 25 TFSI 70 kW (95 PS) Schaltgetriebe",
|
||||
},
|
||||
"symbolicCarlineGroup": {
|
||||
"description": "A1",
|
||||
},
|
||||
"bodyType": {
|
||||
"description": "Sportback",
|
||||
},
|
||||
"driveType": {
|
||||
"code": "drive.fwd",
|
||||
"description": "Frontantrieb",
|
||||
},
|
||||
"gearType": {
|
||||
"code": "gear-type.manual",
|
||||
"description": "Handschaltung",
|
||||
},
|
||||
"fuel": {
|
||||
"code": "B",
|
||||
"description": "Benzin",
|
||||
},
|
||||
"powerDisplay": "70 kW (95 PS)",
|
||||
"modelYear": 2025,
|
||||
"typedPrices": [
|
||||
{"amount": 356.0, "currencyCode": "EUR", "type": "financeRate"},
|
||||
{"amount": 21480.0, "currencyCode": "EUR", "type": "retail"},
|
||||
],
|
||||
"dealer": {
|
||||
"id": "20665",
|
||||
"name": "Autohaus Schnitzler GmbH & Co. KG",
|
||||
"street": "Westring 20-22",
|
||||
"zipCode": "40721",
|
||||
"city": "Hilden",
|
||||
"geoLocation": {
|
||||
"lat": 51.17704,
|
||||
"lon": 6.91641,
|
||||
},
|
||||
},
|
||||
"used": {},
|
||||
"extColor": {"description": "Gletscherweiß Metallic", "code": "2Y2Y"},
|
||||
"intColor": {"description": "Schwarz"},
|
||||
"tilesPictures": ["https://mediaservice.audi.com/media/tile.jpg"],
|
||||
"pictures": ["https://mediaservice.audi.com/media/full.jpg"],
|
||||
}
|
||||
|
||||
v = parse_audi_vehicle(basic_item=basic_item, market_country="DE", requested_stock_type="new")
|
||||
|
||||
assert v.brand == "AUDI"
|
||||
assert v.oem_vehicle_id == "DEU20665177ABSW342025"
|
||||
assert v.series == "A1"
|
||||
assert v.model_name == "Audi A1 Sportback 25 TFSI 70 kW (95 PS) Schaltgetriebe"
|
||||
assert v.body_type == "Sportback"
|
||||
assert v.condition == VehicleCondition.NEW
|
||||
assert v.fuel_type == FuelType.PETROL
|
||||
assert v.transmission == TransmissionType.MANUAL
|
||||
assert v.drivetrain == DrivetrainType.FWD
|
||||
assert v.power_kw == 70
|
||||
assert v.power_hp == 95
|
||||
assert v.model_year == 2025
|
||||
assert v.mileage_km == 0
|
||||
assert v.price_gross == Decimal("21480.0")
|
||||
assert v.monthly_rate == Decimal("356.0")
|
||||
assert v.dealer_name == "Autohaus Schnitzler GmbH & Co. KG"
|
||||
assert v.dealer_id == "20665"
|
||||
assert v.location_city == "Hilden"
|
||||
assert v.location_postal_code == "40721"
|
||||
assert v.latitude == Decimal("51.17704")
|
||||
assert v.longitude == Decimal("6.91641")
|
||||
assert v.exterior_color == "Gletscherweiß Metallic"
|
||||
assert v.status == VehicleStatus.IN_STOCK
|
||||
assert "REVVMjA2NjUxNzdBQlNXMzQyMDI1=" in v.vehicle_url
|
||||
assert len(v.image_urls) == 2
|
||||
|
||||
def test_parse_audi_used_vehicle(self):
|
||||
basic_item = {
|
||||
"id": "REVVMjIyNjNHVzE4ODMw=",
|
||||
"carId": "DEU22263GW18830",
|
||||
"model": {
|
||||
"description": "Audi A3 Sportback 1.4 TFSI attraction",
|
||||
},
|
||||
"symbolicCarlineGroup": {
|
||||
"description": "A3",
|
||||
},
|
||||
"fuel": {
|
||||
"code": "B",
|
||||
"description": "Benzin",
|
||||
},
|
||||
"gearType": {
|
||||
"description": "Handschaltung",
|
||||
},
|
||||
"driveType": {
|
||||
"description": "Frontantrieb",
|
||||
},
|
||||
"powerDisplay": "92 kW (125 PS)",
|
||||
"typedPrices": [
|
||||
{"amount": 10890.0, "currencyCode": "EUR", "type": "retail"}
|
||||
],
|
||||
"dealer": {
|
||||
"name": "Autohaus Pietsch.Bünde GmbH",
|
||||
"city": "Bünde",
|
||||
"zipCode": "32257",
|
||||
"geoLocation": {"lat": 52.2025, "lon": 8.57877},
|
||||
},
|
||||
"warrantyPlus": True,
|
||||
"used": {
|
||||
"initialRegistration": "21.07.2015",
|
||||
"mileage": 108859.0,
|
||||
"numPreviousOwners": 2,
|
||||
"pictureUrls": ["https://vtpimages.audi.com/carimg2/used.jpg"],
|
||||
},
|
||||
}
|
||||
|
||||
v = parse_audi_vehicle(basic_item=basic_item, market_country="DE", requested_stock_type="used")
|
||||
|
||||
assert v.brand == "AUDI"
|
||||
assert v.series == "A3"
|
||||
assert v.condition == VehicleCondition.CERTIFIED_PRE_OWNED # warrantyPlus is True
|
||||
assert v.mileage_km == 108859
|
||||
assert v.first_registration_date == date(2015, 7, 21)
|
||||
assert v.previous_owners_count == 2
|
||||
assert v.price_gross == Decimal("10890.0")
|
||||
assert v.latitude == Decimal("52.2025")
|
||||
assert v.longitude == Decimal("8.57877")
|
||||
assert "gebrauchtwagenboerse" in v.vehicle_url
|
||||
|
||||
def test_parse_audi_ev_with_detail(self):
|
||||
basic_item = {
|
||||
"id": "REVVNDcyNTIxNzdaWlpZNDMyMDI2=",
|
||||
"carId": "DEU47252177ZZZY432026",
|
||||
"model": {
|
||||
"description": "Audi Q4 Sportback e-tron 150 kW",
|
||||
},
|
||||
"symbolicCarlineGroup": {
|
||||
"description": "Q4 e-tron",
|
||||
},
|
||||
"fuel": {
|
||||
"code": "E",
|
||||
"description": "Elektro",
|
||||
},
|
||||
"gearType": {
|
||||
"description": "Automatik",
|
||||
},
|
||||
"powerDisplay": "150 kW (204 PS)",
|
||||
"typedPrices": [
|
||||
{"amount": 48900.0, "currencyCode": "EUR", "type": "retail"}
|
||||
],
|
||||
"dealer": {
|
||||
"name": "Audi Zentrum Hamburg",
|
||||
"city": "Hamburg",
|
||||
"zipCode": "22525",
|
||||
"geoLocation": {"lat": 53.58, "lon": 9.92},
|
||||
},
|
||||
}
|
||||
|
||||
detail_data = {
|
||||
"vehicle": {
|
||||
"detail": {
|
||||
"techData": {
|
||||
"fields": [
|
||||
{"name": "Netto-Batterie-Kapazität", "value": "59 kWh"},
|
||||
{"name": "Brutto-Batterie-Kapazität", "value": "63 kWh"},
|
||||
{"name": "Max. Ladeleistung AC / HPC", "value": "11 kW / 160 kW"},
|
||||
{"name": "Ladedauer DC 10-80% bei max. Ladeleistung", "value": "27 min"},
|
||||
{"name": "Elektrische Reichweite kombiniert (WLTP)", "value": "415 km"},
|
||||
]
|
||||
},
|
||||
"features": [
|
||||
{
|
||||
"key": "5SL",
|
||||
"prno": "MASL5SL",
|
||||
"name": "Außenspiegel links, asphärisch",
|
||||
"groupName": "Außenausstattung",
|
||||
"kind": "serie",
|
||||
},
|
||||
{
|
||||
"key": "KA2",
|
||||
"prno": "MKA2KA2",
|
||||
"name": "Rückfahrkamera",
|
||||
"groupName": "Assistenzsysteme",
|
||||
"kind": "sonder",
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
v = parse_audi_vehicle(
|
||||
basic_item=basic_item,
|
||||
detail_data=detail_data,
|
||||
market_country="DE",
|
||||
requested_stock_type="new",
|
||||
)
|
||||
|
||||
assert v.brand == "AUDI"
|
||||
assert v.series == "Q4 e-tron"
|
||||
assert v.fuel_type == FuelType.BEV
|
||||
assert v.battery_capacity == Decimal("59")
|
||||
assert v.battery_capacity_net_kwh == Decimal("59")
|
||||
assert v.battery_capacity_gross_kwh == Decimal("63")
|
||||
assert v.battery_capacity_unit == BatteryCapacityUnit.KWH
|
||||
assert v.charging_power_ac_kw == Decimal("11")
|
||||
assert v.charging_power_dc_kw == Decimal("160")
|
||||
assert v.charging_time_dc_minutes == 27
|
||||
assert v.electric_range_wltp_km == 415
|
||||
|
||||
# Check options
|
||||
assert len(v.options) == 2
|
||||
assert v.options[0].option_code == "5SL"
|
||||
assert v.options[0].is_standard is True
|
||||
assert v.options[1].option_code == "KA2"
|
||||
assert v.options[1].is_standard is False
|
||||
assert v.options[1].category == "Assistenzsysteme"
|
||||
|
||||
def test_fetch_search_page_pagination_params(self):
|
||||
from unittest.mock import MagicMock
|
||||
from oem_crawler.crawlers.audi.crawler import AudiStockCrawler
|
||||
|
||||
crawler = AudiStockCrawler(market_country="DE")
|
||||
mock_client = MagicMock()
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.json.return_value = {"totalCount": 100, "vehicleBasic": []}
|
||||
mock_client.get.return_value = mock_resp
|
||||
|
||||
crawler.fetch_search_page(mock_client, market_code="de", page=2, size=50)
|
||||
|
||||
mock_client.get.assert_called_once()
|
||||
call_kwargs = mock_client.get.call_args.kwargs
|
||||
assert call_kwargs["params"]["from"] == 100
|
||||
assert call_kwargs["params"]["size"] == 50
|
||||
|
||||
74
tests/test_bmw_parser.py
Normal file
74
tests/test_bmw_parser.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
"""
|
||||
Unit tests for BMW Parser and Registry.
|
||||
"""
|
||||
from decimal import Decimal
|
||||
import unittest
|
||||
|
||||
from oem_crawler.crawlers import CrawlerRegistry
|
||||
from oem_crawler.crawlers.bmw.crawler import BMWStockCrawler
|
||||
from oem_crawler.crawlers.bmw.parser import parse_bmw_vehicle
|
||||
|
||||
|
||||
class TestBMWParserAndRegistry(unittest.TestCase):
|
||||
|
||||
def test_registry_contains_bmw(self):
|
||||
supported = CrawlerRegistry.supported_brands()
|
||||
self.assertIn("bmw", supported)
|
||||
cls = CrawlerRegistry.get("bmw")
|
||||
self.assertEqual(cls, BMWStockCrawler)
|
||||
|
||||
def test_bmw_vehicle_parsing(self):
|
||||
raw_bmw = {
|
||||
"documentId": "2454561-61134216",
|
||||
"vin": "WBA11111111111111",
|
||||
"vehicleSpecification": {
|
||||
"modelAndOption": {
|
||||
"brand": "BMW",
|
||||
"baseFuelType": "GASOLINE",
|
||||
"is48Volt": True,
|
||||
"transmission": "AUTOMATIC",
|
||||
"marketingDriveType": "SDRIVE",
|
||||
"model": {"modelName": "120"},
|
||||
"modelRange": {"description": {"de_DE": "BMW 1er"}},
|
||||
"color": {"name": {"de_DE": "Schwarz Uni"}, "code": "668"},
|
||||
"equipments": {
|
||||
"S05DM": {
|
||||
"name": {"de_DE": "Parking Assistant"},
|
||||
"marketingText": {"category": {"de_DE": "Fahrerassistenz"}},
|
||||
"standard": True
|
||||
}
|
||||
}
|
||||
},
|
||||
"technicalAndEmission": {
|
||||
"technicalData": {
|
||||
"power": {"value": 115},
|
||||
"powerInHp": {"value": 156}
|
||||
}
|
||||
}
|
||||
},
|
||||
"pricing": {
|
||||
"price": {"value": 30960.00},
|
||||
"monthlyInstallment": {"value": 199.00}
|
||||
},
|
||||
"ordering": {
|
||||
"retailData": {
|
||||
"locationOutletName": "BMW Autohaus Test",
|
||||
"locationOutletAddress": {"town": "München", "postalCode": "80809", "countryCode": "DE"}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
parsed = parse_bmw_vehicle(raw_bmw, market_country="DE")
|
||||
self.assertEqual(parsed.brand, "BMW")
|
||||
self.assertEqual(parsed.oem_vehicle_id, "2454561-61134216")
|
||||
self.assertEqual(parsed.series, "BMW 1er")
|
||||
self.assertEqual(parsed.model_name, "120")
|
||||
self.assertEqual(parsed.fuel_type.value, "mhev")
|
||||
self.assertEqual(parsed.drivetrain.value, "fwd")
|
||||
self.assertEqual(parsed.power_kw, 115)
|
||||
self.assertEqual(parsed.power_hp, 156)
|
||||
self.assertEqual(parsed.price_gross, Decimal("30960.00"))
|
||||
self.assertEqual(parsed.monthly_rate, Decimal("199.00"))
|
||||
self.assertEqual(len(parsed.options), 1)
|
||||
self.assertEqual(parsed.options[0].option_code, "S05DM")
|
||||
self.assertTrue(parsed.options[0].is_standard)
|
||||
338
tests/test_mercedes_parser.py
Normal file
338
tests/test_mercedes_parser.py
Normal file
|
|
@ -0,0 +1,338 @@
|
|||
"""
|
||||
Unit tests for Mercedes-Benz stock parser.
|
||||
"""
|
||||
from decimal import Decimal
|
||||
import unittest
|
||||
|
||||
from oem_crawler.db.schemas import (
|
||||
BatteryCapacityUnit,
|
||||
DrivetrainType,
|
||||
FuelType,
|
||||
TransmissionType,
|
||||
VehicleCondition,
|
||||
VehicleStatus,
|
||||
)
|
||||
from oem_crawler.crawlers.mercedes.parser import (
|
||||
extract_city_postal_code,
|
||||
map_mercedes_drivetrain,
|
||||
map_mercedes_fuel_type,
|
||||
map_mercedes_transmission,
|
||||
map_mercedes_used_fuel_type,
|
||||
map_mercedes_used_series,
|
||||
parse_mercedes_used_vehicle,
|
||||
parse_mercedes_vehicle,
|
||||
)
|
||||
|
||||
|
||||
class TestMercedesParser(unittest.TestCase):
|
||||
|
||||
def test_extract_city_postal_code(self):
|
||||
city, zip_code = extract_city_postal_code("Mercedesstraße 120, 70372 Stuttgart")
|
||||
self.assertEqual(city, "Stuttgart")
|
||||
self.assertEqual(zip_code, "70372")
|
||||
|
||||
city2, zip_code2 = extract_city_postal_code("Am Seestern 1, 40547 Düsseldorf")
|
||||
self.assertEqual(city2, "Düsseldorf")
|
||||
self.assertEqual(zip_code2, "40547")
|
||||
|
||||
def test_map_fuel_type(self):
|
||||
self.assertEqual(map_mercedes_fuel_type("ELECTRIC", []), FuelType.BEV)
|
||||
self.assertEqual(map_mercedes_fuel_type("PETROL", [{"id": "90W-BI_MILD"}]), FuelType.MHEV)
|
||||
self.assertEqual(map_mercedes_fuel_type("DIESEL", []), FuelType.DIESEL)
|
||||
self.assertEqual(map_mercedes_fuel_type("PLUGIN", []), FuelType.PHEV)
|
||||
|
||||
def test_map_transmission_and_drivetrain(self):
|
||||
self.assertEqual(map_mercedes_transmission("AUTOMATIC"), TransmissionType.AUTOMATIC)
|
||||
self.assertEqual(map_mercedes_transmission("MANUAL"), TransmissionType.MANUAL)
|
||||
self.assertEqual(map_mercedes_drivetrain("C 400 4MATIC"), DrivetrainType.AWD)
|
||||
self.assertEqual(map_mercedes_drivetrain("A 180", "W177"), DrivetrainType.FWD)
|
||||
self.assertEqual(map_mercedes_drivetrain("C 200", "W206"), DrivetrainType.RWD)
|
||||
|
||||
def test_parse_mercedes_vehicle(self):
|
||||
raw_search = {
|
||||
"identification": {
|
||||
"code": "MB8SXSK01",
|
||||
"commissionNumber": "0555673413",
|
||||
"vin": None,
|
||||
},
|
||||
"vehicleModel": {
|
||||
"name": "A 180",
|
||||
"motorization": "A 180",
|
||||
"vehicleClass": {"formattedValue": "A-Klasse"},
|
||||
"bodyType": {"formattedValue": "Kompaktwagen"},
|
||||
"modelYearCode": "806",
|
||||
"typeClass": "W177",
|
||||
},
|
||||
"technicalInformation": {
|
||||
"engine": {
|
||||
"fuelType": {"value": "PETROL"},
|
||||
"power": {
|
||||
"combinedKw": {"value": 110},
|
||||
"combinedHp": {"value": 150},
|
||||
},
|
||||
},
|
||||
"transmission": {"value": "AUTOMATIC"},
|
||||
},
|
||||
"dealer": {
|
||||
"nameLocalLanguage": "Mercedes-Benz AG",
|
||||
"outletId": "GS00113259",
|
||||
"addressLocalLanguage": "Mercedesstraße 120, 70372 Stuttgart",
|
||||
},
|
||||
"emissionAndConsumption": {
|
||||
"attributes": [
|
||||
{"id": "90W-BI_MILD", "value": "5.9", "unit": "l/100km"},
|
||||
{"id": "85W", "value": "133.0", "unit": "g/km"},
|
||||
]
|
||||
},
|
||||
"stock": {"stockType": "AVAILABLE"},
|
||||
"characteristics": {
|
||||
"highlights": [{"label": "Progressive"}, {"label": "Spiegel-Paket"}]
|
||||
},
|
||||
"images": {
|
||||
"default": "https://media.oneweb.mercedes-benz.com/images/default.jpg"
|
||||
},
|
||||
}
|
||||
|
||||
raw_price = {
|
||||
"currency": "EUR",
|
||||
"purchasePrice": {"amount": "38167.76"},
|
||||
"priceBreakDown": [{"type": "BASE_PRICE", "amount": "37453.76"}],
|
||||
"financeLeasingInfo": {
|
||||
"vehicle": {
|
||||
"prices": [
|
||||
{"id": "grossListPrice", "rawValue": "49938.35"},
|
||||
{"id": "baseListPrice", "rawValue": "39513.95"},
|
||||
]
|
||||
}
|
||||
},
|
||||
"taxProfile": "GROSS",
|
||||
}
|
||||
|
||||
raw_vpc = {
|
||||
"color": {
|
||||
"id": "LU-787",
|
||||
"text": {"de": "mountaingrau"},
|
||||
"group": {"de": "Grau"},
|
||||
},
|
||||
"upholstery": {
|
||||
"id": "AU-311",
|
||||
"group": {"de": "Schwarz"},
|
||||
"polish": {"de": "Ledernachbildung ARTICO"},
|
||||
},
|
||||
"components": {
|
||||
"SA-R31": {
|
||||
"code": "R31",
|
||||
"codeType": "SA",
|
||||
"name": {"de": "18\" Leichtmetallräder"},
|
||||
"standard": True,
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
veh = parse_mercedes_vehicle(raw_search, raw_price=raw_price, raw_vpc=raw_vpc)
|
||||
|
||||
self.assertEqual(veh.brand, "Mercedes-Benz")
|
||||
self.assertEqual(veh.oem_vehicle_id, "MB8SXSK01")
|
||||
self.assertEqual(veh.series, "A-Klasse")
|
||||
self.assertEqual(veh.model_name, "A 180")
|
||||
self.assertEqual(veh.trim_line, "Progressive")
|
||||
self.assertEqual(veh.body_type, "Kompaktwagen")
|
||||
self.assertEqual(veh.condition, VehicleCondition.NEW)
|
||||
self.assertEqual(veh.model_year, 2026)
|
||||
self.assertEqual(veh.fuel_type, FuelType.MHEV)
|
||||
self.assertEqual(veh.transmission, TransmissionType.AUTOMATIC)
|
||||
self.assertEqual(veh.drivetrain, DrivetrainType.FWD)
|
||||
self.assertEqual(veh.power_kw, 110)
|
||||
self.assertEqual(veh.power_hp, 150)
|
||||
self.assertEqual(veh.price_gross, Decimal("38167.76"))
|
||||
self.assertEqual(veh.msrp_gross, Decimal("49938.35"))
|
||||
self.assertEqual(veh.discount_percentage, Decimal("23.57"))
|
||||
self.assertEqual(veh.location_city, "Stuttgart")
|
||||
self.assertEqual(veh.location_postal_code, "70372")
|
||||
self.assertEqual(veh.exterior_color, "mountaingrau")
|
||||
self.assertEqual(veh.exterior_color_code, "LU-787")
|
||||
self.assertEqual(veh.interior_color, "Schwarz")
|
||||
self.assertEqual(len(veh.options), 1)
|
||||
self.assertEqual(veh.options[0].option_code, "R31")
|
||||
|
||||
def test_parse_mercedes_electric_vehicle(self):
|
||||
raw_search = {
|
||||
"identification": {"code": "MB_EV_001"},
|
||||
"vehicleModel": {
|
||||
"name": "EQE 350+ 4MATIC",
|
||||
"vehicleClass": {"formattedValue": "EQE"},
|
||||
"bodyType": {"formattedValue": "Limousine"},
|
||||
},
|
||||
"technicalInformation": {
|
||||
"engine": {
|
||||
"fuelType": {"value": "ELECTRIC"},
|
||||
"power": {"combinedKw": {"value": 215}, "combinedHp": {"value": 292}},
|
||||
},
|
||||
"transmission": {"value": "AUTOMATIC"},
|
||||
},
|
||||
"emissionAndConsumption": {
|
||||
"attributes": [
|
||||
{"id": "237W", "value": "16.5", "unit": "kWh/100km"},
|
||||
{"id": "193W", "value": "620", "unit": "km"},
|
||||
{"id": "85W", "value": "0", "unit": "g/km"},
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
raw_vpc = {
|
||||
"technicalInformation": {
|
||||
"engine": {
|
||||
"batteryCapacity": {"value": 141, "unit": "Ah"},
|
||||
"chargingCapacityAC": {"value": 11, "unit": "kW"},
|
||||
"chargingCapacityDC": {"value": 170, "unit": "kW"},
|
||||
},
|
||||
"chargeTime": {
|
||||
"chargeTimeDCHigh": {"value": 22.0, "unit": "min"}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
raw_price = {
|
||||
"purchasePrice": {"amount": "74500.00"},
|
||||
}
|
||||
|
||||
veh = parse_mercedes_vehicle(raw_search, raw_price=raw_price, raw_vpc=raw_vpc)
|
||||
|
||||
self.assertEqual(veh.fuel_type, FuelType.BEV)
|
||||
self.assertEqual(veh.drivetrain, DrivetrainType.AWD)
|
||||
self.assertEqual(veh.electric_range_wltp_km, 620)
|
||||
self.assertEqual(veh.consumption_combined, Decimal("16.5"))
|
||||
self.assertEqual(veh.battery_capacity, Decimal("141"))
|
||||
self.assertEqual(veh.battery_capacity_unit, BatteryCapacityUnit.AH)
|
||||
self.assertEqual(veh.charging_power_ac_kw, Decimal("11"))
|
||||
self.assertEqual(veh.charging_power_dc_kw, Decimal("170"))
|
||||
self.assertEqual(veh.charging_time_dc_minutes, 22)
|
||||
|
||||
def test_map_mercedes_used_series(self):
|
||||
self.assertEqual(map_mercedes_used_series("V 300 d Avantgarde lang"), "V-Klasse")
|
||||
self.assertEqual(map_mercedes_used_series("EQV 300 lang"), "EQV")
|
||||
self.assertEqual(map_mercedes_used_series("EQE 350 4MATIC"), "EQE")
|
||||
self.assertEqual(map_mercedes_used_series("C 200 T-Modell AMG Line"), "C-Klasse")
|
||||
self.assertEqual(map_mercedes_used_series("A 180 Progressive"), "A-Klasse")
|
||||
|
||||
def test_map_mercedes_used_fuel_type(self):
|
||||
self.assertEqual(map_mercedes_used_fuel_type("Elektro"), FuelType.BEV)
|
||||
self.assertEqual(map_mercedes_used_fuel_type("Diesel"), FuelType.DIESEL)
|
||||
self.assertEqual(map_mercedes_used_fuel_type("Benzin"), FuelType.PETROL)
|
||||
self.assertEqual(map_mercedes_used_fuel_type("Plug-in-Hybrid Benzin"), FuelType.PHEV)
|
||||
|
||||
def test_parse_mercedes_used_vehicle_ice(self):
|
||||
card_html = """
|
||||
<div id="resultListItem19170569" class="list-vehicle-card col-md-6">
|
||||
<div class="vehicle_title_headline"><span class="manufacturer">Mercedes-Benz</span><br>V 300 d Avantgarde lang</div>
|
||||
<div class="vehicle_title_subheadline">Fahrzeugnummer 55229GK030</div>
|
||||
<div class="vehicle_price_headline">95.490 €</div>
|
||||
<div class="financing-box"><div class="financing-data">699,00 €</div></div>
|
||||
<ul class="vehicle-card-list">
|
||||
<li class="vc-vehicle-attribute-row"><svg><use href="#vehicle-box-body-type"></use></svg><p class="vc-vehicle-attribute-text">Van/Kleinbus</p></li>
|
||||
<li class="vc-vehicle-attribute-row"><svg><use href="#vehicle-box-first-registration"></use></svg><p class="vc-vehicle-attribute-text">25.07.2025</p></li>
|
||||
<li class="vc-vehicle-attribute-row"><svg><use href="#vehicle-box-power"></use></svg><p class="vc-vehicle-attribute-text">174 kW (237 PS)</p></li>
|
||||
<li class="vc-vehicle-attribute-row"><svg><use href="#vehicle-box-mileage"></use></svg><p class="vc-vehicle-attribute-text">10.018 km</p></li>
|
||||
<li class="vc-vehicle-attribute-row"><svg><use href="#vehicle-box-engine-ice"></use></svg><p class="vc-vehicle-attribute-text">Diesel</p></li>
|
||||
<li class="vc-vehicle-attribute-row result-box-location-item"><span class="vc-vehicle-attribute-text">Mercedes-Benz AG</span><span class="vc-vehicle-attribute-text">71034 Böblingen</span></li>
|
||||
</ul>
|
||||
</div>
|
||||
"""
|
||||
detail_html = """
|
||||
<html>
|
||||
<body>
|
||||
<div class="basic-data"><div class="data-label">Erstzulassung</div><div class="data-value">25.07.2025</div></div>
|
||||
<div class="basic-data"><div class="data-label">Modeljahr</div><div class="data-value">2025</div></div>
|
||||
<div class="basic-data"><div class="data-label">Laufleistung</div><div class="data-value">10.018 km</div></div>
|
||||
<div class="basic-data"><div class="data-label">Hubraum</div><div class="data-value">1.950 ccm</div></div>
|
||||
<div class="basic-data"><div class="data-label">Getriebe</div><div class="data-value">Automatikgetriebe</div></div>
|
||||
<div class="basic-data"><div class="data-label">Garantie</div><div class="data-value">Mercedes-Benz Junge Sterne</div></div>
|
||||
<div class="basic-data"><div class="data-label">Außenfarbe (grau)</div><div class="data-value">Alpingrau</div></div>
|
||||
<div class="basic-data"><div class="data-label">Polster (Leder)</div><div class="data-value">Leder Lugano schwarz</div></div>
|
||||
<script>
|
||||
dealerLocationMap.setCenterCoordinate('48.682361000000000', '8.982236000000000');
|
||||
</script>
|
||||
<div class="eq-search-container"><div class="eq-item">Leichtmetallfelgen 19 Zoll<span> | R1W</span></div></div>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
veh = parse_mercedes_used_vehicle(card=card_html, detail_html=detail_html)
|
||||
|
||||
self.assertEqual(veh.oem_vehicle_id, "19170569")
|
||||
self.assertEqual(veh.raw_payload.get("commission_number"), "55229GK030")
|
||||
self.assertEqual(veh.series, "V-Klasse")
|
||||
self.assertEqual(veh.model_name, "V 300 d Avantgarde lang")
|
||||
self.assertEqual(veh.trim_line, "Avantgarde")
|
||||
self.assertEqual(veh.body_type, "Van/Kleinbus")
|
||||
self.assertEqual(veh.condition, VehicleCondition.CERTIFIED_PRE_OWNED)
|
||||
self.assertEqual(veh.model_year, 2025)
|
||||
self.assertEqual(veh.fuel_type, FuelType.DIESEL)
|
||||
self.assertEqual(veh.transmission, TransmissionType.AUTOMATIC)
|
||||
self.assertEqual(veh.power_kw, 174)
|
||||
self.assertEqual(veh.power_hp, 237)
|
||||
self.assertEqual(veh.engine_displacement_cc, 1950)
|
||||
self.assertEqual(veh.mileage_km, 10018)
|
||||
self.assertEqual(veh.price_gross, Decimal("95490"))
|
||||
self.assertEqual(veh.monthly_rate, Decimal("699.00"))
|
||||
self.assertEqual(veh.exterior_color, "Alpingrau")
|
||||
self.assertEqual(veh.interior_material, "Leder Lugano schwarz")
|
||||
self.assertEqual(veh.dealer_name, "Mercedes-Benz AG")
|
||||
self.assertEqual(veh.location_postal_code, "71034")
|
||||
self.assertEqual(veh.location_city, "Böblingen")
|
||||
self.assertEqual(veh.latitude, Decimal("48.682361000000000"))
|
||||
self.assertEqual(veh.longitude, Decimal("8.982236000000000"))
|
||||
self.assertEqual(len(veh.options), 1)
|
||||
self.assertEqual(veh.options[0].option_code, "R1W")
|
||||
|
||||
def test_parse_mercedes_used_vehicle_ev(self):
|
||||
card_html = """
|
||||
<div id="resultListItem19169425" class="list-vehicle-card">
|
||||
<div class="vehicle_title_headline"><span class="manufacturer">Mercedes-Benz</span><br>EQV 300 lang</div>
|
||||
<div class="vehicle_title_subheadline">Fahrzeugnummer 56F27180</div>
|
||||
<div class="vehicle_price_headline">33.444 €</div>
|
||||
<ul class="vehicle-card-list">
|
||||
<li class="vc-vehicle-attribute-row"><p class="vc-vehicle-attribute-text">150 kW (204 PS)</p></li>
|
||||
<li class="vc-vehicle-attribute-row"><p class="vc-vehicle-attribute-text">121.962 km</p></li>
|
||||
<li class="vc-vehicle-attribute-row"><p class="vc-vehicle-attribute-text">Elektro</p></li>
|
||||
<li class="vc-vehicle-attribute-row result-box-location-item"><span class="vc-vehicle-attribute-text">KTW</span><span class="vc-vehicle-attribute-text">31137 Hildesheim</span></li>
|
||||
</ul>
|
||||
</div>
|
||||
"""
|
||||
detail_html = """
|
||||
<html>
|
||||
<body>
|
||||
<div class="basic-data"><div class="data-label">Erstzulassung</div><div class="data-value">23.05.2022</div></div>
|
||||
<div class="basic-data"><div class="data-label">Modeljahr</div><div class="data-value">2024</div></div>
|
||||
<div class="basic-data"><div class="data-label">Kraftstoffart</div><div class="data-value">Elektro</div></div>
|
||||
<div class="basic-data"><div class="data-label">Außenfarbe (blau)</div><div class="data-value">Cavansitblau</div></div>
|
||||
<div class="bento-2-item"><div class="bento-2-title">Elektrische Reichweite</div><div class="bento-2-value">348 km</div></div>
|
||||
<div class="bento-2-item"><div class="bento-2-title">Energie</div><div class="bento-2-value">90 kWh</div></div>
|
||||
<p>Die Ladeleistung von 110 kW ermöglicht schnelles Laden an DC-Stationen. An Wallboxen mit bis zu 11 kW AC laden.</p>
|
||||
<script>
|
||||
dealerLocationMap.setCenterCoordinate('52.167469000000000', '9.920900000000000');
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
veh = parse_mercedes_used_vehicle(card=card_html, detail_html=detail_html)
|
||||
|
||||
self.assertEqual(veh.oem_vehicle_id, "19169425")
|
||||
self.assertEqual(veh.raw_payload.get("commission_number"), "56F27180")
|
||||
self.assertEqual(veh.series, "EQV")
|
||||
self.assertEqual(veh.model_name, "EQV 300 lang")
|
||||
self.assertEqual(veh.fuel_type, FuelType.BEV)
|
||||
self.assertEqual(veh.battery_capacity, Decimal("90"))
|
||||
self.assertEqual(veh.battery_capacity_unit, BatteryCapacityUnit.KWH)
|
||||
self.assertEqual(veh.battery_capacity_net_kwh, Decimal("90"))
|
||||
self.assertEqual(veh.electric_range_wltp_km, 348)
|
||||
self.assertEqual(veh.charging_power_ac_kw, Decimal("11"))
|
||||
self.assertEqual(veh.charging_power_dc_kw, Decimal("110"))
|
||||
self.assertEqual(veh.latitude, Decimal("52.167469000000000"))
|
||||
self.assertEqual(veh.longitude, Decimal("9.920900000000000"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
145
tests/test_pipeline.py
Normal file
145
tests/test_pipeline.py
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
"""
|
||||
Unit tests for the StockIngestionPipeline.
|
||||
"""
|
||||
from decimal import Decimal
|
||||
import uuid
|
||||
import pytest
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from oem_crawler.db.models import Base, Vehicle, VehicleOption, VehiclePriceHistory, CrawlRun
|
||||
from oem_crawler.db.pipeline import StockIngestionPipeline
|
||||
from oem_crawler.db.schemas import (
|
||||
OptionSchema,
|
||||
VehicleIngestSchema,
|
||||
VehicleStatus,
|
||||
compute_content_hash,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def in_memory_pipeline():
|
||||
engine = create_engine("sqlite:///:memory:")
|
||||
Base.metadata.create_all(engine)
|
||||
test_session_factory = sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
||||
pipeline = StockIngestionPipeline(session_factory=test_session_factory)
|
||||
return pipeline, test_session_factory
|
||||
|
||||
|
||||
def test_pipeline_insert_update_and_idempotency(in_memory_pipeline):
|
||||
pipeline, session_factory = in_memory_pipeline
|
||||
|
||||
test_vehicle = VehicleIngestSchema(
|
||||
oem_vehicle_id="TEST-MB-EQE-001",
|
||||
brand="MERCEDES-BENZ",
|
||||
model_name="EQE 350 4MATIC",
|
||||
series="EQE",
|
||||
stock_type="used",
|
||||
status=VehicleStatus.IN_STOCK,
|
||||
price_gross=Decimal("65900.00"),
|
||||
price_net=Decimal("55378.15"),
|
||||
currency="EUR",
|
||||
mileage_km=15200,
|
||||
power_kw=215,
|
||||
power_hp=292,
|
||||
battery_capacity=Decimal("90.56"),
|
||||
battery_capacity_unit="kwh",
|
||||
charging_power_ac_kw=Decimal("11.00"),
|
||||
charging_power_dc_kw=Decimal("170.00"),
|
||||
electric_range_wltp_km=620,
|
||||
dealer_name="Mercedes-Benz Niederlassung Berlin",
|
||||
location_city="Berlin",
|
||||
location_postal_code="10115",
|
||||
latitude=Decimal("52.5200"),
|
||||
longitude=Decimal("13.4050"),
|
||||
options=[
|
||||
OptionSchema(option_code="P20", name="Fahrassistenz-Paket Plus", category="Packages", is_standard=False),
|
||||
OptionSchema(option_code="413", name="Panorama-Schiebedach", category="Roof", is_standard=False),
|
||||
],
|
||||
raw_payload={"id": "TEST-MB-EQE-001"},
|
||||
)
|
||||
|
||||
# 1. First Run -> Insert
|
||||
def stream_once():
|
||||
yield test_vehicle
|
||||
|
||||
run1 = pipeline.run_crawler_pipeline(
|
||||
brand="MERCEDES-BENZ",
|
||||
crawler_name="test-crawler",
|
||||
vehicle_stream=stream_once(),
|
||||
market_country="DE",
|
||||
)
|
||||
|
||||
assert run1.items_discovered == 1
|
||||
assert run1.items_inserted == 1
|
||||
assert run1.items_updated == 0
|
||||
assert run1.items_unmodified == 0
|
||||
assert run1.items_failed == 0
|
||||
|
||||
with session_factory() as s:
|
||||
veh = s.query(Vehicle).filter_by(oem_vehicle_id="TEST-MB-EQE-001").first()
|
||||
assert veh is not None
|
||||
assert veh.price_gross == Decimal("65900.00")
|
||||
assert veh.battery_capacity == Decimal("90.56")
|
||||
assert veh.battery_capacity_unit == "kwh"
|
||||
assert veh.charging_power_ac_kw == Decimal("11.00")
|
||||
assert veh.charging_power_dc_kw == Decimal("170.00")
|
||||
assert veh.electric_range_wltp_km == 620
|
||||
assert veh.location_city == "Berlin"
|
||||
assert veh.latitude == Decimal("52.5200")
|
||||
assert veh.longitude == Decimal("13.4050")
|
||||
|
||||
# Check options
|
||||
options = s.query(VehicleOption).filter_by(vehicle_id=veh.id).all()
|
||||
assert len(options) == 2
|
||||
assert {opt.option_code for opt in options} == {"P20", "413"}
|
||||
|
||||
# Check price history
|
||||
hist = s.query(VehiclePriceHistory).filter_by(vehicle_id=veh.id).all()
|
||||
assert len(hist) == 1
|
||||
assert hist[0].price_gross == Decimal("65900.00")
|
||||
|
||||
# 2. Second Run -> Unmodified
|
||||
run2 = pipeline.run_crawler_pipeline(
|
||||
brand="MERCEDES-BENZ",
|
||||
crawler_name="test-crawler",
|
||||
vehicle_stream=stream_once(),
|
||||
market_country="DE",
|
||||
)
|
||||
assert run2.items_discovered == 1
|
||||
assert run2.items_inserted == 0
|
||||
assert run2.items_updated == 0
|
||||
assert run2.items_unmodified == 1
|
||||
assert run2.items_failed == 0
|
||||
|
||||
with session_factory() as s:
|
||||
veh = s.query(Vehicle).filter_by(oem_vehicle_id="TEST-MB-EQE-001").first()
|
||||
hist = s.query(VehiclePriceHistory).filter_by(vehicle_id=veh.id).all()
|
||||
assert len(hist) == 1 # No duplicate price history
|
||||
|
||||
# 3. Third Run -> Price Drop Update
|
||||
test_vehicle_updated = test_vehicle.model_copy()
|
||||
test_vehicle_updated.price_gross = Decimal("63900.00")
|
||||
|
||||
def stream_updated():
|
||||
yield test_vehicle_updated
|
||||
|
||||
run3 = pipeline.run_crawler_pipeline(
|
||||
brand="MERCEDES-BENZ",
|
||||
crawler_name="test-crawler",
|
||||
vehicle_stream=stream_updated(),
|
||||
market_country="DE",
|
||||
)
|
||||
assert run3.items_discovered == 1
|
||||
assert run3.items_inserted == 0
|
||||
assert run3.items_updated == 1
|
||||
assert run3.items_unmodified == 0
|
||||
assert run3.items_failed == 0
|
||||
|
||||
with session_factory() as s:
|
||||
veh = s.query(Vehicle).filter_by(oem_vehicle_id="TEST-MB-EQE-001").first()
|
||||
assert veh.price_gross == Decimal("63900.00")
|
||||
hist = s.query(VehiclePriceHistory).filter_by(vehicle_id=veh.id).order_by(VehiclePriceHistory.recorded_at).all()
|
||||
assert len(hist) == 2
|
||||
assert hist[0].price_gross == Decimal("65900.00")
|
||||
assert hist[1].price_gross == Decimal("63900.00")
|
||||
285
tests/test_porsche_parser.py
Normal file
285
tests/test_porsche_parser.py
Normal file
|
|
@ -0,0 +1,285 @@
|
|||
"""
|
||||
Unit tests for the Porsche Finder parser and normalization logic.
|
||||
"""
|
||||
from decimal import Decimal
|
||||
import pytest
|
||||
|
||||
from oem_crawler.crawlers.base import CrawlerRegistry
|
||||
from oem_crawler.crawlers.porsche.parser import (
|
||||
map_porsche_drivetrain,
|
||||
map_porsche_fuel_type,
|
||||
map_porsche_series,
|
||||
map_porsche_transmission,
|
||||
parse_porsche_vehicle,
|
||||
)
|
||||
from oem_crawler.db.schemas import (
|
||||
BatteryCapacityUnit,
|
||||
DrivetrainType,
|
||||
FuelType,
|
||||
TransmissionType,
|
||||
VehicleCondition,
|
||||
VehicleStatus,
|
||||
)
|
||||
|
||||
|
||||
class TestPorscheParser:
|
||||
def test_registry_contains_porsche(self):
|
||||
assert "porsche" in CrawlerRegistry.supported_brands()
|
||||
crawler_cls = CrawlerRegistry.get("porsche")
|
||||
crawler = crawler_cls(market_country="DE")
|
||||
assert crawler.brand_name == "PORSCHE"
|
||||
|
||||
def test_map_fuel_type(self):
|
||||
assert map_porsche_fuel_type("ELECTRIC") == FuelType.BEV
|
||||
assert map_porsche_fuel_type("Elektro") == FuelType.BEV
|
||||
assert map_porsche_fuel_type("PLUG_IN_HYBRID") == FuelType.PHEV
|
||||
assert map_porsche_fuel_type("MILD_HYBRID") == FuelType.MHEV
|
||||
assert map_porsche_fuel_type("DIESEL") == FuelType.DIESEL
|
||||
assert map_porsche_fuel_type("PETROL") == FuelType.PETROL
|
||||
assert map_porsche_fuel_type("Benzin") == FuelType.PETROL
|
||||
|
||||
def test_map_transmission(self):
|
||||
assert map_porsche_transmission("PDK (Automatik)") == TransmissionType.AUTOMATIC
|
||||
assert map_porsche_transmission("Automatik") == TransmissionType.AUTOMATIC
|
||||
assert map_porsche_transmission("Manuell") == TransmissionType.MANUAL
|
||||
assert map_porsche_transmission(None) is None
|
||||
|
||||
def test_map_drivetrain(self):
|
||||
assert map_porsche_drivetrain("ALL_WHEEL_DRIVE") == DrivetrainType.AWD
|
||||
assert map_porsche_drivetrain("Allradantrieb") == DrivetrainType.AWD
|
||||
assert map_porsche_drivetrain("REAR_WHEEL_DRIVE") == DrivetrainType.RWD
|
||||
assert map_porsche_drivetrain("Heckantrieb") == DrivetrainType.RWD
|
||||
|
||||
def test_map_series(self):
|
||||
assert map_porsche_series("Taycan", None) == "Taycan"
|
||||
assert map_porsche_series("911", None) == "911"
|
||||
assert map_porsche_series("Macan", None) == "Macan"
|
||||
assert map_porsche_series("Cayenne", None) == "Cayenne"
|
||||
assert map_porsche_series("Panamera", None) == "Panamera"
|
||||
assert map_porsche_series("718", None) == "718"
|
||||
assert map_porsche_series(None, "Porsche 911 Carrera 4S") == "911"
|
||||
assert map_porsche_series(None, "Porsche Taycan 4S") == "Taycan"
|
||||
|
||||
def test_parse_porsche_new_ice_vehicle(self):
|
||||
search_item = {
|
||||
"id": "RNR7NZ",
|
||||
"listingUrlSlug": "porsche-911-carrera-4s-cabriolet-neu-RNR7NZ",
|
||||
"meta": {
|
||||
"title": "911 Carrera 4S Cabriolet",
|
||||
"condition": "new",
|
||||
"priceValue": 220231.42,
|
||||
"priceCurrency": "EUR",
|
||||
"numberOfPreviousOwners": 0,
|
||||
"mileage": {"value": 0, "unitCode": "KMT"},
|
||||
"model": "992 II",
|
||||
"interiorColor": "Lederausstattung Schwarz",
|
||||
"color": "Schiefergrau Neo",
|
||||
"transmission": "PDK (Automatik)",
|
||||
"modelYear": 2026,
|
||||
"engineType": "PETROL",
|
||||
"modelCategory": "911",
|
||||
"bodyType": "Cabriolet",
|
||||
"drivetrain": "ALL_WHEEL_DRIVE",
|
||||
"enginePower": 353,
|
||||
"engineDisplacement": 2981,
|
||||
"co2Emissions": 245,
|
||||
"fuelConsumption": 10.8,
|
||||
"seller": {
|
||||
"sellerId": "8066",
|
||||
"porschePartnerNumber": "1110050",
|
||||
"name": "Porsche Zentrum Berlin-Potsdam",
|
||||
"addressComponents": {
|
||||
"postalCode": "14532",
|
||||
"city": "Kleinmachnow",
|
||||
"street": "Albert-Einstein-Ring 49",
|
||||
},
|
||||
},
|
||||
},
|
||||
"dataLayerListingMeta": {
|
||||
"car": {
|
||||
"isPorscheApproved": False,
|
||||
"exteriorColorCode": "2T",
|
||||
"interiorColorCode": "VF",
|
||||
}
|
||||
},
|
||||
"gallery": {
|
||||
"imageIds": ["dcf0d51e-230b-444f-bcb2-d37c8f631d98"]
|
||||
},
|
||||
"description": {
|
||||
"title": "Porsche 911 Carrera 4S Cabriolet",
|
||||
"periodicPayments": {
|
||||
"paymentLabel": "2808 per month",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
detail_data = {
|
||||
"equipment": {
|
||||
"specialEquipment": [
|
||||
{
|
||||
"name": "Exterieur",
|
||||
"items": [
|
||||
{"code": "KQ2", "name": "„PORSCHE“ Schriftzug lackiert in Schwarz", "description": "Hochglanz"},
|
||||
{"code": "3FG", "name": "Panoramadach", "description": "Glas"},
|
||||
],
|
||||
}
|
||||
],
|
||||
"importantEquipmentOptions": [
|
||||
{"value": "bose-sound-system", "label": "BOSE Surround Sound-System"}
|
||||
],
|
||||
},
|
||||
"summary": {
|
||||
"characteristics": [
|
||||
{"title": "Motor", "value": "Benzin"},
|
||||
{"title": "Leistung Verbrennungsmotor", "value": "353 kW / 480 PS"},
|
||||
]
|
||||
},
|
||||
"info": {
|
||||
"stockNumber": "D27829",
|
||||
"commissionNumber": "D27829",
|
||||
"vin": "$undefined",
|
||||
},
|
||||
}
|
||||
|
||||
veh = parse_porsche_vehicle(search_item, detail_data, market_country="DE", requested_stock_type="new")
|
||||
|
||||
assert veh.brand == "PORSCHE"
|
||||
assert veh.oem_vehicle_id == "RNR7NZ"
|
||||
assert veh.vin is None
|
||||
assert veh.series == "911"
|
||||
assert veh.model_name == "911 Carrera 4S Cabriolet"
|
||||
assert veh.trim_line == "992 II"
|
||||
assert veh.body_type == "Cabriolet"
|
||||
assert veh.condition == VehicleCondition.NEW
|
||||
assert veh.fuel_type == FuelType.PETROL
|
||||
assert veh.transmission == TransmissionType.AUTOMATIC
|
||||
assert veh.drivetrain == DrivetrainType.AWD
|
||||
assert veh.power_kw == 353
|
||||
assert veh.power_hp == 480
|
||||
assert veh.engine_displacement_cc == 2981
|
||||
assert veh.price_gross == Decimal("220231.42")
|
||||
assert veh.monthly_rate == Decimal("2808")
|
||||
assert veh.dealer_name == "Porsche Zentrum Berlin-Potsdam"
|
||||
assert veh.location_city == "Kleinmachnow"
|
||||
assert veh.location_postal_code == "14532"
|
||||
assert veh.location_country == "DE"
|
||||
assert len(veh.options) == 3
|
||||
option_codes = {opt.option_code for opt in veh.options}
|
||||
assert "KQ2" in option_codes
|
||||
assert "3FG" in option_codes
|
||||
assert "bose-sound-system" in option_codes
|
||||
assert veh.image_urls == ["https://images.finder.porsche.com/dcf0d51e-230b-444f-bcb2-d37c8f631d98/960.webp"]
|
||||
|
||||
def test_parse_porsche_used_certified_vehicle(self):
|
||||
search_item = {
|
||||
"id": "P00XDK",
|
||||
"listingUrlSlug": "porsche-macan-gebraucht-P00XDK",
|
||||
"meta": {
|
||||
"title": "Macan",
|
||||
"condition": "preowned",
|
||||
"priceValue": 56900,
|
||||
"priceCurrency": "EUR",
|
||||
"numberOfPreviousOwners": 1,
|
||||
"mileage": {"value": 35050, "unitCode": "KMT"},
|
||||
"model": "H1 III",
|
||||
"color": "Tiefschwarzmetallic",
|
||||
"firstRegistration": "2022-01-04",
|
||||
"modelYear": 2022,
|
||||
"engineType": "PETROL",
|
||||
"modelCategory": "Macan",
|
||||
"bodyType": "SUV",
|
||||
"drivetrain": "ALL_WHEEL_DRIVE",
|
||||
"enginePower": 195,
|
||||
"seller": {
|
||||
"sellerId": "15713",
|
||||
"name": "Porsche Zentrum Essen",
|
||||
"addressComponents": {"postalCode": "45143", "city": "Essen"},
|
||||
},
|
||||
},
|
||||
"dataLayerListingMeta": {
|
||||
"car": {
|
||||
"isPorscheApproved": True,
|
||||
}
|
||||
},
|
||||
"description": {
|
||||
"title": "Porsche Macan",
|
||||
"periodicPayments": {
|
||||
"paymentLabel": "765.26 per month",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
veh = parse_porsche_vehicle(search_item, detail_data=None, market_country="DE", requested_stock_type="used")
|
||||
|
||||
assert veh.brand == "PORSCHE"
|
||||
assert veh.oem_vehicle_id == "P00XDK"
|
||||
assert veh.series == "Macan"
|
||||
assert veh.condition == VehicleCondition.CERTIFIED_PRE_OWNED
|
||||
assert veh.mileage_km == 35050
|
||||
assert veh.price_gross == Decimal("56900")
|
||||
assert veh.monthly_rate == Decimal("765.26")
|
||||
assert veh.dealer_name == "Porsche Zentrum Essen"
|
||||
assert veh.location_city == "Essen"
|
||||
assert veh.location_postal_code == "45143"
|
||||
|
||||
def test_parse_porsche_electric_option_c(self):
|
||||
search_item = {
|
||||
"id": "EK6EW6",
|
||||
"listingUrlSlug": "porsche-taycan-sport-turismo-black-edition-neu-EK6EW6",
|
||||
"meta": {
|
||||
"title": "Taycan Sport Turismo Black Edition",
|
||||
"condition": "new",
|
||||
"priceValue": 131063,
|
||||
"priceCurrency": "EUR",
|
||||
"mileage": {"value": 0, "unitCode": "KMT"},
|
||||
"model": "J1 II",
|
||||
"color": "Eisgraumetallic",
|
||||
"engineType": "ELECTRIC",
|
||||
"modelCategory": "Taycan",
|
||||
"bodyType": "Sport Turismo",
|
||||
"drivetrain": "REAR_WHEEL_DRIVE",
|
||||
"enginePower": 320,
|
||||
"seller": {
|
||||
"name": "Porsche Zentrum Berlin-Adlershof",
|
||||
"addressComponents": {"postalCode": "12487", "city": "Berlin"},
|
||||
},
|
||||
},
|
||||
"dataLayerListingMeta": {
|
||||
"car": {"isPorscheApproved": False}
|
||||
},
|
||||
}
|
||||
|
||||
detail_data = {
|
||||
"equipment": {
|
||||
"specialEquipment": [
|
||||
{
|
||||
"name": "Motor, Getriebe und Fahrwerk",
|
||||
"items": [
|
||||
{"code": "KB3", "name": "On-Board AC-Lader mit 22 kW", "description": "Schnelles Laden"},
|
||||
],
|
||||
}
|
||||
]
|
||||
},
|
||||
"summary": {
|
||||
"characteristics": [
|
||||
{"title": "Motor", "value": "Elektro"},
|
||||
{"title": "Reichweite kombiniert (WLTP)", "value": "636 km"},
|
||||
{"title": "Brutto-Energieinhalt Batterie", "value": "105,0 kWh"},
|
||||
{"title": "Ladezeit für Gleichstrom (DC) mit maximaler Ladeleistung (10 auf bis zu 80%)", "value": "18,00 min"},
|
||||
{"title": "Antrieb", "value": "Heckantrieb"},
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
veh = parse_porsche_vehicle(search_item, detail_data, market_country="DE", requested_stock_type="new")
|
||||
|
||||
assert veh.brand == "PORSCHE"
|
||||
assert veh.series == "Taycan"
|
||||
assert veh.fuel_type == FuelType.BEV
|
||||
assert veh.drivetrain == DrivetrainType.RWD
|
||||
assert veh.battery_capacity == Decimal("105.0")
|
||||
assert veh.battery_capacity_unit == BatteryCapacityUnit.KWH
|
||||
assert veh.electric_range_wltp_km == 636
|
||||
assert veh.charging_time_dc_minutes == 18
|
||||
assert veh.charging_power_ac_kw == Decimal("22.00")
|
||||
assert veh.charging_power_dc_kw == Decimal("320.00")
|
||||
206
tests/test_raw_storage.py
Normal file
206
tests/test_raw_storage.py
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
"""
|
||||
Tests for RawStorageManager, external compressed storage, and raw extraction toggle.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import json
|
||||
import shutil
|
||||
from datetime import datetime, timezone
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from oem_crawler.config import settings
|
||||
from oem_crawler.crawlers.audi.parser import parse_audi_vehicle
|
||||
from oem_crawler.crawlers.bmw.parser import parse_bmw_vehicle
|
||||
from oem_crawler.db.models import Base, Vehicle, VehicleOption
|
||||
from oem_crawler.db.pipeline import StockIngestionPipeline
|
||||
from oem_crawler.db.schemas import (
|
||||
OptionSchema,
|
||||
VehicleCondition,
|
||||
VehicleIngestSchema,
|
||||
VehicleStatus,
|
||||
)
|
||||
from oem_crawler.storage import RawStorageManager
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_raw_dir(tmp_path):
|
||||
orig_dir = settings.RAW_STORAGE_DIR
|
||||
orig_enabled = settings.ENABLE_RAW_EXTRACTION
|
||||
raw_dir = tmp_path / "raw_storage"
|
||||
settings.RAW_STORAGE_DIR = str(raw_dir)
|
||||
settings.ENABLE_RAW_EXTRACTION = True
|
||||
yield raw_dir
|
||||
settings.RAW_STORAGE_DIR = orig_dir
|
||||
settings.ENABLE_RAW_EXTRACTION = orig_enabled
|
||||
|
||||
|
||||
def test_raw_storage_manager_save_and_load(temp_raw_dir):
|
||||
payload = {"brand": "BMW", "specs": {"engine": "v8", "power": 390}}
|
||||
timestamp = datetime(2026, 11, 7, 12, 0, 0, tzinfo=timezone.utc)
|
||||
|
||||
# 1. Save
|
||||
rel_path = RawStorageManager.save_payload(
|
||||
brand="BMW",
|
||||
oem_vehicle_id="VIN-123456",
|
||||
payload=payload,
|
||||
timestamp=timestamp,
|
||||
)
|
||||
|
||||
assert rel_path == "bmw/2026-11-07/VIN-123456.json.gz"
|
||||
full_path = temp_raw_dir / rel_path
|
||||
assert full_path.exists()
|
||||
|
||||
# 2. Verify compression
|
||||
with gzip.open(full_path, "rt", encoding="utf-8") as f:
|
||||
loaded_raw = json.load(f)
|
||||
assert loaded_raw == payload
|
||||
|
||||
# 3. Load via Manager
|
||||
loaded = RawStorageManager.load_payload(rel_path)
|
||||
assert loaded == payload
|
||||
|
||||
# 4. Delete
|
||||
deleted = RawStorageManager.delete_payload(rel_path)
|
||||
assert deleted is True
|
||||
assert not full_path.exists()
|
||||
|
||||
|
||||
def test_raw_storage_manager_disabled(temp_raw_dir):
|
||||
settings.ENABLE_RAW_EXTRACTION = False
|
||||
payload = {"key": "val"}
|
||||
|
||||
rel_path = RawStorageManager.save_payload(
|
||||
brand="AUDI",
|
||||
oem_vehicle_id="AUDI-001",
|
||||
payload=payload,
|
||||
)
|
||||
assert rel_path is None
|
||||
assert not (temp_raw_dir / "audi").exists()
|
||||
|
||||
|
||||
def test_parser_include_raw_toggle():
|
||||
# BMW parser with include_raw=False
|
||||
raw_bmw = {
|
||||
"documentId": "TEST-BMW-RAW",
|
||||
"vehicleSpecification": {
|
||||
"modelAndOption": {
|
||||
"model": {"modelName": "BMW 330i"},
|
||||
"modelRange": {"description": {"de_DE": "3er"}},
|
||||
}
|
||||
},
|
||||
"pricing": {"price": {"value": 55000}},
|
||||
"extra_huge_catalog": {"items": [1, 2, 3] * 1000},
|
||||
}
|
||||
|
||||
veh_with_raw = parse_bmw_vehicle(raw_bmw, include_raw=True)
|
||||
assert veh_with_raw.raw_payload == raw_bmw
|
||||
|
||||
veh_no_raw = parse_bmw_vehicle(raw_bmw, include_raw=False)
|
||||
assert veh_no_raw.raw_payload == {}
|
||||
assert veh_no_raw.model_name == "BMW 330i"
|
||||
assert veh_no_raw.series == "3er"
|
||||
|
||||
|
||||
def test_pipeline_ingestion_with_raw_storage(temp_raw_dir, tmp_path):
|
||||
db_file = tmp_path / "test_stock.db"
|
||||
db_url = f"sqlite:///{db_file}"
|
||||
engine = create_engine(db_url)
|
||||
Base.metadata.create_all(engine)
|
||||
session_factory = sessionmaker(bind=engine)
|
||||
|
||||
pipeline = StockIngestionPipeline(session_factory=session_factory)
|
||||
|
||||
test_vehicle = VehicleIngestSchema(
|
||||
brand="BMW",
|
||||
oem_vehicle_id="BMW-INGEST-01",
|
||||
series="3er",
|
||||
model_name="320d",
|
||||
condition=VehicleCondition.NEW,
|
||||
fuel_type="diesel",
|
||||
mileage_km=0,
|
||||
price_gross=Decimal("45000.00"),
|
||||
status=VehicleStatus.IN_STOCK,
|
||||
options=[
|
||||
OptionSchema(option_code="S01", name="Sport Line", raw_data={"pr": "S01"}),
|
||||
],
|
||||
raw_payload={"full_spec": "all details here", "code": "BMW-INGEST-01"},
|
||||
)
|
||||
|
||||
def stream():
|
||||
yield test_vehicle
|
||||
|
||||
run = pipeline.run_crawler_pipeline(
|
||||
brand="BMW",
|
||||
crawler_name="test-crawler",
|
||||
vehicle_stream=stream(),
|
||||
market_country="DE",
|
||||
)
|
||||
assert run.items_inserted == 1
|
||||
|
||||
with session_factory() as session:
|
||||
veh = session.query(Vehicle).filter_by(oem_vehicle_id="BMW-INGEST-01").first()
|
||||
assert veh is not None
|
||||
# In-database raw_payload is kept empty to save disk space
|
||||
assert veh.raw_payload == {}
|
||||
# Relative path points to the compressed file
|
||||
assert veh.raw_payload_path is not None
|
||||
assert veh.raw_payload_path.startswith("bmw/")
|
||||
assert veh.raw_payload_path.endswith(".json.gz")
|
||||
|
||||
# Options raw_data in database is kept empty
|
||||
opt = session.query(VehicleOption).filter_by(vehicle_id=veh.id).first()
|
||||
assert opt.name == "Sport Line"
|
||||
assert opt.raw_data == {}
|
||||
|
||||
# Loading via helper method reads from disk seamlessly
|
||||
recovered_payload = veh.get_raw_payload()
|
||||
assert recovered_payload == {"full_spec": "all details here", "code": "BMW-INGEST-01"}
|
||||
|
||||
|
||||
def test_restore_database_from_raw(temp_raw_dir, tmp_path):
|
||||
from oem_crawler.tools.restore_from_raw import restore_database
|
||||
|
||||
# Create dummy raw payload files in temp_raw_dir
|
||||
payload = {
|
||||
"documentId": "BMW-RESTORE-99",
|
||||
"vehicleSpecification": {
|
||||
"modelAndOption": {
|
||||
"model": {"modelName": "BMW 520d"},
|
||||
"modelRange": {"description": {"de_DE": "5er"}},
|
||||
}
|
||||
},
|
||||
"pricing": {"price": {"value": 62000}},
|
||||
}
|
||||
RawStorageManager.save_payload(
|
||||
brand="bmw",
|
||||
oem_vehicle_id="BMW-RESTORE-99",
|
||||
payload=payload,
|
||||
)
|
||||
|
||||
db_file = tmp_path / "restored.db"
|
||||
db_url = f"sqlite:///{db_file}"
|
||||
|
||||
# Restore into fresh DB
|
||||
restore_database(
|
||||
raw_dir=temp_raw_dir,
|
||||
db_url=db_url,
|
||||
target_brands=["bmw"],
|
||||
)
|
||||
|
||||
engine = create_engine(db_url)
|
||||
session_factory = sessionmaker(bind=engine)
|
||||
with session_factory() as session:
|
||||
veh = session.query(Vehicle).filter_by(oem_vehicle_id="BMW-RESTORE-99").first()
|
||||
assert veh is not None
|
||||
assert veh.model_name == "BMW 520d"
|
||||
assert veh.price_gross == Decimal("62000")
|
||||
assert veh.raw_payload_path.startswith("bmw/")
|
||||
assert veh.raw_payload == {}
|
||||
|
||||
|
||||
82
tests/test_schema.py
Normal file
82
tests/test_schema.py
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
"""
|
||||
Unit tests for data schemas and content hashing.
|
||||
"""
|
||||
from decimal import Decimal
|
||||
import unittest
|
||||
|
||||
from oem_crawler.db.schemas import (
|
||||
OptionSchema,
|
||||
VehicleCondition,
|
||||
VehicleIngestSchema,
|
||||
compute_content_hash,
|
||||
)
|
||||
|
||||
|
||||
class TestSchemas(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.sample_payload = {
|
||||
"brand": "BMW",
|
||||
"oem_vehicle_id": "BMW-DE-STOCK-839210",
|
||||
"vin": "WBA33AY08NFP12345",
|
||||
"series": "3 Series",
|
||||
"model_name": "330e xDrive Touring M Sport",
|
||||
"condition": "new",
|
||||
"fuel_type": "phev",
|
||||
"transmission": "automatic",
|
||||
"power_kw": 215,
|
||||
"mileage_km": 10,
|
||||
"price_gross": Decimal("69250.00"),
|
||||
"options": [
|
||||
{
|
||||
"option_code": "05AU",
|
||||
"name": "Driving Assistant Professional",
|
||||
"category": "Driver Assistance",
|
||||
"price_gross": Decimal("2200.00"),
|
||||
"is_standard": False,
|
||||
},
|
||||
{
|
||||
"option_code": "0402",
|
||||
"name": "Panorama Glass Roof",
|
||||
"price_gross": Decimal("1500.00"),
|
||||
"is_standard": False,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
def test_schema_parsing(self):
|
||||
vehicle = VehicleIngestSchema(**self.sample_payload)
|
||||
self.assertEqual(vehicle.brand, "BMW")
|
||||
self.assertEqual(len(vehicle.options), 2)
|
||||
self.assertEqual(vehicle.options[0].option_code, "05AU")
|
||||
|
||||
def test_content_hash_stability(self):
|
||||
v1 = VehicleIngestSchema(**self.sample_payload)
|
||||
v2 = VehicleIngestSchema(**self.sample_payload)
|
||||
self.assertEqual(compute_content_hash(v1), compute_content_hash(v2))
|
||||
|
||||
def test_hash_sensitivity_on_price_change(self):
|
||||
v1 = VehicleIngestSchema(**self.sample_payload)
|
||||
data2 = dict(self.sample_payload, price_gross=Decimal("68000.00"))
|
||||
v2 = VehicleIngestSchema(**data2)
|
||||
self.assertNotEqual(compute_content_hash(v1), compute_content_hash(v2))
|
||||
|
||||
def test_ev_battery_and_charging_schema(self):
|
||||
ev_payload = dict(
|
||||
self.sample_payload,
|
||||
fuel_type="bev",
|
||||
battery_capacity=Decimal("141.00"),
|
||||
battery_capacity_unit="ah",
|
||||
battery_capacity_net_kwh=Decimal("108.40"),
|
||||
charging_power_ac_kw=Decimal("11.00"),
|
||||
charging_power_dc_kw=Decimal("170.00"),
|
||||
charging_time_dc_minutes=22,
|
||||
)
|
||||
ev_vehicle = VehicleIngestSchema(**ev_payload)
|
||||
self.assertEqual(ev_vehicle.battery_capacity, Decimal("141.00"))
|
||||
self.assertEqual(ev_vehicle.battery_capacity_unit.value, "ah")
|
||||
self.assertEqual(ev_vehicle.battery_capacity_net_kwh, Decimal("108.40"))
|
||||
self.assertEqual(ev_vehicle.charging_power_ac_kw, Decimal("11.00"))
|
||||
self.assertEqual(ev_vehicle.charging_power_dc_kw, Decimal("170.00"))
|
||||
self.assertEqual(ev_vehicle.charging_time_dc_minutes, 22)
|
||||
|
||||
773
uv.lock
generated
Normal file
773
uv.lock
generated
Normal file
|
|
@ -0,0 +1,773 @@
|
|||
version = 1
|
||||
revision = 3
|
||||
requires-python = ">=3.10"
|
||||
resolution-markers = [
|
||||
"python_full_version >= '3.12'",
|
||||
"python_full_version >= '3.11.5' and python_full_version < '3.12'",
|
||||
"python_full_version < '3.11.5'",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "annotated-types"
|
||||
version = "0.8.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyio"
|
||||
version = "4.14.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "exceptiongroup", marker = "python_full_version < '3.11'" },
|
||||
{ name = "idna" },
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "beautifulsoup4"
|
||||
version = "4.15.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "soupsieve", version = "2.10", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11.5'" },
|
||||
{ name = "soupsieve", version = "3.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11.5'" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
version = "2026.7.22"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "colorama"
|
||||
version = "0.4.6"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "exceptiongroup"
|
||||
version = "1.3.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.11.5'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "greenlet"
|
||||
version = "3.5.5"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/0b/d8/7cc97c142388aef03f622e001c572c4f84e9252a439549d483f555771970/greenlet-3.5.5.tar.gz", hash = "sha256:adb4bae02e91a8e863e48b177e4014bdcac8a6b5e047ea1df687a61534b85e6c", size = 207585, upload-time = "2026-08-10T15:09:36.136Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/07/67/07ecd6d85c0f253363cbc4ac9e7dd048ca571a267fbccfea084d6009ac3b/greenlet-3.5.5-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:816230f469381ad0a43abc9fa8dda5a699e32fb78958dde32ded93213b70a667", size = 292971, upload-time = "2026-08-10T13:28:09.837Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/24/35/426733bc24247ee17bb76df90f550c299ff5f8572b2bdd7cacb5c53fd994/greenlet-3.5.5-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a5433cf291e0ef9114bd14d0d824db6e5e4a43033234bca48181a9597acca07b", size = 609290, upload-time = "2026-08-10T14:14:32.273Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9c/dc/17d3a5acceb2fd0bbc9682a228117b719cdfb68085e6dbb33fa325755f27/greenlet-3.5.5-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:19d59f068887d8c5907fc177f27683413ace3011b6ed646c0b309266e74a6502", size = 622650, upload-time = "2026-08-10T14:27:22.004Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/74/3f/b31e6bd6adb8f80492efb6a47e8f9cd0e7335db5aac2795b1876d5afcfee/greenlet-3.5.5-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:86c5113d698cb8d927b2750bb1f1d59eefe3a37e0e0217491aee29a7f84ef52c", size = 629560, upload-time = "2026-08-10T14:30:04.733Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/51/91/00b3c0566316c6f383ea16ee05388ec4f57f6e0afa49e72ae471eda8c47f/greenlet-3.5.5-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ff00e12102358292087274dfb1669132387ff6e7920ebf9d85f4826ce0d3a56", size = 622819, upload-time = "2026-08-10T13:40:46.562Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/18/5e/58c561efde575c4ca070030e2e0513e8d1b07b76d8a2d84a9bcef1becd42/greenlet-3.5.5-cp310-cp310-manylinux_2_39_riscv64.whl", hash = "sha256:c69bed34470abfcd456984fdadaa18e62169af4480335c45f3c32d1d9c12e638", size = 425489, upload-time = "2026-08-10T14:29:59.768Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3c/78/52de4f7ac9152ad1dbc8f437895c1e573d30ee1e1427e0f91a280e2417b6/greenlet-3.5.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:523bb8e27614d77101ea7a8cf59f8d91219b72d5c29f6a038c92b50828bfa8d0", size = 1582164, upload-time = "2026-08-10T14:15:02.645Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f6/ff/c3855a00c2417e8f61c7b9f0bc4f8f599c6928f5c15681ad251e78a91e05/greenlet-3.5.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f1e2db190db51c17433eee424803818cf0670bf049d9cfe0dd07be111d1aa7c4", size = 1648807, upload-time = "2026-08-10T13:40:27.471Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/70/21/d29ff6a79dbd1ec1fe74c155d25d4c5a85e221d6b9ffc2cc7a709e7c8c39/greenlet-3.5.5-cp310-cp310-win_amd64.whl", hash = "sha256:740e544169527b82695ce76af2f7ad6f030904658f2f3921a1d245771fb88cfc", size = 322832, upload-time = "2026-08-10T13:28:10.85Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4e/a3/07297917485ee2ca85bc3c8dc6ed85ad3fffcf424047fba62671dba68e97/greenlet-3.5.5-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:be63afcbbccfad3dd95a1ba12ada84dab2ef32031973d80b5b92df67fa763a61", size = 294165, upload-time = "2026-08-10T13:25:17.987Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/db/51/6f732f9314cda54c5fd48a7620c7160f4f286967e8045ad94b9d66ce80b7/greenlet-3.5.5-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a268024ce2d7d2b04694bf1594058981a9fa663d1df4b762dee499211ed7c1c", size = 613610, upload-time = "2026-08-10T14:14:33.829Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d8/c0/b27589e25d220289edcd4d582b2b17b83058d1a56d53d971b6ea1a34f10d/greenlet-3.5.5-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:35cbb8bf55ace57fbccb4fb8622c4521713acd8691e77f4696d416ea7ca527da", size = 625481, upload-time = "2026-08-10T14:27:23.647Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/39/82/5c873dbb4fb001d22fbbd50e80d4c1b0181ddae106856132160f84b94e88/greenlet-3.5.5-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:abc8bc8d9f935cd685457545b6a53863a877fdc12c2c0f5ee9beee18d9db139c", size = 633329, upload-time = "2026-08-10T14:30:06.062Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/51/2d/f2c928218ac52f26d7a2c188c171d1b7e728b23782cb3347e7b4fce1493a/greenlet-3.5.5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74cc6df89ec5302337adc9cf096221cbed2510fd444b0e0f1586cf0470740864", size = 624562, upload-time = "2026-08-10T13:40:48.064Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/70/12/f7df98e72a8eb4a7edfec5a08d6d1a4ab53a52c95ba0b2ea6c10b8dd9bd0/greenlet-3.5.5-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:3134291427bb0f3526e9d90311988caf336eb43730e95244997a4fb15f45144f", size = 428145, upload-time = "2026-08-10T14:30:01.071Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3e/4a/92fc51d5d35912f4f06eec037ba347985defd0be47463a010a325634d9d2/greenlet-3.5.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d9b454c5fc48aeaa7c4337813dbf513a6870468e426438a04d922c6d0fe63db", size = 1584909, upload-time = "2026-08-10T14:15:04.343Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ac/58/ed98b80ac5738c149a5258544843c45601ade1fd70f61740cdaead6351b3/greenlet-3.5.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03551ed792cb1b4fc0277a0c60dfd8c343894a0ba06fe60dcd22f568b433da39", size = 1651184, upload-time = "2026-08-10T13:40:28.879Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d8/be/b582ceb80cefdf9d8da34078714e4b12b3d16f509dee0f65e40a5cc8fc7d/greenlet-3.5.5-cp311-cp311-win_amd64.whl", hash = "sha256:ab3df3dffb58bf70564e93a5cec7941e4d9faa5a36cc4234a10d3131afe04f53", size = 323280, upload-time = "2026-08-10T13:26:07.495Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4d/18/5313c4c58598c38b0373c013e4ff2b3e6d258aaaa338f373335ebecdaddd/greenlet-3.5.5-cp311-cp311-win_arm64.whl", hash = "sha256:2b70a766135540c472ac1393d57c2e1b4a2eb85bf526a1e41e6d096173a8cee5", size = 307785, upload-time = "2026-08-10T13:28:34.874Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2e/7e/9ecd0285e3153532ae07aeb88063c43c72b4221cf0d4d123b02f3682e3ff/greenlet-3.5.5-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:49520f0c95a48b42cf55414b8e8479beb274ea70431afc33e3f79903c71f4380", size = 295809, upload-time = "2026-08-10T13:25:34.023Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/73/60e4bbcc89252037b18087f2ec16405d5b2d5be42dde191bbf3667e96102/greenlet-3.5.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55272212cbc5f43d1d723725ab931f1939969b7e9523882ca58b55061769d053", size = 611910, upload-time = "2026-08-10T14:14:35.18Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a4/17/cd5134be659cd4a443e7a61ae670dabec165a814c51162916d637b6dd38e/greenlet-3.5.5-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:655bca754a2ef4efcb0eb48a94d3f4593536d0f3d48f8ed44343c01d16a92f95", size = 624198, upload-time = "2026-08-10T14:27:25.229Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9b/30/87c212b5c684d0e72974f1063b7a9687631e8985902c06e1016542c874e7/greenlet-3.5.5-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6ca5d6ae0739e5764f2cfcfaa562ac5a990cbdaedca93251c5e3cf07c362371f", size = 629504, upload-time = "2026-08-10T14:30:07.967Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/78/ac/5c5b959999b6f09c3026b5dfe171575bc3121c5236ce74f495096f25b203/greenlet-3.5.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:147b25a42e5ca5be3d42356e8f608b37af715a1c196e9bf9d1627f3341adfe1d", size = 621439, upload-time = "2026-08-10T13:40:49.391Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/63/2c/eb487fafc9f50ffff2b1e0b697f70fb34bf150821c08ab225aacf5583a7e/greenlet-3.5.5-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:1b5ed9162c0c098e0bbc2cf88a94f433c1b8926f831745252e099e5d83e17759", size = 432462, upload-time = "2026-08-10T14:30:02.309Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c8/8b/6acf112ed8aee499f25b4d6949820fb02ac950ff9c1f3d793bd5be0599f2/greenlet-3.5.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:27493374cff1d1b7919dc8126547f2aea582737e3046147b434b1e12de56389b", size = 1581342, upload-time = "2026-08-10T14:15:05.653Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b8/d7/734e5f198888876b42d7616ff6644c075baf6b8a2412deadd6b0e1b8b20c/greenlet-3.5.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:12e2ee66c2aba86133f10fd99d6a8856c6d351ffb7be0e4d52ef2cc5fbb705b2", size = 1645744, upload-time = "2026-08-10T13:40:30.353Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/de/30/1f42b88dc587b5899ee50616ad56ee40cafaf225df4fb829f10183c62a5c/greenlet-3.5.5-cp312-cp312-win_amd64.whl", hash = "sha256:49ddacd36af37735fab103846f4ee4d18a492dde72730d1699c0c8ebe30d9f18", size = 324171, upload-time = "2026-08-10T13:28:44.472Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/76/e5/4dee4d8d2e603fe5fdd7b444e63219f7b9bd852c60c6214511c7157cbe88/greenlet-3.5.5-cp312-cp312-win_arm64.whl", hash = "sha256:5f1b1ff4828cdc1aba4266aff814085d04a1d07959287219af021b838b265d52", size = 308362, upload-time = "2026-08-10T13:26:46.839Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fb/3d/8cef5f724ec0d4add2af8961d504535ec60c3cca9e464f6d03bdba29d85b/greenlet-3.5.5-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:b79fd2a5bc099b5e744f34c4c9a58954a5f4cb7529fb4b6e8446057d61b6edaa", size = 294730, upload-time = "2026-08-10T13:27:51.206Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/88/4b/8e7aa3f514273aecff30a16ab1bac09ff54cfc7e6860fdd8058c37ff2499/greenlet-3.5.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:634cf15a233a949136879dd388e25d3296e16f3f1e217d2456797b8579ebc6ed", size = 614536, upload-time = "2026-08-10T14:14:36.589Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/85/48/4e95e9dd5a8a397dc6a6345dd7f1935113d0fca4f85e89d3976da9cd988d/greenlet-3.5.5-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:499adea519f748407fc6806d20eedabac2884fd73b9f38d81236e190ba20dfef", size = 626924, upload-time = "2026-08-10T14:27:27.048Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0e/84/eaa476d6bf3816828d0d70e80dcc36bf30a058233bd889e707e693f6e860/greenlet-3.5.5-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7278591501941bb2456af102bb9cd59aab48c6cfd6e2dd68fa1290bb0c49a42", size = 632726, upload-time = "2026-08-10T14:30:09.874Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/89/5d/398a1c71fa7a277deeb376c999979de6786f08fc2d5747a0b9d6e11738dd/greenlet-3.5.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2eabb980975cba5b93a95f6f69287d05fc05ac955bfd6a320a7c083eeb52c0b0", size = 623906, upload-time = "2026-08-10T13:40:50.501Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/f2/0cc2849ede68579291e9c59b3ab6ec1958f98681cca5b14d8fc75bf674a4/greenlet-3.5.5-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:4dfc7c4470354e7b09184d1a3a985761053a2fd694ddb5b5c80242afc2c8c90b", size = 434966, upload-time = "2026-08-10T14:30:03.729Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/04/1b/745450fc5ea9e0cb17d840d248f284db3363de736d362c7d2d883e3eadba/greenlet-3.5.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:03115c2e0a371999bf8ae616aa8d653f96641d4705c457aebaa187276e9f7537", size = 1581430, upload-time = "2026-08-10T14:15:06.853Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d4/29/d51b296e3191bb15d3d81ec375af1909e4466c0f395d744ed475801798a9/greenlet-3.5.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4441153ffba21b90d3ca89fe3d31f5c093ae6c0bf0cfdfc98f54cde22f95b62e", size = 1645684, upload-time = "2026-08-10T13:40:32.133Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/12/63/369f1a1625e64e9e31df3963c6044056e3fdfa3fa3fdba3c54ffefa6e987/greenlet-3.5.5-cp313-cp313-win_amd64.whl", hash = "sha256:95c5b1f4b3a193f8a0c2de4bfdcb48d119f7f1063941f1de1f2168051b3e52dd", size = 324075, upload-time = "2026-08-10T13:26:58.974Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/45/78/649cb5c09d4d81f6dd1444e75474a7206784743283a21d24171562ac4899/greenlet-3.5.5-cp313-cp313-win_arm64.whl", hash = "sha256:1af90aa4bc129883b340cdd6957a3bc74f60528a4993bbd1f53aaebe1d9981cc", size = 308260, upload-time = "2026-08-10T13:27:50.795Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7f/8c/080e881fa2be95ff1ddbd6994b2bab3b1a78df3b3fcab39306011764fcc7/greenlet-3.5.5-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d4a389a852e392a6366058651a20fa5ba40d979865aa81bea2ccbdc44805070d", size = 295309, upload-time = "2026-08-10T13:26:03.032Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/25/cc/0ac614e6586c0e42d4cc281a5819150f4f43685744a4c5ff77139286409d/greenlet-3.5.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70b157cd319873e8b544ddc2de158f55bbd0a9b0218c8ce9332039801518e328", size = 661185, upload-time = "2026-08-10T14:14:37.867Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5e/b9/6808725354be8ad305dfe5172377664fc9642d4fc043be246b3314cf4482/greenlet-3.5.5-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8bdfd1424abcf26832961e766570cae79efdb9599d709088c9cb6ef82b194926", size = 673419, upload-time = "2026-08-10T14:27:28.652Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/eb/52/f005d579acde46c3d1cc3cab1c9f3d5708c8a3006a4120e8cf5da801afe9/greenlet-3.5.5-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d98ef6f92e67c6dbf299dbfd8facc1b0d2d9cedf91e325e73b3d0373fe4309d8", size = 677863, upload-time = "2026-08-10T14:30:11.663Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/42/2e/40c509967da7f254680826a2fa0dd22138ec79946c70b97542d74cde8b43/greenlet-3.5.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:182de51c6b572a705f2fafaab2e783bcf7d2760940229dfe73086cbae037af3e", size = 670822, upload-time = "2026-08-10T13:40:51.833Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/8a/a75f8a2bdcef3c358a3147cdc9db3aa83755f0a038f766ab0bedb66f512c/greenlet-3.5.5-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:159df1942d88e8f784cbb38d6f18bdb365cd11319cfbb3e89623de2b97892d53", size = 480554, upload-time = "2026-08-10T14:30:05.171Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2d/22/c3c2eee4a8fe191d6d1d183086c56133d646024e3d70bfd414829f64560b/greenlet-3.5.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8fec3f165dfe332e490c3247c0f6c23b0bfc45f06496ad7f00ddb00e3d35e4dc", size = 1628469, upload-time = "2026-08-10T14:15:08.11Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f7/87/25babd09b94cb1f03e71db815fde463f0262e40cfbd953d58a8d77311351/greenlet-3.5.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c6ce25fee6cabc8bf22cb8b52e642cbb821be5b9aec8094d07ff03378141b8e9", size = 1691952, upload-time = "2026-08-10T13:40:33.502Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2e/3d/5cc9701117ea4dc0eb7bf1f4f9b7888a6e2e5277ddfae095805ace50f2b6/greenlet-3.5.5-cp314-cp314-win_amd64.whl", hash = "sha256:7dffc5c859fe6059974df1e37d7923d654a83e2ae18fdd616994270e001115e1", size = 327458, upload-time = "2026-08-10T13:27:02.868Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a7/6b/594fa2de7fae7629168a404a4305d7d7e31a5742c50a801b1839543cb93d/greenlet-3.5.5-cp314-cp314-win_arm64.whl", hash = "sha256:5e2afcfc4d4305dd715809b03da5cbe437c8984f61d8917751eb5fe4aefa3e07", size = 311146, upload-time = "2026-08-10T13:27:25.046Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/24/e0/50cd600b469e5734c72709b6b1838b6bc63f307b573c772c3132d6ecfe92/greenlet-3.5.5-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:0e5a7de979d764aea1f5b6e95cf92b5b37741b9823702041f34b126e7f690277", size = 305471, upload-time = "2026-08-10T13:26:20.568Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/75/a3/77acd66dfc6387b5219b2080806c0cabb73c10eb1bb44b413c40a62015ba/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fef01bd457f11fc158b130ca0027a3c365693280e8e231b65bdaf57999f39f5b", size = 672470, upload-time = "2026-08-10T14:14:39.058Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b9/71/0d178142dca3ec19f46fb2212ae73d30ad53b9d548dc64804086033a7089/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5173a72310725a74afc82c164f0e52cb8ad0de62f2bb623f24f6c0cc07d80272", size = 679973, upload-time = "2026-08-10T14:27:30.072Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/aa/ac/0d7887aa4bbfc9eba075cc428244dfc96f623478454d5ec81180d0d6bd5a/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5e9ec2e7c98e895fcea0c5cc57b2606cf86ece6d0a56578f3eb225e2af4f0387", size = 681587, upload-time = "2026-08-10T14:30:13.519Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6e/31/46eb8567302eaf787abf88d09df014e14ae3baf460af1b8b0efdbd3efcd5/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44f08341873200ba8a60a8bc14ace3d91f1754f7fa7bc66157714a8cd420a476", size = 676634, upload-time = "2026-08-10T13:40:53.004Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4f/18/8d58ba1c429b0383e3219a3d0e0bba241d0444d8ed05b73349953c7d7c7b/greenlet-3.5.5-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:102817506f6090b5176c746a82603341a549b40e5c3d5b72a4c672228a918c41", size = 510175, upload-time = "2026-08-10T14:30:07.047Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a3/e9/b88bbf5b29970cb84172dc2c32aa3e5e579ceb94c808e81c826454138850/greenlet-3.5.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d246c0db9a2513cd45f019ba178ea4d4d4705bd210ee465e2c15d76a1ab13874", size = 1637320, upload-time = "2026-08-10T14:15:09.317Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6d/8c/7631ed29cc6f0392f11830076e172ce4885e70b0bc2c1bce1731176d4b4e/greenlet-3.5.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:72507285b5caa1d17904a3f7c322ca780823a54170a0e04ec3f37bcc60d4db71", size = 1697412, upload-time = "2026-08-10T13:40:34.924Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/da/0f/f7dd935f9c4cb1be49098770587f54d8a78518e55c89bce86c4fb4109057/greenlet-3.5.5-cp314-cp314t-win_amd64.whl", hash = "sha256:7805655781fb8f28a55d05fe57ed61f5f10f1892fb587673e3bb5264f28041f0", size = 331514, upload-time = "2026-08-10T13:29:20.611Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b7/e5/681b01f8fbc1b55232822f99e8f8afeb78a55a7c76a7bf9dbdc7ccb03a6d/greenlet-3.5.5-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:c0db80fcd5b8aece93f66c64f78a786bbb6b96c5fe63ef5a5a4581ecf8bab206", size = 295975, upload-time = "2026-08-10T13:28:45.985Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/11/f2/69b488cd9e7267bf4b0fe8cdebf25d8d6df680d21bdf41150d23e23d6652/greenlet-3.5.5-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b241c32f912ada659808d68e308c568baf577eebf757d15471472de0c18cfad", size = 666823, upload-time = "2026-08-10T14:14:40.222Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/84/d4/d5bc2fdebbdda0c94555925ba79948b8395d75a7f6a36cc85dce5bab9f11/greenlet-3.5.5-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ef6a08349401d8eaf3cb12688ac8557de95788556b8631ef17555a4a173022c0", size = 677613, upload-time = "2026-08-10T14:27:31.543Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/65/53/4e13642efc4d7ad6554ecb2242a5be42666b2e1a067323e88dfc0124a04b/greenlet-3.5.5-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:37faa97daccb6d9f4c2141ce3118d023c3c5506864a7d8bdf726f665018c1f76", size = 681436, upload-time = "2026-08-10T14:30:14.839Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bd/93/542d8a3a90f3b35c6ad8bf7e56a03010287f2cafa289a5b7985b5207db39/greenlet-3.5.5-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f2e3d061b8e13aec2f0441689b3c71b244a20e5d274a52cb0f7e31bd1d139552", size = 675930, upload-time = "2026-08-10T13:40:54.205Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cd/32/188447c9a468d6977d2989397226b0c6b65ab6f4cf943f931643328512fc/greenlet-3.5.5-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:b18007dc2473a7942fd157366b55f01da6fed7ce85318591005b419e0a439474", size = 487404, upload-time = "2026-08-10T14:30:08.903Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/52/b5/89c9f2e8460d71101037d47a1feed11928615a5edd42370be290e0657eeb/greenlet-3.5.5-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:9ab5f5b93655e77fe0d6c2dfd22b5eac751bb1f876d8ec21761b7c1fb9266007", size = 1633878, upload-time = "2026-08-10T14:15:10.693Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b8/60/297de93f3b02ac78a5e04d32bb8bbe3080f4a73d8ed95016561463b70618/greenlet-3.5.5-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:f0e5a21bd4452a88cf032fc43c4a5b307ab1380eacb63b5988f9c0317885e773", size = 1696597, upload-time = "2026-08-10T13:40:36.252Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/18/25/54c6eaff4f337fb670215e89eb2d00d9499487b658e709d4b477be4a342e/greenlet-3.5.5-cp315-cp315-win_amd64.whl", hash = "sha256:469dbb0a78625642f4a626cfd0c6e8bccc0385b5e49189b6308bbe849ec88a8e", size = 327700, upload-time = "2026-08-10T13:28:06.752Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/67/67/857e88a36301caa0e029870132c2478bd55d896630321432afab03a3115f/greenlet-3.5.5-cp315-cp315-win_arm64.whl", hash = "sha256:2d57406c3efd32d7a81e17a674314e8bd00792cdab49ea3228a49aa1bfb2e769", size = 311750, upload-time = "2026-08-10T13:34:08.815Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/10/e2/3144c0a116067ac1e30457b0139a94d60d1d36a86e015de68e9ac87cb3bc/greenlet-3.5.5-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:68184dfcf50ccaa8e864770fe0633a7e27250ea9329f8192ef47ee9ecfd78e1c", size = 306387, upload-time = "2026-08-10T13:27:00.897Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5c/a1/cb4223a7e9b9f43b8807e8eb212358bfe2dfaa174a9ea2889eb1714dcba2/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9ec0dc0e59dc9c61af5c47348365ccbbd7addfafe0a93b00336ff3da2907bdc6", size = 676472, upload-time = "2026-08-10T14:14:41.417Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/cd/a154b4498e5d8f12ada291cfb3b8d596eadde2177f5bf09a9be699d2a446/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e604f58e35833fc46ef20302bcb314dddbfd3fcf33a4f936216d51dd678d63ae", size = 684238, upload-time = "2026-08-10T14:27:32.946Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ce/f4/e450a68a152f819491d8c7df6a8254e761d87e6a78759268961f8c5bd4dd/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2888a3a38bc5ee5bb6c438372197152e815837e4fab7ed7a1f86ef18ffd58ad1", size = 686022, upload-time = "2026-08-10T14:30:15.96Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bf/bb/b0031d260c2968a3c87deebc51d80c64e499377f993aafe06ee3b7488cc2/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:40239b5384f96da3963585cc6d7eaa9b56f8ae67e8d92cc82dd9e202fc847de3", size = 681246, upload-time = "2026-08-10T13:40:55.402Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/18/23/17e63d6bf3b9c9b9dbea981b7f643a71f79603bdfb4f1c3a9cf353e22aed/greenlet-3.5.5-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:1e8d9391fe77f15649589a907cef972dbbd6352ef7ff7dc0492f658c0c26495f", size = 516951, upload-time = "2026-08-10T14:30:10.907Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9a/07/da554b71ab88e649da146e1065d86a48a5c5d92e50ab74ef41b504aa7f56/greenlet-3.5.5-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:a1eaccf5c3a1d3e46dead602c72e6836731e8e245c9de6a27764567b6b62d4c0", size = 1642735, upload-time = "2026-08-10T14:15:11.92Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/78/76/26a3782a051677668af9d92beaa47cd87ba9dd5072f762961144a03dd4c6/greenlet-3.5.5-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:19e4e026fe20691f333b8eb1a3bc9625eceba8c3f9d62ec5a6f8581afbc6b5a5", size = 1700925, upload-time = "2026-08-10T13:40:37.656Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/28/d9/fe7baf4190c2ae71f267efb9de21b3172bb35bc0ed1ef53dd6027d658e33/greenlet-3.5.5-cp315-cp315t-win_amd64.whl", hash = "sha256:712aee154f648bde84634654bb38bb78c69ac640c37a45c9effed800735049d8", size = 331829, upload-time = "2026-08-10T13:26:48.851Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/df/af/419a4e383bd600858a9b67e9b280a60fdc383ee3f2fe5b6c0c1ef04e74d1/greenlet-3.5.5-cp315-cp315t-win_arm64.whl", hash = "sha256:7f049911ee81a16a03c33d5450d8d5867d27f596ca5fb201b86f4524e874468b", size = 315093, upload-time = "2026-08-10T13:29:34.949Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h11"
|
||||
version = "0.16.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "4.4.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "hpack" },
|
||||
{ name = "hyperframe" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e7/85/7c366e69d84c17bb778fe41419e1fbcce3033d5b7ce29bbffff0a98b859f/h2-4.4.1.tar.gz", hash = "sha256:4e866ffb1a869ae14dd9b5e6beb5c24a13da0495ad72b65925ded182521c1516", size = 2157281, upload-time = "2026-08-03T11:45:09.509Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/22/e85faf23bd72a92d1921e37d674ca56eb298a3c8be31fdecef0ff2b3aaac/h2-4.4.1-py3-none-any.whl", hash = "sha256:0e25f1462b23c9cb82d9eb02e28bc706dac2a68cb457c6a0d74d63c8a2a5d0e6", size = 62636, upload-time = "2026-08-03T11:44:59.164Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hpack"
|
||||
version = "4.2.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/26/5b/fcabf6028144a8723726318b07a32c2f3314acdff6265743cf08a344b18e/hpack-4.2.0.tar.gz", hash = "sha256:0895cfa3b5531fc65fe439c05eb65144f123bf7a394fcaa56aa423548d8e45c0", size = 51300, upload-time = "2026-06-23T18:34:46.667Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/71/b4/4a9fcfb2aef6ba44d9073ecd301443aa00b3dac95de5619f2a7de7ec8a91/hpack-4.2.0-py3-none-any.whl", hash = "sha256:858ac0b02280fa582b5080d68db0899c62a80375e0e5413a74970c5e518b6986", size = 34246, upload-time = "2026-06-23T18:34:45.472Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "httpcore"
|
||||
version = "1.0.9"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "certifi" },
|
||||
{ name = "h11" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "httpx"
|
||||
version = "0.28.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "anyio" },
|
||||
{ name = "certifi" },
|
||||
{ name = "httpcore" },
|
||||
{ name = "idna" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
http2 = [
|
||||
{ name = "h2" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyperframe"
|
||||
version = "6.1.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "3.19"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/5f/f7/abb373e5757eaec4b922b92f97ec8d6d7e057cf06778247604fbc4e7c3f3/idna-3.19.tar.gz", hash = "sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15", size = 215237, upload-time = "2026-08-18T05:14:24.27Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4", size = 68550, upload-time = "2026-08-18T05:14:22.343Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iniconfig"
|
||||
version = "2.3.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "oem-crawler"
|
||||
version = "0.1.0"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "beautifulsoup4" },
|
||||
{ name = "httpx", extra = ["http2"] },
|
||||
{ name = "playwright" },
|
||||
{ name = "psycopg", extra = ["binary"] },
|
||||
{ name = "pydantic" },
|
||||
{ name = "pydantic-settings" },
|
||||
{ name = "sqlalchemy" },
|
||||
]
|
||||
|
||||
[package.dev-dependencies]
|
||||
dev = [
|
||||
{ name = "pytest" },
|
||||
]
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "beautifulsoup4", specifier = ">=4.15.0" },
|
||||
{ name = "httpx", extras = ["http2"], specifier = ">=0.27.0" },
|
||||
{ name = "playwright", specifier = ">=1.64.1" },
|
||||
{ name = "psycopg", extras = ["binary"], specifier = ">=3.1.18" },
|
||||
{ name = "pydantic", specifier = ">=2.7.0" },
|
||||
{ name = "pydantic-settings", specifier = ">=2.2.0" },
|
||||
{ name = "sqlalchemy", specifier = ">=2.0.30" },
|
||||
]
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
dev = [{ name = "pytest", specifier = ">=8.0.0" }]
|
||||
|
||||
[[package]]
|
||||
name = "packaging"
|
||||
version = "26.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "playwright"
|
||||
version = "1.64.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "greenlet" },
|
||||
{ name = "pyee", version = "13.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
|
||||
{ name = "pyee", version = "14.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/8d/c8/cc677856d89a0c077673b40ab915ed5597bf1347b205cabe70121c431ca9/playwright-1.64.1-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:29453ff76e25c17d0b45658cacd6b92e9eec46e8db21e134fb3b7143df50f013", size = 44508971, upload-time = "2026-10-22T16:29:58.12Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/72/4f/572bbbe60a5787ca61a84bad934c3ce19c08c2bc9c0e9781bf0d99cad1a6/playwright-1.64.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:59f7b041284ce8d1b72098ae50ca48a2d83692eec25154ab5ba79d4b6b4fe8a9", size = 43155232, upload-time = "2026-10-22T16:30:02.415Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/44/fa/468f313a56d33a6420705147c8d47bef558f087856edf38c5b13eef8aa6c/playwright-1.64.1-py3-none-macosx_11_0_universal2.whl", hash = "sha256:6a6f99adad1ae40e6ee62efe89785db861ce7ddd57651521cb31894183cc4ae3", size = 44508977, upload-time = "2026-10-22T16:30:07.632Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/15/b9/e481a34572933b5cf179d5637c98da488c732ee339dff5b73a0624d2e42b/playwright-1.64.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:577b8d88778c245cfbd145ae17cf53044f084b1b46c55c69081d4069d56057e2", size = 48471931, upload-time = "2026-10-22T16:30:12.831Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0c/11/545f25e47ec54e1e8a2892ef9994fc2b298f909ab37b88578e3a1f87ac2e/playwright-1.64.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a29fda44725e11b0b625c4bff6458a1f6dbee97b88c9788d12e18a8344458c2d", size = 48150994, upload-time = "2026-10-22T16:30:18.297Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/33/87/8ff7af99387ee72ae9b46a66ece7a0c0aa63f73320ecf6dd3dd087fa840a/playwright-1.64.1-py3-none-win32.whl", hash = "sha256:d3c48b31da426c2f3562d0919cb9ea4e5415f6dbd29d7211fa4d66cca24fbd2f", size = 38801828, upload-time = "2026-10-22T16:30:23.154Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/96/0a/64db6851aa1852a4f76be28eea71d4e2bf9f3ce493ee2e66939426729046/playwright-1.64.1-py3-none-win_amd64.whl", hash = "sha256:5e240a79464434c9260957b9ef59a5808e02e96e3fcabdc717fd7485260b9730", size = 38801836, upload-time = "2026-10-22T16:30:28.092Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c5/4b/d7ddde9467a5fcfe6a0d3c954732332e98a4f4a9025fd42c8d091c16e5f9/playwright-1.64.1-py3-none-win_arm64.whl", hash = "sha256:983ccd085453ea5a7588193657fc349a6ffbf7bf7ba663fd58dc81e4aa3af9d9", size = 34693875, upload-time = "2026-10-22T16:30:32.724Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pluggy"
|
||||
version = "1.6.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "psycopg"
|
||||
version = "3.3.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
|
||||
{ name = "tzdata", marker = "sys_platform == 'win32'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/db/2f/cb91e5502ec9de1de6f1b76cfbf69531932725361168bb06963620c77e2e/psycopg-3.3.4.tar.gz", hash = "sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc", size = 165799, upload-time = "2026-05-01T23:31:55.179Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/5c/e0/7b3dee031daae7743609ce3c746565d4a3ed7c2c186479eb48e34e838c64/psycopg-3.3.4-py3-none-any.whl", hash = "sha256:b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a", size = 213001, upload-time = "2026-05-01T23:20:50.816Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
binary = [
|
||||
{ name = "psycopg-binary", marker = "implementation_name != 'pypy'" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "psycopg-binary"
|
||||
version = "3.3.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/b7/bf/70d8a60488f9955cbbcd538beae44d56bb2f1d19e673b72788f2d343ff55/psycopg_binary-3.3.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b7bfff1ca23732b488cbca3076fc11bc98d520ee122514fdb17a8e20d3338f5a", size = 4609750, upload-time = "2026-05-01T23:24:20.06Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/db/b0/29e98ba210c9dbc75a6dc91e3f99b9e06ea901a62ca95804e02a1ae13e6b/psycopg_binary-3.3.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:32a6fbf8481e3a370d0d72b860d35948a693cb01281da217f7b2f307636e591a", size = 4676700, upload-time = "2026-05-01T23:25:21.727Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8e/ab/3df087b3c12bf74e47c08204172b2fabb5a144679110d5c7ad12d9201323/psycopg_binary-3.3.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bdef84570ebbce1d42b4e7ea952d21c414c5f118ad02fee00c5625f35e134429", size = 5496319, upload-time = "2026-05-01T23:25:28.271Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/87/9a/f088207b4cd6772f9e0d8a91807e79fa2458d4eb9eb1ae406c68415f2bec/psycopg_binary-3.3.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa1cbc10768a796c96d3243656016bf4e337c81c71097270bb7b0ad6210d9765", size = 5171906, upload-time = "2026-05-01T23:25:34.004Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/48/45/4523a857f253871d75c22e1c2e79fd47e599e736bcba1bad58d83e24be02/psycopg_binary-3.3.4-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cf7f73a4a792bc5db58a4b385d8a1467e8d468f7548702fb0ed1e9b7501b1c13", size = 6762621, upload-time = "2026-05-01T23:25:41.392Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/d1/925bf776503345bef428e6c45fb017d0139ddbe0e211814b585c4253dca8/psycopg_binary-3.3.4-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d7b4d40c153fa352ab3cca530f3a0baedf7621b2ebcbd7f084009522c21788fc", size = 5006319, upload-time = "2026-05-01T23:25:51.419Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6f/aa/99727337206fbba357ca084bf4ea8b29dc986f61842a2685859af61416db/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9b1c2533af01cd7648378599f82b0b8ae32f293296e6eec5753a625bc97ef28", size = 4535388, upload-time = "2026-05-01T23:25:57.957Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0b/a4/567ba2c37d19d8c2f63d836385dfd2495aa5897bbee6cfab104d9ee58624/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ad3bc94054876155549fdaedf4a46d1ec69d39a5bcee377148afe498e84c4b8e", size = 4224544, upload-time = "2026-05-01T23:26:03.832Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b7/23/86457f5a82731685d7701de7bfaa5eb783dd1fecbf875321897d9d9ce33a/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:eb4eed2079c01a4850bf467deacfab56d356d4225040170af03dc9958321242d", size = 3956282, upload-time = "2026-05-01T23:26:09.983Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a7/d8/249456df16d47de082abd9b73bce8ccdeb0293eb12e590f9150c7cbdb788/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f80e3f2b5331dbbf0901bcb658056c03eeb2c1ef31d774afb0d61598b242e744", size = 4261736, upload-time = "2026-05-01T23:26:16.798Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/15/6b/c4abe228acafd8a385c1fb615d4f1e3c9b8ad7a4e4f0e84118ba3ffeed9c/psycopg_binary-3.3.4-cp310-cp310-win_amd64.whl", hash = "sha256:574ea21a9651958f1535c5a1c649c7409e9168bcbffa29a3f2f961f58b322949", size = 3570620, upload-time = "2026-05-01T23:26:22.655Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b6/82/df3312c0ca083d5b43b352f27d4dd8b1e614bd334473074715d9e0000da4/psycopg_binary-3.3.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:612a627d733f695b1de1f9b4bd511c15f999a5d8b915d444bbd7dd71cf3370da", size = 4609813, upload-time = "2026-05-01T23:26:30.612Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1f/b5/d74d542458d3e8ac0571d8a88f57ca369999b9a82f4fa528052d0d7d3e4c/psycopg_binary-3.3.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:13a7f380824c35896dcac7fe0f61440f7ca49d6dc73f3c13a9a4471e6a3b302e", size = 4676799, upload-time = "2026-05-01T23:26:38.475Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/09/67/06bab9c60671999f4c6ceff1b334f3ac1f9fc5789eb467c714623ea21de9/psycopg_binary-3.3.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:276904e3452d6a23d474ef9a21eee19f20eed3d53ddd2576af033827e0ba0992", size = 5497050, upload-time = "2026-05-01T23:26:47.061Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/72/9b/023433e2b20f970de1e22d29132a95281277646da0b2e2879dd4ee94b8c1/psycopg_binary-3.3.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab8cca8ef8fb1ccf5b048ae5bd78ba55b9e4b5d472e3ce5ca39ff4d2a9c249e4", size = 5172428, upload-time = "2026-05-01T23:26:56.708Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/08/cd/ae16da8fde228a38b2fe9269bbc13cf89e0186173f2265600f02d6a71e64/psycopg_binary-3.3.4-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7465bfe6087d2d5b42d4c53b9b11ca9f218e477317a4a162a10e3c19e984ba8e", size = 6762746, upload-time = "2026-05-01T23:27:07.023Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4f/81/0ba09fa5f5f88779093a2541a8e02489825721f258ab88058b11d68b3eb5/psycopg_binary-3.3.4-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:22cdbf5f91ef7bb91fe0c5757e1962d3127a8010256eefd9c61fcaf441802097", size = 5006033, upload-time = "2026-05-01T23:27:12.221Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/73/6a/629136040cc3497adb442a305710b5913f2a754d4630fc3d3717c4c0df65/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e2631da29253a98bd496e6c4813b24e09a4fe3fb2a9e88513305d6f8747cce95", size = 4534175, upload-time = "2026-05-01T23:27:18.248Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/32/1027f843c6dc2d5d51960ee62cc0c2cf755a4c39455aff1371173edbef7d/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7f7668f30b9dd5163197e5cbf4e0efd54e00f0a859cc566ce56cfc31f4054839", size = 4224203, upload-time = "2026-05-01T23:27:24.3Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0b/e1/380a724d9093c74adb14d4fce920ea8327838abb61f760b1448586b14a8e/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:cffc3408d77a27973f33e5d909b624cce683db5fc25964b02fe0aae7886c1007", size = 3954509, upload-time = "2026-05-01T23:27:30.815Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/db/cd/895893ae575a09c97ccfd5def070d88993d955ef34df45a881fd5ff506d6/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0579252a1202cd73e4da137a1426e2dae993ae44e757605344282af3a082848c", size = 4259551, upload-time = "2026-05-01T23:27:38.828Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/dd/c6/2330a20794e37a3ec609ef2fd8522919ec7a4395a1abf979a8e2d1775cd5/psycopg_binary-3.3.4-cp311-cp311-win_amd64.whl", hash = "sha256:41f2ec0fea529832982bcb6c9415de3c86264ebe562b77a467c0fbcd7efbba8d", size = 3572054, upload-time = "2026-05-01T23:27:45.455Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/95/7d/03818e13ba7f36de93573c93ee3482006d3dfa8b0f8d28df511bad0a1a92/psycopg_binary-3.3.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5ab28a2a7649df3b72e6b674b4c190e448e8e77cf496a65bd846472048de2089", size = 4591122, upload-time = "2026-05-01T23:27:56.162Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a5/b9/11b341edf8d54e2694726b273fe9652b254d989f4f63e3ac6816ad6b55f4/psycopg_binary-3.3.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6402a9d8146cf4b3974ded3fd28a971e83dc6a0333eb7822524a3aa20b546578", size = 4669943, upload-time = "2026-05-01T23:28:04.522Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/18/4665bacd65e7865b4372fcd8abb8b9186ada4b0025f8c2ca691b364a556c/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:580ae30a5f95ccd90008ec697d3ed6a4a2047a516407ad904283fa42086936e9", size = 5469697, upload-time = "2026-05-01T23:28:11.337Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/b1/b83136c6e510593d9b0c759ba5384337bc4ad82d19fda675adc4b2703c84/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e7510c37550f91a187e3660a8cc50d4b760f8c3b8b2f89ebc5698cd2c7f2c85d", size = 5152995, upload-time = "2026-05-01T23:28:20.529Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/67/8d/a9821e2a648afe6091989929982a3b0f00b2631a859cb81379728f08fb75/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77df19583501ea288eaf15ac0fe7ad01e6d8091a91d5c41df5c718f307d8e31b", size = 6738180, upload-time = "2026-05-01T23:28:30.654Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/58/2e349e8d23905dc2317b80ac65f48fb6f821a4777a4e994a60da91c4850f/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:018fbed325936da502feb546642c982dcc4b9ffdea32dfef78dbf3b7f7ad4070", size = 4978828, upload-time = "2026-05-01T23:28:37.277Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/45/48/57b00d03b4721878326122a1f1e6b0a90b85bcaec56b5b2f8ea6cfa45235/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:17a21953a9e5ff3a16dab692625a3676e2f101db5e40072f39dbee2250194d68", size = 4509757, upload-time = "2026-05-01T23:28:43.078Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/25/37/33b47d8c007df69aec500df5889767c4d313748e8e9e27a2fef8a6dabcee/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:eb05ee1c2b817d27c537333224c9e83c7afb86fe7296ba970990068baf819b16", size = 4190546, upload-time = "2026-05-01T23:28:50.016Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ca/c6/32b0835dbc2122617902b649d76a91c1e75406e76bf3d595b0c3bb5ffad6/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:773d573e11f437ce0bdb95b7c18dc58390494f96d43f8b45b9760436114f7652", size = 3926197, upload-time = "2026-05-01T23:28:55.55Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cd/68/d190ef0c0c5b16ded07831dabc8ddd412f4cdab07ec6e30ed38d9bda0e1f/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e55ccbdfae79a2ed9c6369c3008a3025817ff9d7e27b32a2d84e2a4267e66e", size = 4236627, upload-time = "2026-05-01T23:29:05.336Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/25/8f/81dcbc2e8454b74d14881275ea45f00791052dac531a9fa8be1730d1685b/psycopg_binary-3.3.4-cp312-cp312-win_amd64.whl", hash = "sha256:494ca54901be8cf9eb7e02c25b731f2317c378efa44f43e8f9bd0e1184ae7be4", size = 3560782, upload-time = "2026-05-01T23:29:11.967Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/09/43/13e9c406fbbf354580476e248a16b64802a376873ebe6339e30bb655572d/psycopg_binary-3.3.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7", size = 4590377, upload-time = "2026-05-01T23:29:18.782Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/be/2923cd7c3683e7afdecf4f10796a18de02f5c5ddc0969aa2ad0a8cdd3bbd/psycopg_binary-3.3.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:75a9067e236f9b9ae3535b66fe99bddb33d39c0de10112e49b9ab11eee53dc31", size = 4669023, upload-time = "2026-05-01T23:29:25.884Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/96/a0/2c913d6fe13d6a8bd13597d36739bf47af063ad9399e402cfecab16f3c1e/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b56b603ebcea8aa10b46228b8410ba7f13e7c2ee54389d4d9be0927fd8ce2a70", size = 5467423, upload-time = "2026-05-01T23:29:33.416Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e7/38/205d10bc1ad0df4a21c5c51659126bd3ea0ef98fcad1e852f78c249bb9c3/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c677c4ad433cb7150c8cd304a0769ae3bcfbe5ea0676eb53faa7b1443b16d0d3", size = 5151137, upload-time = "2026-05-01T23:29:42.013Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/36/fc/f0381ddcd45eff3bb70dbca6823a996048d7f507b2ec3fc92c6fabc0fe87/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26df2717e59c0473e4465a97dfb1b7afebaa479277870fd5784d1436470db47c", size = 6736671, upload-time = "2026-05-01T23:29:51.626Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/95/40/fa545ae152c24327651e5624e4902121e808270be36c10b12e9939be09bc/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dc1f79fd16bb1f3f4421417a514607539f17804d95c7ed617265369d1981cae", size = 4979601, upload-time = "2026-05-01T23:29:56.961Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/86/e4/2f8a47ee97f90cd2b933d0463081d35631ff419de2b8c984a5f369857de0/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:136f199a407b5348b9b857c504aff60c77622a28482e7195839ce1b51238c4cc", size = 4510513, upload-time = "2026-05-01T23:30:07.243Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0e/0e/94e842ff4a7f98ed162580ca2e8b8864b28c1e0350f2443f8ee47f821167/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b6f5a29e9c775b9f12a1a717aa7a2c80f9e1db6f27ba44a5b59c80ac61d2ffcf", size = 4187243, upload-time = "2026-05-01T23:30:15.352Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/83/fc6c174b672e29b7de996ea77b6cbddf46c891751c3355f6974292baa6b4/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:ee17a2cf4943cde261adfad1bbc5bf38d6b3776d7afff74c7cabcbeaeb08c260", size = 3927347, upload-time = "2026-05-01T23:30:21.186Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e9/65/768364d4a97a15b1a7f47ba52688c1686f22941d8332a8398cefc468e25f/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c4ab71be17bdca30cb34c34c4e1496e2f5d6f20c199c12bad226070b22ef9bf", size = 4236393, upload-time = "2026-05-01T23:30:26.211Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bd/3b/218efbc9e645becd80cdf651acda05f85cfe546b7a9c0458c7cbc8fe1f74/psycopg_binary-3.3.4-cp313-cp313-win_amd64.whl", hash = "sha256:dbfdb9b6cc79f31104a7b162a2b921b765fcc62af6c00540a167a8de47e4ed38", size = 3564592, upload-time = "2026-05-01T23:30:31.764Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/48/a6/828c9185701dab71b234c2a76c38a08b098ebfec5020716b4e93807492b5/psycopg_binary-3.3.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:28b7398fdd19db3232c884fb24550bdfe951221f510e195e233299e4c9b78f97", size = 4607292, upload-time = "2026-05-01T23:30:38.962Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/92/58/5b40dbc9d839045c9dae956960e4fb6d20bcabe6c59a2aa34fc3a371913f/psycopg_binary-3.3.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1fbaa292a3c8bb61b45df1ad3da1908ccee7cb889db9425e3557d9e34e2a4829", size = 4687023, upload-time = "2026-05-01T23:30:47.227Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/85/a9/793f0ac107a9003b48441d0d1f9f616d96e0f37458dd8dc12528ceff55fb/psycopg_binary-3.3.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:94596f9e7633ee3f6440711d43bb70aa31cc0a46a900ab8b4201a366ace5c9e7", size = 5486985, upload-time = "2026-05-01T23:30:55.517Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8f/26/42e8533497e2592334f68ec529cf5f840f7fa4e99575a4bb61aa184dbfbf/psycopg_binary-3.3.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8c0056529e68dbe9184cd4019a1f3d8f3a4ead2f6fc7a5afcf27d3314edd1277", size = 5168745, upload-time = "2026-05-01T23:31:01.904Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/15/af/b7151776cc08d5935d45c833ec818a9beb417cf7c08239af1aafbdae78ee/psycopg_binary-3.3.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c09aad7051326e7603c14e50636db9c01f78272dc54b3accff03d46370461e6", size = 6761486, upload-time = "2026-05-01T23:31:14.511Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/ed/c92533b9124712d592cbf1cd6c76da933a2e0acea81dfe1fbe7e735f0cff/psycopg_binary-3.3.4-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:514404ed543efd620c85602b747df2a23cf1241b4067199e1a66f2d2757aaa41", size = 4997427, upload-time = "2026-05-01T23:31:20.901Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a2/23/ccadfd0de416aa188356daa199453af24087b042e296088706d190ae0295/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:46893c26858be12cc49ca4226ed6a60b4bfccadd946b3bebb783a60b38788228", size = 4533549, upload-time = "2026-05-01T23:31:26.204Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fd/a0/c8f43cee36386f7bc891ab41a9d31ea07cf9826038e732da79f26b1e5f34/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:df1d567fc430f6df15c9fcf67d87685fc49bdb325adc0db5af1adfb2f44eb5c9", size = 4210256, upload-time = "2026-05-01T23:31:33.884Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4e/2c/c1547871be3790676e8868b38655496422f94f0978dfb66b74bdba2f1676/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:6b9016b1714da4dd5ecaaa75b82098aa5a0b87854ce9b092e21c27c4ae23e014", size = 3946204, upload-time = "2026-05-01T23:31:39.626Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/b1/f6670f00fa7ea601584623f6c11602ab92117d83eaff885e0210f6de7418/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:47c656a8a7ba6eb0cff1801a4caaa9c8bdc12d03080e273aff1c8ac39971a77e", size = 4255811, upload-time = "2026-05-01T23:31:44.986Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pydantic"
|
||||
version = "2.13.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "annotated-types" },
|
||||
{ name = "pydantic-core" },
|
||||
{ name = "typing-extensions" },
|
||||
{ name = "typing-inspection" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pydantic-core"
|
||||
version = "2.46.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e7/08/f1ba952f1c8ae5581c70fa9c6da89f247b83e3dd8c09c035d5d7931fc23d/pydantic_core-2.46.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4", size = 2113146, upload-time = "2026-05-06T13:37:36.537Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5", size = 1949769, upload-time = "2026-05-06T13:37:46.365Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/64/ba/bfb1d928fd5b49e1258935ff104ae356e9fd89384a55bf9f847e9193ad40/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba", size = 1974958, upload-time = "2026-05-06T13:37:28.611Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4e/74/76223bfb117b64af743c9b6670d1364516f5c0604f96b48f3272f6af6cc6/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b", size = 2042118, upload-time = "2026-05-06T13:36:55.216Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cb/7b/848732968bc8f48f3187542f08358b9d842db564147b256669426ebb1652/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c", size = 2222876, upload-time = "2026-05-06T13:38:25.455Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b5/2f/e90b63ee2e14bd8d3db8f705a6d75d64e6ee1b7c2c8833747ce706e1e0ce/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50", size = 2286703, upload-time = "2026-05-06T13:37:53.304Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd", size = 2092042, upload-time = "2026-05-06T13:38:46.981Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a9/da/0a422b57bf8504102bf3c4ccea9c41bab5a5cee6a54650acf8faf67f5a24/pydantic_core-2.46.4-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01", size = 2117231, upload-time = "2026-05-06T13:39:23.146Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bd/2a/2ac13c3af305843e23c5078c53d135656b3f05a2fd78cb7bbbb12e97b473/pydantic_core-2.46.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d", size = 2168388, upload-time = "2026-05-06T13:40:08.06Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/72/04/2beacf7e1607e93eefe4aed1b4709f079b905fb77530179d4f7c71745f22/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4", size = 2184769, upload-time = "2026-05-06T13:38:13.901Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/29/d2b9fd9f539133548eaf622c06a4ce176cb46ac59f32d0359c4abc0de047/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f", size = 2319312, upload-time = "2026-05-06T13:39:08.24Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/af/0f7a5b85fec6075bea96e3ef9187de38fccced0de92c1e7feda8d5cc7bb9/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39", size = 2361817, upload-time = "2026-05-06T13:38:43.2Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/25/a4/73363fec545fd3ec025490bdda2743c56d0dd5b6266b1a53bbe9e4265375/pydantic_core-2.46.4-cp310-cp310-win32.whl", hash = "sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d", size = 1987085, upload-time = "2026-05-06T13:39:25.497Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/01/aa/62f082da2c91fac1c234bc9ee0066257ce83f0604abd72e4c9d5991f2d84/pydantic_core-2.46.4-cp310-cp310-win_amd64.whl", hash = "sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf", size = 2074311, upload-time = "2026-05-06T13:39:59.922Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pydantic-settings"
|
||||
version = "2.15.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "pydantic" },
|
||||
{ name = "python-dotenv" },
|
||||
{ name = "typing-inspection" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/68/ca/31c57507b13119d7d3cfa1576dad2911a4861e3be07b579395f4e9d393f9/pydantic_settings-2.15.0.tar.gz", hash = "sha256:694b793e84f766ba76a90ebdefc01d0a9a045dab0382bee70393da93712ad117", size = 261253, upload-time = "2026-08-07T09:24:57.419Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/30/a4/2bffa9f8e804325a09867f0e9d30795c80ea9f8d62560bd1b6ad6220eb2f/pydantic_settings-2.15.0-py3-none-any.whl", hash = "sha256:0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42", size = 69413, upload-time = "2026-08-07T09:24:55.839Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyee"
|
||||
version = "13.0.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version >= '3.11.5' and python_full_version < '3.12'",
|
||||
"python_full_version < '3.11.5'",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.12'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyee"
|
||||
version = "14.0.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version >= '3.12'",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "typing-extensions", marker = "python_full_version >= '3.12'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/1d/f1/fdedc2c75c3e31a330659c85e5793bb18b3397981fbf0844c6dee5b18926/pyee-14.0.0.tar.gz", hash = "sha256:76dd0f4314ecd27f02dc73589dea7fd3853f9b6176d8ef9b122860657e3602de", size = 98760, upload-time = "2026-08-13T04:26:11.021Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/81/12/5347938b1f9a6453f0dbdfcc3e2388a1320ef9b9ec17fbefbc4ab647ea98/pyee-14.0.0-py3-none-any.whl", hash = "sha256:3ac2d3229a9677f7de2c33d7f52fe25b638a46b19c413fea2edc8c6d0a644e4d", size = 15553, upload-time = "2026-08-13T04:26:09.916Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pygments"
|
||||
version = "2.21.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pytest"
|
||||
version = "9.1.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "colorama", marker = "sys_platform == 'win32'" },
|
||||
{ name = "exceptiongroup", marker = "python_full_version < '3.11'" },
|
||||
{ name = "iniconfig" },
|
||||
{ name = "packaging" },
|
||||
{ name = "pluggy" },
|
||||
{ name = "pygments" },
|
||||
{ name = "tomli", marker = "python_full_version < '3.11'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "python-dotenv"
|
||||
version = "1.2.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/6a/53/ed9d74092561d4b01a2ef1349d52cdbc135e526c245f366b089cfca6de49/python_dotenv-1.2.3.tar.gz", hash = "sha256:a20a594dabeaa385725aa239d5244871c143ecb356add8a20fcf23773a6c3a35", size = 58945, upload-time = "2026-08-16T16:54:54.067Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/0d/17/c5c6b53ddc18f297992099b3d9ec16c855c0ccc83263a21fe4d1c625ec6c/python_dotenv-1.2.3-py3-none-any.whl", hash = "sha256:904552145e8bfed22162c09dab1c2b9b54fefa7b23ba780f4f26ca0316b0f0d9", size = 22780, upload-time = "2026-08-16T16:54:52.473Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "soupsieve"
|
||||
version = "2.10"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version < '3.11.5'",
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/71/c3/1b817965ac12dc002d7c9cd7dfffdd7d4fbf9b45763ef2ffe7b86ee94670/soupsieve-2.10.tar.gz", hash = "sha256:49e9380d7d2905463583bafe285e818c7366a9ed7b3aee221c1ac79c905d8bc0", size = 124337, upload-time = "2026-09-24T02:36:35.048Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/66/87/5ed59e1d0290564e2027ed066c52b059892c4637741337fc0967183c8d4d/soupsieve-2.10-py3-none-any.whl", hash = "sha256:8596eb8967d744174820280fa62b4542a2e955bfaccca73ed8a13c6eb8e9b502", size = 38019, upload-time = "2026-09-24T02:36:33.709Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "soupsieve"
|
||||
version = "3.0.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version >= '3.12'",
|
||||
"python_full_version >= '3.11.5' and python_full_version < '3.12'",
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/19/63/0b6bae54546aed98d72f12d09df57b19a287f01c84d22047fa2e23772ab9/soupsieve-3.0.4.tar.gz", hash = "sha256:924fab84ae21604c19e175e1f39815eb6fa83ba23836e09e9de180925e5bc4c0", size = 119017, upload-time = "2026-10-22T00:45:26.362Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/07/66/473f7ca0be0b06e38ee8c85ee2db184e3981bd769fac2e7c19c8a8865211/soupsieve-3.0.4-py3-none-any.whl", hash = "sha256:8412826614f9c632227067d0985aeb9f584bd15efc852955430e097a9fe7afbf", size = 42458, upload-time = "2026-10-22T00:45:24.843Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sqlalchemy"
|
||||
version = "2.0.52"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/3b/21/77b4c147963073040dc3c3a5cb7a8c3001a1893c0209432cb77f9df836aa/sqlalchemy-2.0.52.tar.gz", hash = "sha256:5e2d46356ac2ccb7d268ab6c2319ac6a2b42f1b8d5fd8bd3d46855cd82abee97", size = 9945637, upload-time = "2026-08-11T19:07:09.829Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a6/d7/e0354e7334d33ea2795db3ecbe2977026c05a1ecf8ba4b5953c329872453/sqlalchemy-2.0.52-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a7438774e1091192fc50a2bd8ceff5c596912d00ecd46587e88effdea7826101", size = 2172616, upload-time = "2026-08-11T20:58:21.078Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/64/98eef682e6946eb1b4195a9a2393db4662ebfcc89f823ae78b938765c3c0/sqlalchemy-2.0.52-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c1b7ed45bf87b214e0a9def9c2313949067efe6269db5ef18d542ee13250af7", size = 3279798, upload-time = "2026-08-11T21:00:03.535Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/20/05/5b96afc1407c314347ad006b72bb251fb68ef84d05505ebf8a39bc47fcde/sqlalchemy-2.0.52-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:309cc8ba50fc5d2174189dfcd49cdf7aa711f8346afcff19f2642ae4fc449c14", size = 3277555, upload-time = "2026-08-11T21:05:47.932Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/50/69/ce6776724511d1b5dd40477b08d6a5f0953a45375e092dfa852b1857732c/sqlalchemy-2.0.52-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2f9eccf8793c8c3f8dd2dfd11b9e400cb27d1d19370ef732b66017e212107822", size = 3231246, upload-time = "2026-08-11T21:00:05.184Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/72/19/ab0cb9ccdafa2419c796ae62f8740aedb903f1e93bb326064b1a0147e458/sqlalchemy-2.0.52-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9255ceb65a80c1b001129060b63ee776a2e9c288be3b662be36dfbb888fffdcd", size = 3250763, upload-time = "2026-08-11T21:05:49.664Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/41/f2/3c9b54b61bec4f493c0007dc9e2700c963c5b32667e21a506f1aca7a8115/sqlalchemy-2.0.52-cp310-cp310-win32.whl", hash = "sha256:2e15b1d1116a64fc399b8c2694a83f3e792fdc58df28514a81e1dc4f8cf22729", size = 2132169, upload-time = "2026-08-11T21:09:48.108Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b3/a1/934bb6cf543a398c72784d1fc777eb530559c16ebe1549fa7611e5989ce9/sqlalchemy-2.0.52-cp310-cp310-win_amd64.whl", hash = "sha256:11560064cc4696e772298b6221ede59e646386d9f2a85d549365473b972f7850", size = 2156289, upload-time = "2026-08-11T21:09:49.429Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6b/08/cc5f7627b92f1456bc0b5fb7e98af4600248abe422a44da0d17a3fe6a448/sqlalchemy-2.0.52-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0c3ce43907374889f3352bdcc6195c970148a2cb71574cd0237a5071a37fb6c", size = 2172460, upload-time = "2026-08-11T20:58:22.429Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ed/dc/9a2abad8bfc8fdcd38c64adc056aeefab7aaa96ecd32f5e8c140e6375f17/sqlalchemy-2.0.52-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a0d48c4b80717c61385b4e966e087c839a66cfd7b780641dcb428f4dba65608", size = 3355720, upload-time = "2026-08-11T21:00:06.746Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a8/73/e75597b5841043e3c74055d00d4feb53d9a49a5c89ba2450d2d9aab53597/sqlalchemy-2.0.52-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:938325a5373267afc53bfbe72983b20fbd64ca47842aac62433c3da1137ecff1", size = 3354394, upload-time = "2026-08-11T21:05:51.454Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/12/25/410fbc6c2f1fa8310f4ef1b6847d47d0ac1c042c7b4e81eaaca063d030a9/sqlalchemy-2.0.52-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5f8438a98d49424acf69d0d53c0a522951dfe49a6f2d86417fbb37ad3066ab43", size = 3306991, upload-time = "2026-08-11T21:00:08.603Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b2/ba/25ffd5c24681ea4b46e62c80ceca8200ce204de1773366321306cf3f608a/sqlalchemy-2.0.52-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4699dbb8d396d199e7e78fd4d525e3ad3d6008a9c8c0160b87e74c606c2c3736", size = 3327454, upload-time = "2026-08-11T21:05:53.368Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c2/f1/0f1b1d4800e51218e736a06ed55a3b2a59c257600bbaca7673bf13d2dbec/sqlalchemy-2.0.52-cp311-cp311-win32.whl", hash = "sha256:cef328349452ae152637df4d11ce5a0919ecdf0a363e16c830c3518ee33bde72", size = 2131248, upload-time = "2026-08-11T21:09:50.765Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7a/f0/04d2ac5ad66f3d31278f37064ed5f5ef3fe653f7bdaa67036663f223d186/sqlalchemy-2.0.52-cp311-cp311-win_amd64.whl", hash = "sha256:f1c850792a3b25a3ad74dade3f05e4f402cdebfea27438bcadafaa1617f77bcc", size = 2156943, upload-time = "2026-08-11T21:09:51.979Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e0/d5/1b77a026d161f98a08f11af1a5f6c47b98ee7c7e2648af525a1004826c78/sqlalchemy-2.0.52-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:be8c49131665dfe2cc74c498aa1240ffb548d0fd901325dd11c2c7a18956f727", size = 2170940, upload-time = "2026-08-11T20:58:11.25Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/54/bd/f444444adb37b5d53753fb1730ee7a421628e2e3b756c4da461af7e6394a/sqlalchemy-2.0.52-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b2d9e507a458832adcfbd8af6e2036ddf069b7710b799448542ebccae2dceee", size = 3383415, upload-time = "2026-08-11T21:02:38.534Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/be/57/2eadf93a552568c57e8680b7e58bb5e9770d80942a1bdbaf4f2f63f0d7c8/sqlalchemy-2.0.52-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8738008376d22f30f411ea3efecf39b51110b6996d80bb73786f30bcfdd5fd3b", size = 3398577, upload-time = "2026-08-11T21:16:59.092Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/15/c3/2887cf9dd111d1fbf05d22165b404c221ef43e029f7a2695e7302f27a7cc/sqlalchemy-2.0.52-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37a4d548327b6cab9c7d8cdb4e0e82feabee0110c4d150059068e2d1cfbd99ee", size = 3328225, upload-time = "2026-08-11T21:02:40.183Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/02/0f/466bdf9e1feeeef5587f868c187d8687e21ff8c85b1775e9041130181132/sqlalchemy-2.0.52-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e49f51a5d59857a7a0dcaf9469febf7197d9394bd88f00d69c2c4e848112cdbf", size = 3357374, upload-time = "2026-08-11T21:17:01.076Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/20/5c2b4583904af4173076dda1c9e53c9e2ffc7a702d2efde0216bbacbf7cb/sqlalchemy-2.0.52-cp312-cp312-win32.whl", hash = "sha256:afda3ec521d0517d0de783fc70030775841900896d832de5bbd066549290470e", size = 2129366, upload-time = "2026-08-11T21:14:50.991Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ed/06/543dab8ef62d4e9fb96fb31a30c2b8b14a8763bccf48d428294d6b3041c0/sqlalchemy-2.0.52-cp312-cp312-win_amd64.whl", hash = "sha256:2d5e53e36e37129fe0be8b9d08b6e4052c10a963ee6cda56c8c10dcc194b99ca", size = 2157344, upload-time = "2026-08-11T21:14:52.453Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7f/18/e30c6fe1eca1bf34a39fbdd6066121cc9974c850faf6f349eac563697a26/sqlalchemy-2.0.52-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2eb3c6a64b1bfe6704777cfd504e7b8ad093a5f3e03ce67663a5e6742f294e43", size = 2167724, upload-time = "2026-08-11T20:58:12.679Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/56/2e17d161a4f7ecc1c2ffb93e607b4e1898bb551b451b283235acb8f6ce47/sqlalchemy-2.0.52-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:923bb183c1dc64fdf7b717965e3d59938ec4f8b8710b419a21ce403e5da9a9e1", size = 3321189, upload-time = "2026-08-11T21:02:41.932Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cf/b8/8490916e893f3f8d74dc9cc54c078619364999dee37047a188e73abbc852/sqlalchemy-2.0.52-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:651d6d8782e80679e6151707c7b490834d46ada526328895abf567f25e63d29c", size = 3338185, upload-time = "2026-08-11T21:17:02.597Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/f7/752cc8ee453da222829b3f5c4613614bf750d97429363b70414fa10478e4/sqlalchemy-2.0.52-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b08cddb8989775e3c88799d86704bdfc3ee6e9846118201aa5997f16f27e3a15", size = 3271698, upload-time = "2026-08-11T21:02:43.963Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/51/e6/074ade0c07b9e4c8e8bca46820320ed94df9702afdb6f2af06623068d2e6/sqlalchemy-2.0.52-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ab66fa9618269390d4dfa222f2f2f88f7bc4bf5da13905131b818217db7e8057", size = 3308936, upload-time = "2026-08-11T21:17:04.172Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/66/07/557c0d04716705599227945ac14e0a17ad0338e899f37d8c2ddff4dcc663/sqlalchemy-2.0.52-cp313-cp313-win32.whl", hash = "sha256:c63bda077685c85ca513286547a531ba57e7a68cf0a7ed3bafcc2bbd18896f4d", size = 2127308, upload-time = "2026-08-11T21:14:53.879Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/96/4e/226eda27654318ce525d043025221f689abef883da2c7126f9065121618c/sqlalchemy-2.0.52-cp313-cp313-win_amd64.whl", hash = "sha256:9876b09b9f1ce7398b0ffece585c0a911244c53191187341f6bcae640e133751", size = 2153876, upload-time = "2026-08-11T21:14:55.527Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d5/f5/71cb30af58c9b80a4e1fac0b73bb48f86d497a774a6a2eb6d2f1e657bb73/sqlalchemy-2.0.52-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:410d52be41d17f1a236d19520fbe776257dc16516ed06bd16d433311842aefd9", size = 2169537, upload-time = "2026-08-11T20:58:13.855Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4c/93/d07ebd645d1b07b6b5ed63450a70f063a346a7e0f2c8810daf2e532400cb/sqlalchemy-2.0.52-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfe9ce533dbe4d0a2ae1486546619bd30b76bcd670539a44d910361376175f5e", size = 3319606, upload-time = "2026-08-11T21:02:45.829Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ae/5c/290c84c7c2566ecd3b65baaae0fddec9bc33b033b398a06123bb86fbfc6e/sqlalchemy-2.0.52-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:812bae5138bfc0aa46fb0686da0fc7f581f68e2bbb05bc24c3713bebaedd1437", size = 3323642, upload-time = "2026-08-11T21:17:05.675Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/13/f5/2cc160590ca49173359557880b92a0572293ccb899e8f6cedf150c5a3ddf/sqlalchemy-2.0.52-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:50bff43b632a56fbf5ed9afdd76307e1512b62051bcd5afb341ae67205bbb6c8", size = 3268125, upload-time = "2026-08-11T21:02:47.649Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/f3/ea8933fc9f7d1353e9c2ff9965eae687c4cef181120574591ed2fa0633e1/sqlalchemy-2.0.52-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:49565daf5af554f538e23aef1fc81a95a4e49658f152285e45c02f5fc44f04cd", size = 3289516, upload-time = "2026-08-11T21:17:07.267Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/45/67/05cf86541c1e1716fca1e4a996954a439cd74501707cda607fb7cb02ef50/sqlalchemy-2.0.52-cp314-cp314-win32.whl", hash = "sha256:ab9da41e61b9979b910499d633b241df20c51ee5037e5405b11c2faac3cbe1a2", size = 2130249, upload-time = "2026-08-11T21:14:57.273Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/96/d7/8ac6ffa1e36169e762ef65bd835046abb2251b1bc17f8f6708e14ed8d31f/sqlalchemy-2.0.52-cp314-cp314-win_amd64.whl", hash = "sha256:a593db51b3bae75db17a5738ad5f992244b3a03863f83c28117ee482c6a3f76d", size = 2156718, upload-time = "2026-08-11T21:14:58.667Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/dc/4b/e01a737eef378e734cc6394a82248a6ce13b167dfa36c731075ce9fc9c64/sqlalchemy-2.0.52-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1e61d08bdf4ee2f41024569e3400de7d6734ba498144766b11260936ccfa582", size = 2190344, upload-time = "2026-08-11T19:53:21.393Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b3/3f/3582293d1e185e71d19d7c731c3e2ee20ba21981c4a1115c0806c1f62120/sqlalchemy-2.0.52-py3-none-any.whl", hash = "sha256:3b81b8363a919ce53453591cdb93702e6bd54ade6c4fa2f468fc053baee5ed89", size = 1950700, upload-time = "2026-08-11T20:47:21.603Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tomli"
|
||||
version = "2.4.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typing-extensions"
|
||||
version = "4.16.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typing-inspection"
|
||||
version = "0.4.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tzdata"
|
||||
version = "2026.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" },
|
||||
]
|
||||
Loading…
Add table
Reference in a new issue