60 lines
2.2 KiB
Python
60 lines
2.2 KiB
Python
"""
|
|
Demo Script: Live BMW Stock Crawling and Database Ingestion Prototype
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from sqlalchemy import select, func
|
|
|
|
from oem_crawler.crawlers.bmw.crawler import BMWStockCrawler
|
|
from oem_crawler.db.models import Vehicle, VehicleOption, VehiclePriceHistory
|
|
from oem_crawler.db.pipeline import StockIngestionPipeline
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
|
|
|
|
|
def main():
|
|
print("=" * 70)
|
|
print("🚀 Initializing OEM Stock Crawler Prototype")
|
|
print("=" * 70)
|
|
|
|
pipeline = StockIngestionPipeline()
|
|
crawler = BMWStockCrawler(market_country="DE")
|
|
|
|
limit = 6
|
|
print(f"\n📡 Crawling first {limit} live vehicles from BMW Neuwagensuche REST API...")
|
|
stream = crawler.crawl(max_vehicles=limit, page_size=limit)
|
|
|
|
crawl_run = pipeline.run_crawler_pipeline(
|
|
brand="BMW",
|
|
crawler_name="bmw-rest-crawler-v1",
|
|
vehicle_stream=stream,
|
|
market_country="DE",
|
|
config_snapshot={"max_vehicles": limit, "is_full_market_scan": False},
|
|
)
|
|
|
|
print("\n" + "=" * 70)
|
|
print(f"✅ Crawl Run Complete: {crawl_run.id}")
|
|
print(f" - Status: {crawl_run.status}")
|
|
print(f" - Duration: {crawl_run.duration_ms} ms")
|
|
print(f" - Discovered: {crawl_run.items_discovered}")
|
|
print(f" - Inserted: {crawl_run.items_inserted}")
|
|
print(f" - Updated: {crawl_run.items_updated}")
|
|
print(f" - Unmodified: {crawl_run.items_unmodified}")
|
|
print("=" * 70)
|
|
|
|
with pipeline.session_factory() as session:
|
|
vehicles = session.scalars(select(Vehicle).limit(3)).all()
|
|
print("\n🔍 Sample Stored Vehicles:")
|
|
for v in vehicles:
|
|
options_count = session.scalar(
|
|
select(func.count(VehicleOption.id)).where(VehicleOption.vehicle_id == v.id)
|
|
)
|
|
print(f" • [{v.brand}] {v.series} {v.model_name}")
|
|
print(f" Gross Price: {v.price_gross:,.2f} {v.currency} | Options: {options_count} items stored")
|
|
print(f" Dealer: {v.dealer_name or 'N/A'} ({v.location_city or 'N/A'})")
|
|
print("-" * 60)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|