spouliot
c59d55529f
Standalone .NET 8 console app (not part of the main solution) that scrapes the Prismatic Powders catalog via Playwright and pushes it into the app's catalog import. Prismatic has no API, so this runs on a workstation (Task Scheduler), never the deployed server. - Discovery: incremental newest-first via ?category=created_at (stops once it reaches already-known URLs — cheap, finds new colors) and a full all-colors crawl for occasional reconcile. - Scraper: resumable product-page scrape (sku/color/description/price tiers/ SDS/TDS/app-guide/image), with --refresh-older-than to re-scrape stale products and catch price changes. Output matches the app import format so it flows through the same shared upsert as the Columbia sync. - Resilience: brisk randomized base delay, escalating 403 cooldown-and-retry to avoid hard bans, periodic rest. All configurable. - Visibility: streams every product + the inter-product wait to the console (colored) and a log file, with an up-front ETA. - Push: token-authenticated POST to the app import endpoint (skips to manual upload when unconfigured). The app-side token import endpoint is a separate follow-up. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
70 lines
3.1 KiB
C#
70 lines
3.1 KiB
C#
namespace PrismaticSync.Infrastructure;
|
||
|
||
/// <summary>Strongly-typed config bound from the "Sync" section of appsettings.json.</summary>
|
||
public class SyncConfig
|
||
{
|
||
public string BaseUrl { get; set; } = "https://www.prismaticpowders.com";
|
||
public string ColorsPath { get; set; } = "/shop/powder-coating-colors";
|
||
|
||
public string ProductUrlsFile { get; set; } = "product-urls.txt";
|
||
public string OutputJsonFile { get; set; } = "prismatic_powders.json";
|
||
public string LogFile { get; set; } = "prismatic-sync.log";
|
||
|
||
/// <summary>Politeness delay between product scrapes (randomized within the range).</summary>
|
||
public int MinDelaySeconds { get; set; } = 6;
|
||
public int MaxDelaySeconds { get; set; } = 14;
|
||
|
||
/// <summary>On a 403/block, cool down this many seconds × the consecutive-block count, then retry.</summary>
|
||
public int BlockedCooldownSeconds { get; set; } = 120;
|
||
|
||
/// <summary>Upper bound on a single cooldown so escalation can't run away.</summary>
|
||
public int BlockedCooldownMaxSeconds { get; set; } = 600;
|
||
|
||
/// <summary>How many times to cool-down-and-retry a blocked product before recording it as an error.</summary>
|
||
public int BlockedMaxRetries { get; set; } = 3;
|
||
|
||
/// <summary>Take a longer rest after this many products (0 disables). Eases load and looks less robotic.</summary>
|
||
public int LongRestEveryProducts { get; set; } = 150;
|
||
|
||
/// <summary>Length of the periodic long rest, in seconds.</summary>
|
||
public int LongRestSeconds { get; set; } = 45;
|
||
|
||
/// <summary>Extra settle time after a product page loads before reading it.</summary>
|
||
public int PageSettleSeconds { get; set; } = 4;
|
||
|
||
/// <summary>Pause after each scroll while a listing lazy-loads more items.</summary>
|
||
public int ScrollWaitMs { get; set; } = 1500;
|
||
|
||
/// <summary>Hard cap on scrolls per listing, as a safety stop.</summary>
|
||
public int MaxScrolls { get; set; } = 400;
|
||
|
||
/// <summary>Full discovery: stop a listing after this many scrolls add no new links.</summary>
|
||
public int StopAfterNoNewScrolls { get; set; } = 10;
|
||
|
||
/// <summary>
|
||
/// Incremental discovery: stop the newest-first listing after this many consecutive scrolls
|
||
/// that surfaced only already-known URLs — i.e. we've scrolled past the new products.
|
||
/// </summary>
|
||
public int StopAfterKnownScrolls { get; set; } = 8;
|
||
|
||
/// <summary>Color filter params used by full discovery.</summary>
|
||
public string[] ColorParams { get; set; } = Array.Empty<string>();
|
||
|
||
public ImportConfig Import { get; set; } = new();
|
||
|
||
public string ColorsUrl => $"{BaseUrl.TrimEnd('/')}{ColorsPath}";
|
||
}
|
||
|
||
/// <summary>Where and how to push the scraped catalog into the app.</summary>
|
||
public class ImportConfig
|
||
{
|
||
/// <summary>Full URL of the app's token-authenticated catalog import endpoint.</summary>
|
||
public string EndpointUrl { get; set; } = "";
|
||
|
||
/// <summary>Shared secret sent in the X-Import-Token header. Must match the app's config.</summary>
|
||
public string Token { get; set; } = "";
|
||
|
||
/// <summary>Vendor name applied to every record on import.</summary>
|
||
public string VendorName { get; set; } = "Prismatic Powders";
|
||
}
|