spouliot
c59d55529f
Standalone .NET 8 console app (not part of the main solution) that scrapes the Prismatic Powders catalog via Playwright and pushes it into the app's catalog import. Prismatic has no API, so this runs on a workstation (Task Scheduler), never the deployed server. - Discovery: incremental newest-first via ?category=created_at (stops once it reaches already-known URLs — cheap, finds new colors) and a full all-colors crawl for occasional reconcile. - Scraper: resumable product-page scrape (sku/color/description/price tiers/ SDS/TDS/app-guide/image), with --refresh-older-than to re-scrape stale products and catch price changes. Output matches the app import format so it flows through the same shared upsert as the Columbia sync. - Resilience: brisk randomized base delay, escalating 403 cooldown-and-retry to avoid hard bans, periodic rest. All configurable. - Visibility: streams every product + the inter-product wait to the console (colored) and a log file, with an up-front ETA. - Push: token-authenticated POST to the app import endpoint (skips to manual upload when unconfigured). The app-side token import endpoint is a separate follow-up. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
64 lines
2.3 KiB
C#
64 lines
2.3 KiB
C#
using System.Text;
|
|
using PrismaticSync.Infrastructure;
|
|
|
|
namespace PrismaticSync.Services;
|
|
|
|
/// <summary>
|
|
/// Pushes the scraped JSON to the app's token-authenticated catalog import endpoint. When no
|
|
/// endpoint is configured it no-ops (the JSON is still on disk for a manual upload), so the tool is
|
|
/// useful before the endpoint exists.
|
|
/// </summary>
|
|
public class CatalogPusher
|
|
{
|
|
private readonly SyncConfig _config;
|
|
|
|
public CatalogPusher(SyncConfig config) => _config = config;
|
|
|
|
public async Task<bool> PushAsync()
|
|
{
|
|
if (string.IsNullOrWhiteSpace(_config.Import.EndpointUrl))
|
|
{
|
|
Log.Warn($"No import endpoint configured (Sync.Import.EndpointUrl) — skipping push. " +
|
|
$"Upload {_config.OutputJsonFile} manually via the Powder Catalog admin instead.");
|
|
return false;
|
|
}
|
|
|
|
if (!File.Exists(_config.OutputJsonFile))
|
|
{
|
|
Log.Warn($"Output file {_config.OutputJsonFile} not found — nothing to push.");
|
|
return false;
|
|
}
|
|
|
|
var json = await File.ReadAllTextAsync(_config.OutputJsonFile);
|
|
Log.Info($"Pushing {_config.OutputJsonFile} to {_config.Import.EndpointUrl} (vendor: {_config.Import.VendorName})...");
|
|
|
|
using var http = new HttpClient { Timeout = TimeSpan.FromMinutes(5) };
|
|
using var request = new HttpRequestMessage(HttpMethod.Post, _config.Import.EndpointUrl);
|
|
request.Headers.Add("X-Import-Token", _config.Import.Token);
|
|
request.Headers.Add("X-Vendor-Name", _config.Import.VendorName);
|
|
request.Content = new StringContent(json, Encoding.UTF8, "application/json");
|
|
|
|
try
|
|
{
|
|
using var response = await http.SendAsync(request);
|
|
var body = await response.Content.ReadAsStringAsync();
|
|
|
|
if (response.IsSuccessStatusCode)
|
|
{
|
|
Log.Info($"Push succeeded ({(int)response.StatusCode}): {Trim(body)}");
|
|
return true;
|
|
}
|
|
|
|
Log.Error($"Push failed ({(int)response.StatusCode}): {Trim(body)}");
|
|
return false;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Error($"Push error: {ex.Message}");
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private static string Trim(string s) => s.Length > 500 ? s[..500] + "…" : s;
|
|
}
|