Add Columbia catalog mapper, shared upsert, and sync service
Phase 2: the mapping and sync core. - ColumbiaCatalogMapper (pure/static, unit-tested): maps an API product to a PowderCatalogItem. Derives manufacturer (PPG/KP Pigments/Columbia) from taxonomy+SKU; flags additives into the Powder Additives category; takes base price from the top-level price with variant fallback; captures variation / tiered pricing as JSON; parses the free-text cure schedule into all curves (three degree glyphs, @/at, multi-curve in order, partial-cure -> none) with the first as the primary temp/time; strips HTML descriptions; joins color groups; normalizes chemistry; flags clear-coat powders. - PowderCatalogUpsertService (IPowderCatalogUpsertService): single upsert path matching on (VendorName, SKU). Copies only feed-sourced fields and leaves enrichment fields (specific gravity, coverage, transfer efficiency, finish) untouched so syncs never wipe lazily-enriched TDS/AI data. - ColumbiaCatalogSyncService (IColumbiaCatalogSyncService): pulls the full catalog, maps + de-dupes, upserts, then reconciles discontinuations ONLY on a complete pull (a partial pull throws and aborts before the sweep). Reactivates reappearing items; records last-synced/last-result platform settings. - 25 mapper unit tests covering the cure parser, manufacturer derivation, simple/variable pricing, chemistry, color, and HTML cases from real records. Full suite green (261 passed). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -54,10 +54,11 @@ public class ColumbiaProduct
|
||||
/// <summary>
|
||||
/// Quantity-break pricing. POLYMORPHIC on the wire: an object
|
||||
/// (<c>{type, minimum_quantity, tiers:[...]}</c>) on simple products, but an empty ARRAY
|
||||
/// (<c>[]</c>) on variable products. Captured as a raw <see cref="JsonElement"/> so
|
||||
/// deserialization never throws on the type mismatch; the mapper interprets it.
|
||||
/// (<c>[]</c>) on variable products. Captured as a nullable raw <see cref="JsonElement"/> so
|
||||
/// deserialization never throws on the type mismatch and an absent value is null (not an
|
||||
/// invalid <c>Undefined</c> element); the mapper interprets it.
|
||||
/// </summary>
|
||||
public JsonElement TieredPricing { get; set; }
|
||||
public JsonElement? TieredPricing { get; set; }
|
||||
|
||||
/// <summary>Per-variant pricing for variable products (e.g. Bulk vs 1 lb Bags, or gram sizes).
|
||||
/// Each variant has its own SKU and price. Empty for simple products.</summary>
|
||||
@@ -108,8 +109,8 @@ public class ColumbiaVariationPricing
|
||||
public string RegularPrice { get; set; } = string.Empty;
|
||||
public string SalePrice { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Same polymorphic object-or-array shape as the parent; captured raw.</summary>
|
||||
public JsonElement TieredPricing { get; set; }
|
||||
/// <summary>Same polymorphic object-or-array shape as the parent; captured raw (nullable).</summary>
|
||||
public JsonElement? TieredPricing { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>An image object — featured or gallery.</summary>
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
namespace PowderCoating.Application.Interfaces;
|
||||
|
||||
/// <summary>
|
||||
/// Orchestrates a full Columbia Coatings catalog sync: pull every product, map and upsert it, then
|
||||
/// (only on a complete pull) reconcile discontinuations. Used by both the scheduled background job
|
||||
/// and the manual "Sync now" admin action.
|
||||
/// </summary>
|
||||
public interface IColumbiaCatalogSyncService
|
||||
{
|
||||
/// <summary>
|
||||
/// Runs one full sync. Assumes the caller has already decided it should run (enabled / due).
|
||||
/// Returns a result describing the outcome; never throws for an expected failure (not
|
||||
/// configured, partial pull, HTTP error) — those are reported on the result instead.
|
||||
/// </summary>
|
||||
Task<ColumbiaSyncResult> RunSyncAsync(CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>Outcome of a Columbia catalog sync run.</summary>
|
||||
public class ColumbiaSyncResult
|
||||
{
|
||||
public bool Success { get; set; }
|
||||
public string? ErrorMessage { get; set; }
|
||||
|
||||
public int TotalFetched { get; set; }
|
||||
public int Inserted { get; set; }
|
||||
public int Updated { get; set; }
|
||||
public int Unchanged { get; set; }
|
||||
public int Skipped { get; set; }
|
||||
public int Discontinued { get; set; }
|
||||
public int Reactivated { get; set; }
|
||||
|
||||
public DateTime StartedAt { get; set; }
|
||||
public TimeSpan Duration { get; set; }
|
||||
|
||||
/// <summary>One-line summary suitable for storing in the last-result platform setting / UI.</summary>
|
||||
public string Summary =>
|
||||
Success
|
||||
? $"{TotalFetched} fetched: {Inserted} new, {Updated} updated, {Unchanged} unchanged, " +
|
||||
$"{Discontinued} discontinued, {Reactivated} reactivated ({Duration.TotalSeconds:F0}s)"
|
||||
: $"Failed: {ErrorMessage}";
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
using PowderCoating.Core.Entities;
|
||||
|
||||
namespace PowderCoating.Application.Interfaces;
|
||||
|
||||
/// <summary>
|
||||
/// Shared upsert for the platform powder catalog: matches incoming records to existing rows by
|
||||
/// (VendorName, SKU), inserts new ones, and updates changed ones in place. Used by BOTH the manual
|
||||
/// JSON file import and the Columbia API sync so there is a single upsert path, only the mapping
|
||||
/// differs. Does NOT handle discontinuation — that is a sync-specific concern.
|
||||
/// </summary>
|
||||
public interface IPowderCatalogUpsertService
|
||||
{
|
||||
/// <summary>
|
||||
/// Applies <paramref name="incoming"/> mapped catalog items. Only fields sourced from the feed
|
||||
/// are copied on update; enrichment fields (specific gravity, coverage, transfer efficiency,
|
||||
/// finish) are preserved so they are not wiped by a feed that never carries them. Changed and
|
||||
/// inserted rows get <paramref name="runTimestamp"/> stamped on LastSyncedAt/UpdatedAt.
|
||||
/// </summary>
|
||||
Task<PowderCatalogUpsertResult> UpsertAsync(
|
||||
IReadOnlyList<PowderCatalogItem> incoming,
|
||||
DateTime runTimestamp,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>Counts from an upsert run.</summary>
|
||||
public class PowderCatalogUpsertResult
|
||||
{
|
||||
public int Inserted { get; set; }
|
||||
public int Updated { get; set; }
|
||||
public int Unchanged { get; set; }
|
||||
public int Skipped { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
using System.Globalization;
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using PowderCoating.Application.Constants;
|
||||
using PowderCoating.Application.DTOs.Columbia;
|
||||
using PowderCoating.Core.Entities;
|
||||
|
||||
namespace PowderCoating.Infrastructure.Services.Columbia;
|
||||
|
||||
/// <summary>
|
||||
/// Maps a Columbia Coatings API product onto our platform <see cref="PowderCatalogItem"/>.
|
||||
/// Pure, static, side-effect free so the tricky bits (manufacturer derivation, the free-text
|
||||
/// cure-schedule parser, HTML stripping) can be unit tested directly against captured fixtures.
|
||||
/// <para>
|
||||
/// Columbia is a distributor reselling multiple brands, so <see cref="PowderCatalogItem.VendorName"/>
|
||||
/// holds the DERIVED manufacturer (PPG / KP Pigments / Columbia) while
|
||||
/// <see cref="PowderCatalogItem.Source"/> records the feed ("Columbia Coatings API") for
|
||||
/// right-to-delete purges. The vendor's own categories/tags are read here only to derive the
|
||||
/// manufacturer and additive flag — they are never stored raw.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static class ColumbiaCatalogMapper
|
||||
{
|
||||
/// <summary>A single parsed cure curve — hold <see cref="Minutes"/> at <see cref="TempF"/>.</summary>
|
||||
public readonly record struct CureCurve(int TempF, int Minutes);
|
||||
|
||||
// Resin chemistries that mean "polyester + TGIC" but arrive formatted three different ways.
|
||||
private static readonly Regex PolyesterTgic =
|
||||
new(@"^\s*(polyester\s*[/ ]\s*tgic|tgic\s*[/ ]\s*polyester)\s*$", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
|
||||
// "10 minutes @ 400°F", "7 minutes at 375 F", "Metal Temperature: 10 minutes at 400°F (204°C)".
|
||||
// Degree glyph is optional and may be ° (U+00B0), ˚ (U+02DA), or º (U+00BA).
|
||||
private static readonly Regex CureCurveRegex =
|
||||
new(@"(\d+)\s*min(?:ute)?s?\.?\s*(?:@|at)\s*(\d{2,3})\s*[°˚º]?\s*F",
|
||||
RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
|
||||
private static readonly Regex HtmlTag = new("<[^>]+>", RegexOptions.Compiled);
|
||||
private static readonly Regex WhitespaceRun = new(@"\s{2,}", RegexOptions.Compiled);
|
||||
|
||||
private static readonly JsonSerializerOptions JsonOut = new() { WriteIndented = false };
|
||||
|
||||
/// <summary>Maps a Columbia product into a fully populated (unsaved) catalog item.</summary>
|
||||
public static PowderCatalogItem Map(ColumbiaProduct p)
|
||||
{
|
||||
var curves = ParseCureCurves(p.CureSchedule);
|
||||
var primary = curves.Count > 0 ? curves[0] : (CureCurve?)null;
|
||||
|
||||
return new PowderCatalogItem
|
||||
{
|
||||
VendorName = DeriveManufacturer(p),
|
||||
Sku = p.Sku.Trim(),
|
||||
ColorName = p.Name.Trim(),
|
||||
Source = ColumbiaIntegrationConstants.SourceName,
|
||||
Category = IsAdditive(p) ? ColumbiaIntegrationConstants.CategoryPowderAdditives : null,
|
||||
|
||||
Description = StripHtml(p.Description),
|
||||
UnitPrice = ParseBasePrice(p),
|
||||
PriceTiersJson = BuildPriceTiersJson(p),
|
||||
|
||||
ImageUrl = NullIfBlank(p.FeaturedImage?.Src),
|
||||
SdsUrl = NullIfBlank(p.SafetyDataSheet),
|
||||
TdsUrl = NullIfBlank(p.TechnicalDataSheet),
|
||||
ApplicationGuideUrl = NullIfBlank(FirstNonBlank(p.ProductFlyer, p.ProductBrochure)),
|
||||
ProductUrl = NullIfBlank(p.Permalink),
|
||||
|
||||
ChemistryType = NormalizeChemistry(p.Type),
|
||||
MilThickness = NullIfBlank(p.MilThickness),
|
||||
CureScheduleText = NullIfBlank(p.CureSchedule),
|
||||
CureCurvesJson = curves.Count > 0 ? JsonSerializer.Serialize(curves, JsonOut) : null,
|
||||
CureTemperatureF = primary?.TempF,
|
||||
CureTimeMinutes = primary?.Minutes,
|
||||
RequiresClearCoat = DetectRequiresClearCoat(p),
|
||||
|
||||
ColorFamilies = BuildColorFamilies(p),
|
||||
FormulationChanges = NullIfBlank(p.FormulationDateChanges),
|
||||
|
||||
// Coverage / specific gravity / transfer efficiency are not in the API — left null for
|
||||
// lazy TDS/AI enrichment on first use. IsDiscontinued is handled by the sync sweep.
|
||||
};
|
||||
}
|
||||
|
||||
// ── Manufacturer derivation ───────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Derives the manufacturer from the product's taxonomy/SKU. Columbia resells PPG powders and
|
||||
/// KP Pigments additives through the same feed; everything else is Columbia's own brand.
|
||||
/// </summary>
|
||||
public static string DeriveManufacturer(ColumbiaProduct p)
|
||||
{
|
||||
if (IsKpPigments(p))
|
||||
return ColumbiaIntegrationConstants.ManufacturerKp;
|
||||
if (IsPpg(p))
|
||||
return ColumbiaIntegrationConstants.ManufacturerPpg;
|
||||
return ColumbiaIntegrationConstants.ManufacturerColumbia;
|
||||
}
|
||||
|
||||
private static bool IsKpPigments(ColumbiaProduct p) =>
|
||||
p.Sku.StartsWith("ADD-", StringComparison.OrdinalIgnoreCase)
|
||||
|| CategoryStartsWith(p, "KP");
|
||||
|
||||
private static bool IsPpg(ColumbiaProduct p) =>
|
||||
CategoryStartsWith(p, "PPG");
|
||||
|
||||
/// <summary>
|
||||
/// True for pigments/additives sold by weight (grams) rather than sprayed powders. These get
|
||||
/// forced into the "Powder Additives" category. Keyed off the broad Additives category and the
|
||||
/// ADD- SKU prefix, not just the KP brand (there are ~98 non-KP additives).
|
||||
/// </summary>
|
||||
public static bool IsAdditive(ColumbiaProduct p) =>
|
||||
p.Sku.StartsWith("ADD-", StringComparison.OrdinalIgnoreCase)
|
||||
|| p.Categories.Any(c => c.Name.Equals("Additives", StringComparison.OrdinalIgnoreCase))
|
||||
|| CategoryStartsWith(p, "KP");
|
||||
|
||||
private static bool CategoryStartsWith(ColumbiaProduct p, string prefix) =>
|
||||
p.Categories.Any(c => c.Name.StartsWith(prefix, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
// ── Pricing ───────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Base unit price = the top-level <c>price</c> (falling back to <c>regular_price</c>). For
|
||||
/// variable products the parent <c>price</c> already carries the lead variant's price, while
|
||||
/// <c>regular_price</c> is often "0", so price is preferred.
|
||||
/// </summary>
|
||||
public static decimal ParseBasePrice(ColumbiaProduct p)
|
||||
{
|
||||
if (TryParseMoney(p.Price, out var price) && price > 0)
|
||||
return price;
|
||||
if (TryParseMoney(p.RegularPrice, out var regular) && regular > 0)
|
||||
return regular;
|
||||
|
||||
// Variable product with a zero parent price: fall back to the lowest variant price.
|
||||
var variantPrices = (p.VariationPricing ?? new List<ColumbiaVariationPricing>())
|
||||
.Select(v => TryParseMoney(v.Price, out var vp) ? vp : 0m)
|
||||
.Where(v => v > 0)
|
||||
.ToList();
|
||||
return variantPrices.Count > 0 ? variantPrices.Min() : 0m;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Captures quantity-break / variant pricing as JSON for later use. For variable products this
|
||||
/// is the per-variant pricing (Bulk vs 1 lb Bags, gram sizes); for simple products it's the
|
||||
/// tiered_pricing object. Null when neither is present.
|
||||
/// </summary>
|
||||
public static string? BuildPriceTiersJson(ColumbiaProduct p)
|
||||
{
|
||||
if (p.VariationPricing is { Count: > 0 })
|
||||
return JsonSerializer.Serialize(p.VariationPricing, JsonOut);
|
||||
|
||||
if (p.TieredPricing is { ValueKind: JsonValueKind.Object } tiered)
|
||||
{
|
||||
// Only keep it if it actually carries tiers (avoid storing empty {type,...} shells).
|
||||
if (tiered.TryGetProperty("tiers", out var tiers)
|
||||
&& tiers.ValueKind == JsonValueKind.Array
|
||||
&& tiers.GetArrayLength() > 0)
|
||||
{
|
||||
return tiered.GetRawText();
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool TryParseMoney(string? s, out decimal value) =>
|
||||
decimal.TryParse(s, NumberStyles.Any, CultureInfo.InvariantCulture, out value);
|
||||
|
||||
// ── Cure schedule parsing ─────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Extracts every "N minutes at/@ TTT°F" curve from a free-text cure schedule, in document
|
||||
/// order. The first is treated as the primary/standard curve; the rest are alternate (often
|
||||
/// lower-temperature) curves preserved for heat-sensitive substrates. Returns an empty list for
|
||||
/// schedules with no parseable temp/time pair (partial-cure / clear-coat instructions).
|
||||
/// </summary>
|
||||
public static List<CureCurve> ParseCureCurves(string? cureSchedule)
|
||||
{
|
||||
var result = new List<CureCurve>();
|
||||
if (string.IsNullOrWhiteSpace(cureSchedule))
|
||||
return result;
|
||||
|
||||
foreach (Match m in CureCurveRegex.Matches(cureSchedule))
|
||||
{
|
||||
if (int.TryParse(m.Groups[1].Value, out var minutes)
|
||||
&& int.TryParse(m.Groups[2].Value, out var tempF)
|
||||
&& tempF is >= 150 and <= 600 // sanity: real cure temps
|
||||
&& minutes is > 0 and <= 120)
|
||||
{
|
||||
var curve = new CureCurve(tempF, minutes);
|
||||
if (!result.Contains(curve))
|
||||
result.Add(curve);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Flags powders that need a clear coat — explicit "clear coat"/"requires a clear" mentions, or
|
||||
/// multi-step partial-cure (Illusion) instructions that have no single cure temperature.
|
||||
/// </summary>
|
||||
public static bool DetectRequiresClearCoat(ColumbiaProduct p)
|
||||
{
|
||||
var haystack = $"{p.CureSchedule} {p.Description}";
|
||||
return haystack.Contains("clear coat", StringComparison.OrdinalIgnoreCase)
|
||||
|| haystack.Contains("requires a clear", StringComparison.OrdinalIgnoreCase)
|
||||
|| haystack.Contains("Illusion", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
// ── Misc field helpers ────────────────────────────────────────────────
|
||||
|
||||
/// <summary>Joins the color-group taxonomy ({name} entries) into a comma-separated families string.</summary>
|
||||
public static string? BuildColorFamilies(ColumbiaProduct p)
|
||||
{
|
||||
var groups = p.PaColorGroup.Select(g => g.Name.Trim()).Where(n => n.Length > 0).Distinct().ToList();
|
||||
|
||||
if (groups.Count == 0)
|
||||
{
|
||||
// Fall back to the "Color Group" attribute options when the taxonomy is empty.
|
||||
groups = p.Attributes
|
||||
.Where(a => a.Name.Equals("Color Group", StringComparison.OrdinalIgnoreCase))
|
||||
.SelectMany(a => a.Options.Select(o => o.Name.Trim()))
|
||||
.Where(n => n.Length > 0)
|
||||
.Distinct()
|
||||
.ToList();
|
||||
}
|
||||
|
||||
return groups.Count > 0 ? string.Join(",", groups) : null;
|
||||
}
|
||||
|
||||
/// <summary>Normalizes resin chemistry — trims, and collapses the three Polyester/TGIC spellings.</summary>
|
||||
public static string? NormalizeChemistry(string? type)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(type))
|
||||
return null;
|
||||
var trimmed = type.Trim();
|
||||
return PolyesterTgic.IsMatch(trimmed) ? "Polyester/TGIC" : trimmed;
|
||||
}
|
||||
|
||||
/// <summary>Strips HTML tags/entities from a description and collapses whitespace to plain text.</summary>
|
||||
public static string? StripHtml(string? html)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(html))
|
||||
return null;
|
||||
|
||||
var text = HtmlTag.Replace(html, " ");
|
||||
text = WebUtility.HtmlDecode(text);
|
||||
text = text.Replace("\r", " ").Replace("\n", " ").Replace("\t", " ");
|
||||
text = WhitespaceRun.Replace(text, " ").Trim();
|
||||
return text.Length > 0 ? text : null;
|
||||
}
|
||||
|
||||
private static string? NullIfBlank(string? s) => string.IsNullOrWhiteSpace(s) ? null : s.Trim();
|
||||
|
||||
private static string? FirstNonBlank(params string?[] values) =>
|
||||
values.FirstOrDefault(v => !string.IsNullOrWhiteSpace(v));
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using PowderCoating.Application.Constants;
|
||||
using PowderCoating.Application.Interfaces;
|
||||
using PowderCoating.Core.Entities;
|
||||
using PowderCoating.Core.Interfaces;
|
||||
|
||||
namespace PowderCoating.Infrastructure.Services.Columbia;
|
||||
|
||||
/// <summary>
|
||||
/// Full Columbia Coatings catalog sync: pages the API, maps each product, upserts via the shared
|
||||
/// <see cref="IPowderCatalogUpsertService"/>, then reconciles discontinuations against the complete
|
||||
/// pull. The discontinuation sweep runs ONLY after a successful full fetch — a partial pull (any
|
||||
/// page failure throws from the client) aborts before the sweep so a transient error can never mass
|
||||
/// flag the catalog as discontinued.
|
||||
/// </summary>
|
||||
public class ColumbiaCatalogSyncService : IColumbiaCatalogSyncService
|
||||
{
|
||||
private readonly IColumbiaCoatingsApiClient _client;
|
||||
private readonly IPowderCatalogUpsertService _upsert;
|
||||
private readonly IUnitOfWork _unitOfWork;
|
||||
private readonly IPlatformSettingsService _settings;
|
||||
private readonly ILogger<ColumbiaCatalogSyncService> _logger;
|
||||
|
||||
public ColumbiaCatalogSyncService(
|
||||
IColumbiaCoatingsApiClient client,
|
||||
IPowderCatalogUpsertService upsert,
|
||||
IUnitOfWork unitOfWork,
|
||||
IPlatformSettingsService settings,
|
||||
ILogger<ColumbiaCatalogSyncService> logger)
|
||||
{
|
||||
_client = client;
|
||||
_upsert = upsert;
|
||||
_unitOfWork = unitOfWork;
|
||||
_settings = settings;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<ColumbiaSyncResult> RunSyncAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var result = new ColumbiaSyncResult { StartedAt = DateTime.UtcNow };
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
|
||||
if (!_client.IsConfigured)
|
||||
{
|
||||
result.ErrorMessage = "Columbia API key is not configured.";
|
||||
await RecordResultAsync(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// Full pull — throws on any page failure, which we treat as an incomplete sync.
|
||||
var products = await _client.GetAllProductsAsync(cancellationToken);
|
||||
result.TotalFetched = products.Count;
|
||||
|
||||
// Map and de-duplicate by (VendorName, SKU) in case the feed repeats a SKU.
|
||||
var mapped = products
|
||||
.Select(ColumbiaCatalogMapper.Map)
|
||||
.Where(m => !string.IsNullOrWhiteSpace(m.Sku))
|
||||
.GroupBy(m => $"{m.VendorName}|{m.Sku}", StringComparer.OrdinalIgnoreCase)
|
||||
.Select(g => g.First())
|
||||
.ToList();
|
||||
|
||||
var upsertResult = await _upsert.UpsertAsync(mapped, result.StartedAt, cancellationToken);
|
||||
result.Inserted = upsertResult.Inserted;
|
||||
result.Updated = upsertResult.Updated;
|
||||
result.Unchanged = upsertResult.Unchanged;
|
||||
result.Skipped = upsertResult.Skipped;
|
||||
|
||||
// Complete pull succeeded — safe to reconcile discontinuations.
|
||||
var incomingKeys = mapped
|
||||
.Select(m => $"{m.VendorName}|{m.Sku}")
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
(result.Discontinued, result.Reactivated) =
|
||||
await ReconcileDiscontinuationsAsync(incomingKeys, result.StartedAt);
|
||||
|
||||
result.Success = true;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Columbia catalog sync failed; skipping discontinuation sweep.");
|
||||
result.Success = false;
|
||||
result.ErrorMessage = ex.Message;
|
||||
}
|
||||
|
||||
stopwatch.Stop();
|
||||
result.Duration = stopwatch.Elapsed;
|
||||
await RecordResultAsync(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Flags catalog items sourced from Columbia that were NOT in this complete pull as discontinued,
|
||||
/// and reactivates any previously-discontinued item that has reappeared. Returns (discontinued,
|
||||
/// reactivated) counts.
|
||||
/// </summary>
|
||||
private async Task<(int Discontinued, int Reactivated)> ReconcileDiscontinuationsAsync(
|
||||
HashSet<string> incomingKeys, DateTime runTimestamp)
|
||||
{
|
||||
var sourced = await _unitOfWork.PowderCatalog.FindAsync(
|
||||
p => p.Source == ColumbiaIntegrationConstants.SourceName);
|
||||
|
||||
var discontinued = 0;
|
||||
var reactivated = 0;
|
||||
|
||||
foreach (var item in sourced)
|
||||
{
|
||||
var present = incomingKeys.Contains($"{item.VendorName}|{item.Sku}");
|
||||
|
||||
if (!present && !item.IsDiscontinued)
|
||||
{
|
||||
item.IsDiscontinued = true;
|
||||
item.UpdatedAt = runTimestamp;
|
||||
await _unitOfWork.PowderCatalog.UpdateAsync(item);
|
||||
discontinued++;
|
||||
}
|
||||
else if (present && item.IsDiscontinued)
|
||||
{
|
||||
item.IsDiscontinued = false;
|
||||
item.UpdatedAt = runTimestamp;
|
||||
await _unitOfWork.PowderCatalog.UpdateAsync(item);
|
||||
reactivated++;
|
||||
}
|
||||
}
|
||||
|
||||
if (discontinued > 0 || reactivated > 0)
|
||||
await _unitOfWork.CompleteAsync();
|
||||
|
||||
return (discontinued, reactivated);
|
||||
}
|
||||
|
||||
/// <summary>Persists the run outcome to the last-synced / last-result platform settings.</summary>
|
||||
private async Task RecordResultAsync(ColumbiaSyncResult result)
|
||||
{
|
||||
if (result.Success)
|
||||
{
|
||||
await _settings.SetAsync(
|
||||
ColumbiaIntegrationConstants.SettingLastSyncedAt,
|
||||
result.StartedAt.ToString("O", CultureInfo.InvariantCulture),
|
||||
updatedBy: "Columbia Sync");
|
||||
}
|
||||
|
||||
await _settings.SetAsync(
|
||||
ColumbiaIntegrationConstants.SettingLastResult,
|
||||
result.Summary,
|
||||
updatedBy: "Columbia Sync");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using PowderCoating.Application.Interfaces;
|
||||
using PowderCoating.Core.Entities;
|
||||
using PowderCoating.Core.Interfaces;
|
||||
|
||||
namespace PowderCoating.Infrastructure.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Single upsert path for the platform <see cref="PowderCatalogItem"/> master list, shared by the
|
||||
/// JSON file import and the Columbia API sync. Match key is (VendorName, SKU), case-insensitive.
|
||||
/// </summary>
|
||||
public class PowderCatalogUpsertService : IPowderCatalogUpsertService
|
||||
{
|
||||
private readonly IUnitOfWork _unitOfWork;
|
||||
private readonly ILogger<PowderCatalogUpsertService> _logger;
|
||||
|
||||
public PowderCatalogUpsertService(IUnitOfWork unitOfWork, ILogger<PowderCatalogUpsertService> logger)
|
||||
{
|
||||
_unitOfWork = unitOfWork;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<PowderCatalogUpsertResult> UpsertAsync(
|
||||
IReadOnlyList<PowderCatalogItem> incoming,
|
||||
DateTime runTimestamp,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var result = new PowderCatalogUpsertResult();
|
||||
|
||||
// Load existing rows for just the vendors we're touching, keyed by (vendor|sku) lower-cased.
|
||||
var vendorNames = incoming
|
||||
.Select(i => i.VendorName)
|
||||
.Where(v => !string.IsNullOrWhiteSpace(v))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
var existing = (await _unitOfWork.PowderCatalog.FindAsync(p => vendorNames.Contains(p.VendorName)))
|
||||
.ToDictionary(KeyOf, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var toAdd = new List<PowderCatalogItem>();
|
||||
|
||||
foreach (var item in incoming)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(item.Sku) || string.IsNullOrWhiteSpace(item.ColorName))
|
||||
{
|
||||
result.Skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (existing.TryGetValue(KeyOf(item), out var record))
|
||||
{
|
||||
if (ApplyFeedFields(record, item))
|
||||
{
|
||||
record.UpdatedAt = runTimestamp;
|
||||
record.LastSyncedAt = runTimestamp;
|
||||
await _unitOfWork.PowderCatalog.UpdateAsync(record);
|
||||
result.Updated++;
|
||||
}
|
||||
else
|
||||
{
|
||||
result.Unchanged++;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
item.CreatedAt = runTimestamp;
|
||||
item.LastSyncedAt = runTimestamp;
|
||||
toAdd.Add(item);
|
||||
result.Inserted++;
|
||||
}
|
||||
}
|
||||
|
||||
if (toAdd.Count > 0)
|
||||
await _unitOfWork.PowderCatalog.AddRangeAsync(toAdd);
|
||||
|
||||
await _unitOfWork.CompleteAsync();
|
||||
|
||||
_logger.LogInformation(
|
||||
"Powder catalog upsert: {Inserted} inserted, {Updated} updated, {Unchanged} unchanged, {Skipped} skipped.",
|
||||
result.Inserted, result.Updated, result.Unchanged, result.Skipped);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static string KeyOf(PowderCatalogItem p) => $"{p.VendorName}|{p.Sku}";
|
||||
|
||||
/// <summary>
|
||||
/// Copies feed-sourced fields from <paramref name="src"/> onto <paramref name="dest"/> and
|
||||
/// returns true if anything changed. Deliberately leaves enrichment fields (SpecificGravity,
|
||||
/// CoverageSqFtPerLb, TransferEfficiency, Finish) and lifecycle flags untouched — those are
|
||||
/// owned by lazy TDS/AI enrichment and the discontinuation sweep, not the feed.
|
||||
/// </summary>
|
||||
private static bool ApplyFeedFields(PowderCatalogItem dest, PowderCatalogItem src)
|
||||
{
|
||||
var changed = false;
|
||||
|
||||
changed |= Set(() => dest.ColorName, v => dest.ColorName = v, src.ColorName);
|
||||
changed |= Set(() => dest.Description, v => dest.Description = v, src.Description);
|
||||
changed |= dest.UnitPrice != src.UnitPrice && Assign(() => dest.UnitPrice = src.UnitPrice);
|
||||
changed |= Set(() => dest.PriceTiersJson, v => dest.PriceTiersJson = v, src.PriceTiersJson);
|
||||
changed |= Set(() => dest.ImageUrl, v => dest.ImageUrl = v, src.ImageUrl);
|
||||
changed |= Set(() => dest.SdsUrl, v => dest.SdsUrl = v, src.SdsUrl);
|
||||
changed |= Set(() => dest.TdsUrl, v => dest.TdsUrl = v, src.TdsUrl);
|
||||
changed |= Set(() => dest.ApplicationGuideUrl, v => dest.ApplicationGuideUrl = v, src.ApplicationGuideUrl);
|
||||
changed |= Set(() => dest.ProductUrl, v => dest.ProductUrl = v, src.ProductUrl);
|
||||
changed |= Set(() => dest.ChemistryType, v => dest.ChemistryType = v, src.ChemistryType);
|
||||
changed |= Set(() => dest.MilThickness, v => dest.MilThickness = v, src.MilThickness);
|
||||
changed |= Set(() => dest.CureScheduleText, v => dest.CureScheduleText = v, src.CureScheduleText);
|
||||
changed |= Set(() => dest.CureCurvesJson, v => dest.CureCurvesJson = v, src.CureCurvesJson);
|
||||
changed |= dest.CureTemperatureF != src.CureTemperatureF && Assign(() => dest.CureTemperatureF = src.CureTemperatureF);
|
||||
changed |= dest.CureTimeMinutes != src.CureTimeMinutes && Assign(() => dest.CureTimeMinutes = src.CureTimeMinutes);
|
||||
changed |= dest.RequiresClearCoat != src.RequiresClearCoat && Assign(() => dest.RequiresClearCoat = src.RequiresClearCoat);
|
||||
changed |= Set(() => dest.ColorFamilies, v => dest.ColorFamilies = v, src.ColorFamilies);
|
||||
changed |= Set(() => dest.FormulationChanges, v => dest.FormulationChanges = v, src.FormulationChanges);
|
||||
changed |= Set(() => dest.Category, v => dest.Category = v, src.Category);
|
||||
changed |= Set(() => dest.Source, v => dest.Source = v, src.Source);
|
||||
|
||||
return changed;
|
||||
}
|
||||
|
||||
/// <summary>Sets a nullable-string property when it differs (ordinal compare); returns whether it changed.</summary>
|
||||
private static bool Set(Func<string?> get, Action<string?> set, string? newValue)
|
||||
{
|
||||
if (!string.Equals(get(), newValue, StringComparison.Ordinal))
|
||||
{
|
||||
set(newValue);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>Helper so a value assignment can participate in a boolean OR chain.</summary>
|
||||
private static bool Assign(Action assign)
|
||||
{
|
||||
assign();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ using PowderCoating.Core.Interfaces.Services;
|
||||
using PowderCoating.Infrastructure.Data;
|
||||
using PowderCoating.Infrastructure.Repositories;
|
||||
using PowderCoating.Infrastructure.Services;
|
||||
using PowderCoating.Infrastructure.Services.Columbia;
|
||||
using PowderCoating.Application.Interfaces;
|
||||
using PowderCoating.Application.Services;
|
||||
using PowderCoating.Application.Configuration;
|
||||
@@ -223,6 +224,8 @@ builder.Services.AddScoped<ICustomFormulaAiService, CustomFormulaAiService>();
|
||||
builder.Services.AddScoped<IFormulaLibraryService, FormulaLibraryService>();
|
||||
builder.Services.AddHttpClient();
|
||||
builder.Services.AddScoped<IColumbiaCoatingsApiClient, ColumbiaCoatingsApiClient>();
|
||||
builder.Services.AddScoped<IPowderCatalogUpsertService, PowderCatalogUpsertService>();
|
||||
builder.Services.AddScoped<IColumbiaCatalogSyncService, ColumbiaCatalogSyncService>();
|
||||
builder.Services.AddScoped<ICompanyLogoService, CompanyLogoService>();
|
||||
builder.Services.AddScoped<IEquipmentManualService, EquipmentManualService>();
|
||||
builder.Services.AddScoped<IPricingCalculationService, PricingCalculationService>();
|
||||
|
||||
Reference in New Issue
Block a user