Add Columbia catalog mapper, shared upsert, and sync service

Phase 2: the mapping and sync core.

- ColumbiaCatalogMapper (pure/static, unit-tested): maps an API product to a
  PowderCatalogItem. Derives manufacturer (PPG/KP Pigments/Columbia) from
  taxonomy+SKU; flags additives into the Powder Additives category; takes base
  price from the top-level price with variant fallback; captures variation /
  tiered pricing as JSON; parses the free-text cure schedule into all curves
  (three degree glyphs, @/at, multi-curve in order, partial-cure -> none) with
  the first as the primary temp/time; strips HTML descriptions; joins color
  groups; normalizes chemistry; flags clear-coat powders.

- PowderCatalogUpsertService (IPowderCatalogUpsertService): single upsert path
  matching on (VendorName, SKU). Copies only feed-sourced fields and leaves
  enrichment fields (specific gravity, coverage, transfer efficiency, finish)
  untouched so syncs never wipe lazily-enriched TDS/AI data.

- ColumbiaCatalogSyncService (IColumbiaCatalogSyncService): pulls the full
  catalog, maps + de-dupes, upserts, then reconciles discontinuations ONLY on a
  complete pull (a partial pull throws and aborts before the sweep). Reactivates
  reappearing items; records last-synced/last-result platform settings.

- 25 mapper unit tests covering the cure parser, manufacturer derivation,
  simple/variable pricing, chemistry, color, and HTML cases from real records.
  Full suite green (261 passed).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-17 11:02:12 -04:00
parent a4a3dde7e4
commit 2b420d4623
8 changed files with 877 additions and 5 deletions
@@ -54,10 +54,11 @@ public class ColumbiaProduct
/// <summary>
/// Quantity-break pricing. POLYMORPHIC on the wire: an object
/// (<c>{type, minimum_quantity, tiers:[...]}</c>) on simple products, but an empty ARRAY
/// (<c>[]</c>) on variable products. Captured as a raw <see cref="JsonElement"/> so
/// deserialization never throws on the type mismatch; the mapper interprets it.
/// (<c>[]</c>) on variable products. Captured as a nullable raw <see cref="JsonElement"/> so
/// deserialization never throws on the type mismatch and an absent value is null (not an
/// invalid <c>Undefined</c> element); the mapper interprets it.
/// </summary>
public JsonElement TieredPricing { get; set; }
public JsonElement? TieredPricing { get; set; }
/// <summary>Per-variant pricing for variable products (e.g. Bulk vs 1 lb Bags, or gram sizes).
/// Each variant has its own SKU and price. Empty for simple products.</summary>
@@ -108,8 +109,8 @@ public class ColumbiaVariationPricing
public string RegularPrice { get; set; } = string.Empty;
public string SalePrice { get; set; } = string.Empty;
/// <summary>Same polymorphic object-or-array shape as the parent; captured raw.</summary>
public JsonElement TieredPricing { get; set; }
/// <summary>Same polymorphic object-or-array shape as the parent; captured raw (nullable).</summary>
public JsonElement? TieredPricing { get; set; }
}
/// <summary>An image object — featured or gallery.</summary>
@@ -0,0 +1,41 @@
namespace PowderCoating.Application.Interfaces;
/// <summary>
/// Orchestrates a full Columbia Coatings catalog sync: pull every product, map and upsert it, then
/// (only on a complete pull) reconcile discontinuations. Used by both the scheduled background job
/// and the manual "Sync now" admin action.
/// </summary>
public interface IColumbiaCatalogSyncService
{
/// <summary>
/// Runs one full sync. Assumes the caller has already decided it should run (enabled / due).
/// Returns a result describing the outcome; never throws for an expected failure (not
/// configured, partial pull, HTTP error) — those are reported on the result instead.
/// </summary>
Task<ColumbiaSyncResult> RunSyncAsync(CancellationToken cancellationToken = default);
}
/// <summary>Outcome of a Columbia catalog sync run.</summary>
public class ColumbiaSyncResult
{
public bool Success { get; set; }
public string? ErrorMessage { get; set; }
public int TotalFetched { get; set; }
public int Inserted { get; set; }
public int Updated { get; set; }
public int Unchanged { get; set; }
public int Skipped { get; set; }
public int Discontinued { get; set; }
public int Reactivated { get; set; }
public DateTime StartedAt { get; set; }
public TimeSpan Duration { get; set; }
/// <summary>One-line summary suitable for storing in the last-result platform setting / UI.</summary>
public string Summary =>
Success
? $"{TotalFetched} fetched: {Inserted} new, {Updated} updated, {Unchanged} unchanged, " +
$"{Discontinued} discontinued, {Reactivated} reactivated ({Duration.TotalSeconds:F0}s)"
: $"Failed: {ErrorMessage}";
}
@@ -0,0 +1,32 @@
using PowderCoating.Core.Entities;
namespace PowderCoating.Application.Interfaces;
/// <summary>
/// Shared upsert for the platform powder catalog: matches incoming records to existing rows by
/// (VendorName, SKU), inserts new ones, and updates changed ones in place. Used by BOTH the manual
/// JSON file import and the Columbia API sync so there is a single upsert path, only the mapping
/// differs. Does NOT handle discontinuation — that is a sync-specific concern.
/// </summary>
public interface IPowderCatalogUpsertService
{
/// <summary>
/// Applies <paramref name="incoming"/> mapped catalog items. Only fields sourced from the feed
/// are copied on update; enrichment fields (specific gravity, coverage, transfer efficiency,
/// finish) are preserved so they are not wiped by a feed that never carries them. Changed and
/// inserted rows get <paramref name="runTimestamp"/> stamped on LastSyncedAt/UpdatedAt.
/// </summary>
Task<PowderCatalogUpsertResult> UpsertAsync(
IReadOnlyList<PowderCatalogItem> incoming,
DateTime runTimestamp,
CancellationToken cancellationToken = default);
}
/// <summary>Counts from an upsert run.</summary>
public class PowderCatalogUpsertResult
{
public int Inserted { get; set; }
public int Updated { get; set; }
public int Unchanged { get; set; }
public int Skipped { get; set; }
}
@@ -0,0 +1,256 @@
using System.Globalization;
using System.Net;
using System.Text.Json;
using System.Text.RegularExpressions;
using PowderCoating.Application.Constants;
using PowderCoating.Application.DTOs.Columbia;
using PowderCoating.Core.Entities;
namespace PowderCoating.Infrastructure.Services.Columbia;
/// <summary>
/// Maps a Columbia Coatings API product onto our platform <see cref="PowderCatalogItem"/>.
/// Pure, static, side-effect free so the tricky bits (manufacturer derivation, the free-text
/// cure-schedule parser, HTML stripping) can be unit tested directly against captured fixtures.
/// <para>
/// Columbia is a distributor reselling multiple brands, so <see cref="PowderCatalogItem.VendorName"/>
/// holds the DERIVED manufacturer (PPG / KP Pigments / Columbia) while
/// <see cref="PowderCatalogItem.Source"/> records the feed ("Columbia Coatings API") for
/// right-to-delete purges. The vendor's own categories/tags are read here only to derive the
/// manufacturer and additive flag — they are never stored raw.
/// </para>
/// </summary>
public static class ColumbiaCatalogMapper
{
/// <summary>A single parsed cure curve — hold <see cref="Minutes"/> at <see cref="TempF"/>.</summary>
public readonly record struct CureCurve(int TempF, int Minutes);
// Resin chemistries that mean "polyester + TGIC" but arrive formatted three different ways.
private static readonly Regex PolyesterTgic =
new(@"^\s*(polyester\s*[/ ]\s*tgic|tgic\s*[/ ]\s*polyester)\s*$", RegexOptions.IgnoreCase | RegexOptions.Compiled);
// "10 minutes @ 400°F", "7 minutes at 375 F", "Metal Temperature: 10 minutes at 400°F (204°C)".
// Degree glyph is optional and may be ° (U+00B0), ˚ (U+02DA), or º (U+00BA).
private static readonly Regex CureCurveRegex =
new(@"(\d+)\s*min(?:ute)?s?\.?\s*(?:@|at)\s*(\d{2,3})\s*[°˚º]?\s*F",
RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex HtmlTag = new("<[^>]+>", RegexOptions.Compiled);
private static readonly Regex WhitespaceRun = new(@"\s{2,}", RegexOptions.Compiled);
private static readonly JsonSerializerOptions JsonOut = new() { WriteIndented = false };
/// <summary>Maps a Columbia product into a fully populated (unsaved) catalog item.</summary>
public static PowderCatalogItem Map(ColumbiaProduct p)
{
var curves = ParseCureCurves(p.CureSchedule);
var primary = curves.Count > 0 ? curves[0] : (CureCurve?)null;
return new PowderCatalogItem
{
VendorName = DeriveManufacturer(p),
Sku = p.Sku.Trim(),
ColorName = p.Name.Trim(),
Source = ColumbiaIntegrationConstants.SourceName,
Category = IsAdditive(p) ? ColumbiaIntegrationConstants.CategoryPowderAdditives : null,
Description = StripHtml(p.Description),
UnitPrice = ParseBasePrice(p),
PriceTiersJson = BuildPriceTiersJson(p),
ImageUrl = NullIfBlank(p.FeaturedImage?.Src),
SdsUrl = NullIfBlank(p.SafetyDataSheet),
TdsUrl = NullIfBlank(p.TechnicalDataSheet),
ApplicationGuideUrl = NullIfBlank(FirstNonBlank(p.ProductFlyer, p.ProductBrochure)),
ProductUrl = NullIfBlank(p.Permalink),
ChemistryType = NormalizeChemistry(p.Type),
MilThickness = NullIfBlank(p.MilThickness),
CureScheduleText = NullIfBlank(p.CureSchedule),
CureCurvesJson = curves.Count > 0 ? JsonSerializer.Serialize(curves, JsonOut) : null,
CureTemperatureF = primary?.TempF,
CureTimeMinutes = primary?.Minutes,
RequiresClearCoat = DetectRequiresClearCoat(p),
ColorFamilies = BuildColorFamilies(p),
FormulationChanges = NullIfBlank(p.FormulationDateChanges),
// Coverage / specific gravity / transfer efficiency are not in the API — left null for
// lazy TDS/AI enrichment on first use. IsDiscontinued is handled by the sync sweep.
};
}
// ── Manufacturer derivation ───────────────────────────────────────────
/// <summary>
/// Derives the manufacturer from the product's taxonomy/SKU. Columbia resells PPG powders and
/// KP Pigments additives through the same feed; everything else is Columbia's own brand.
/// </summary>
public static string DeriveManufacturer(ColumbiaProduct p)
{
if (IsKpPigments(p))
return ColumbiaIntegrationConstants.ManufacturerKp;
if (IsPpg(p))
return ColumbiaIntegrationConstants.ManufacturerPpg;
return ColumbiaIntegrationConstants.ManufacturerColumbia;
}
private static bool IsKpPigments(ColumbiaProduct p) =>
p.Sku.StartsWith("ADD-", StringComparison.OrdinalIgnoreCase)
|| CategoryStartsWith(p, "KP");
private static bool IsPpg(ColumbiaProduct p) =>
CategoryStartsWith(p, "PPG");
/// <summary>
/// True for pigments/additives sold by weight (grams) rather than sprayed powders. These get
/// forced into the "Powder Additives" category. Keyed off the broad Additives category and the
/// ADD- SKU prefix, not just the KP brand (there are ~98 non-KP additives).
/// </summary>
public static bool IsAdditive(ColumbiaProduct p) =>
p.Sku.StartsWith("ADD-", StringComparison.OrdinalIgnoreCase)
|| p.Categories.Any(c => c.Name.Equals("Additives", StringComparison.OrdinalIgnoreCase))
|| CategoryStartsWith(p, "KP");
private static bool CategoryStartsWith(ColumbiaProduct p, string prefix) =>
p.Categories.Any(c => c.Name.StartsWith(prefix, StringComparison.OrdinalIgnoreCase));
// ── Pricing ───────────────────────────────────────────────────────────
/// <summary>
/// Base unit price = the top-level <c>price</c> (falling back to <c>regular_price</c>). For
/// variable products the parent <c>price</c> already carries the lead variant's price, while
/// <c>regular_price</c> is often "0", so price is preferred.
/// </summary>
public static decimal ParseBasePrice(ColumbiaProduct p)
{
if (TryParseMoney(p.Price, out var price) && price > 0)
return price;
if (TryParseMoney(p.RegularPrice, out var regular) && regular > 0)
return regular;
// Variable product with a zero parent price: fall back to the lowest variant price.
var variantPrices = (p.VariationPricing ?? new List<ColumbiaVariationPricing>())
.Select(v => TryParseMoney(v.Price, out var vp) ? vp : 0m)
.Where(v => v > 0)
.ToList();
return variantPrices.Count > 0 ? variantPrices.Min() : 0m;
}
/// <summary>
/// Captures quantity-break / variant pricing as JSON for later use. For variable products this
/// is the per-variant pricing (Bulk vs 1 lb Bags, gram sizes); for simple products it's the
/// tiered_pricing object. Null when neither is present.
/// </summary>
public static string? BuildPriceTiersJson(ColumbiaProduct p)
{
if (p.VariationPricing is { Count: > 0 })
return JsonSerializer.Serialize(p.VariationPricing, JsonOut);
if (p.TieredPricing is { ValueKind: JsonValueKind.Object } tiered)
{
// Only keep it if it actually carries tiers (avoid storing empty {type,...} shells).
if (tiered.TryGetProperty("tiers", out var tiers)
&& tiers.ValueKind == JsonValueKind.Array
&& tiers.GetArrayLength() > 0)
{
return tiered.GetRawText();
}
}
return null;
}
private static bool TryParseMoney(string? s, out decimal value) =>
decimal.TryParse(s, NumberStyles.Any, CultureInfo.InvariantCulture, out value);
// ── Cure schedule parsing ─────────────────────────────────────────────
/// <summary>
/// Extracts every "N minutes at/@ TTT°F" curve from a free-text cure schedule, in document
/// order. The first is treated as the primary/standard curve; the rest are alternate (often
/// lower-temperature) curves preserved for heat-sensitive substrates. Returns an empty list for
/// schedules with no parseable temp/time pair (partial-cure / clear-coat instructions).
/// </summary>
public static List<CureCurve> ParseCureCurves(string? cureSchedule)
{
var result = new List<CureCurve>();
if (string.IsNullOrWhiteSpace(cureSchedule))
return result;
foreach (Match m in CureCurveRegex.Matches(cureSchedule))
{
if (int.TryParse(m.Groups[1].Value, out var minutes)
&& int.TryParse(m.Groups[2].Value, out var tempF)
&& tempF is >= 150 and <= 600 // sanity: real cure temps
&& minutes is > 0 and <= 120)
{
var curve = new CureCurve(tempF, minutes);
if (!result.Contains(curve))
result.Add(curve);
}
}
return result;
}
/// <summary>
/// Flags powders that need a clear coat — explicit "clear coat"/"requires a clear" mentions, or
/// multi-step partial-cure (Illusion) instructions that have no single cure temperature.
/// </summary>
public static bool DetectRequiresClearCoat(ColumbiaProduct p)
{
var haystack = $"{p.CureSchedule} {p.Description}";
return haystack.Contains("clear coat", StringComparison.OrdinalIgnoreCase)
|| haystack.Contains("requires a clear", StringComparison.OrdinalIgnoreCase)
|| haystack.Contains("Illusion", StringComparison.OrdinalIgnoreCase);
}
// ── Misc field helpers ────────────────────────────────────────────────
/// <summary>Joins the color-group taxonomy ({name} entries) into a comma-separated families string.</summary>
public static string? BuildColorFamilies(ColumbiaProduct p)
{
var groups = p.PaColorGroup.Select(g => g.Name.Trim()).Where(n => n.Length > 0).Distinct().ToList();
if (groups.Count == 0)
{
// Fall back to the "Color Group" attribute options when the taxonomy is empty.
groups = p.Attributes
.Where(a => a.Name.Equals("Color Group", StringComparison.OrdinalIgnoreCase))
.SelectMany(a => a.Options.Select(o => o.Name.Trim()))
.Where(n => n.Length > 0)
.Distinct()
.ToList();
}
return groups.Count > 0 ? string.Join(",", groups) : null;
}
/// <summary>Normalizes resin chemistry — trims, and collapses the three Polyester/TGIC spellings.</summary>
public static string? NormalizeChemistry(string? type)
{
if (string.IsNullOrWhiteSpace(type))
return null;
var trimmed = type.Trim();
return PolyesterTgic.IsMatch(trimmed) ? "Polyester/TGIC" : trimmed;
}
/// <summary>Strips HTML tags/entities from a description and collapses whitespace to plain text.</summary>
public static string? StripHtml(string? html)
{
if (string.IsNullOrWhiteSpace(html))
return null;
var text = HtmlTag.Replace(html, " ");
text = WebUtility.HtmlDecode(text);
text = text.Replace("\r", " ").Replace("\n", " ").Replace("\t", " ");
text = WhitespaceRun.Replace(text, " ").Trim();
return text.Length > 0 ? text : null;
}
private static string? NullIfBlank(string? s) => string.IsNullOrWhiteSpace(s) ? null : s.Trim();
private static string? FirstNonBlank(params string?[] values) =>
values.FirstOrDefault(v => !string.IsNullOrWhiteSpace(v));
}
@@ -0,0 +1,155 @@
using System.Diagnostics;
using System.Globalization;
using Microsoft.Extensions.Logging;
using PowderCoating.Application.Constants;
using PowderCoating.Application.Interfaces;
using PowderCoating.Core.Entities;
using PowderCoating.Core.Interfaces;
namespace PowderCoating.Infrastructure.Services.Columbia;
/// <summary>
/// Full Columbia Coatings catalog sync: pages the API, maps each product, upserts via the shared
/// <see cref="IPowderCatalogUpsertService"/>, then reconciles discontinuations against the complete
/// pull. The discontinuation sweep runs ONLY after a successful full fetch — a partial pull (any
/// page failure throws from the client) aborts before the sweep so a transient error can never mass
/// flag the catalog as discontinued.
/// </summary>
public class ColumbiaCatalogSyncService : IColumbiaCatalogSyncService
{
private readonly IColumbiaCoatingsApiClient _client;
private readonly IPowderCatalogUpsertService _upsert;
private readonly IUnitOfWork _unitOfWork;
private readonly IPlatformSettingsService _settings;
private readonly ILogger<ColumbiaCatalogSyncService> _logger;
public ColumbiaCatalogSyncService(
IColumbiaCoatingsApiClient client,
IPowderCatalogUpsertService upsert,
IUnitOfWork unitOfWork,
IPlatformSettingsService settings,
ILogger<ColumbiaCatalogSyncService> logger)
{
_client = client;
_upsert = upsert;
_unitOfWork = unitOfWork;
_settings = settings;
_logger = logger;
}
/// <inheritdoc />
public async Task<ColumbiaSyncResult> RunSyncAsync(CancellationToken cancellationToken = default)
{
var result = new ColumbiaSyncResult { StartedAt = DateTime.UtcNow };
var stopwatch = Stopwatch.StartNew();
if (!_client.IsConfigured)
{
result.ErrorMessage = "Columbia API key is not configured.";
await RecordResultAsync(result);
return result;
}
try
{
// Full pull — throws on any page failure, which we treat as an incomplete sync.
var products = await _client.GetAllProductsAsync(cancellationToken);
result.TotalFetched = products.Count;
// Map and de-duplicate by (VendorName, SKU) in case the feed repeats a SKU.
var mapped = products
.Select(ColumbiaCatalogMapper.Map)
.Where(m => !string.IsNullOrWhiteSpace(m.Sku))
.GroupBy(m => $"{m.VendorName}|{m.Sku}", StringComparer.OrdinalIgnoreCase)
.Select(g => g.First())
.ToList();
var upsertResult = await _upsert.UpsertAsync(mapped, result.StartedAt, cancellationToken);
result.Inserted = upsertResult.Inserted;
result.Updated = upsertResult.Updated;
result.Unchanged = upsertResult.Unchanged;
result.Skipped = upsertResult.Skipped;
// Complete pull succeeded — safe to reconcile discontinuations.
var incomingKeys = mapped
.Select(m => $"{m.VendorName}|{m.Sku}")
.ToHashSet(StringComparer.OrdinalIgnoreCase);
(result.Discontinued, result.Reactivated) =
await ReconcileDiscontinuationsAsync(incomingKeys, result.StartedAt);
result.Success = true;
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
_logger.LogError(ex, "Columbia catalog sync failed; skipping discontinuation sweep.");
result.Success = false;
result.ErrorMessage = ex.Message;
}
stopwatch.Stop();
result.Duration = stopwatch.Elapsed;
await RecordResultAsync(result);
return result;
}
/// <summary>
/// Flags catalog items sourced from Columbia that were NOT in this complete pull as discontinued,
/// and reactivates any previously-discontinued item that has reappeared. Returns (discontinued,
/// reactivated) counts.
/// </summary>
private async Task<(int Discontinued, int Reactivated)> ReconcileDiscontinuationsAsync(
HashSet<string> incomingKeys, DateTime runTimestamp)
{
var sourced = await _unitOfWork.PowderCatalog.FindAsync(
p => p.Source == ColumbiaIntegrationConstants.SourceName);
var discontinued = 0;
var reactivated = 0;
foreach (var item in sourced)
{
var present = incomingKeys.Contains($"{item.VendorName}|{item.Sku}");
if (!present && !item.IsDiscontinued)
{
item.IsDiscontinued = true;
item.UpdatedAt = runTimestamp;
await _unitOfWork.PowderCatalog.UpdateAsync(item);
discontinued++;
}
else if (present && item.IsDiscontinued)
{
item.IsDiscontinued = false;
item.UpdatedAt = runTimestamp;
await _unitOfWork.PowderCatalog.UpdateAsync(item);
reactivated++;
}
}
if (discontinued > 0 || reactivated > 0)
await _unitOfWork.CompleteAsync();
return (discontinued, reactivated);
}
/// <summary>Persists the run outcome to the last-synced / last-result platform settings.</summary>
private async Task RecordResultAsync(ColumbiaSyncResult result)
{
if (result.Success)
{
await _settings.SetAsync(
ColumbiaIntegrationConstants.SettingLastSyncedAt,
result.StartedAt.ToString("O", CultureInfo.InvariantCulture),
updatedBy: "Columbia Sync");
}
await _settings.SetAsync(
ColumbiaIntegrationConstants.SettingLastResult,
result.Summary,
updatedBy: "Columbia Sync");
}
}
@@ -0,0 +1,139 @@
using Microsoft.Extensions.Logging;
using PowderCoating.Application.Interfaces;
using PowderCoating.Core.Entities;
using PowderCoating.Core.Interfaces;
namespace PowderCoating.Infrastructure.Services;
/// <summary>
/// Single upsert path for the platform <see cref="PowderCatalogItem"/> master list, shared by the
/// JSON file import and the Columbia API sync. Match key is (VendorName, SKU), case-insensitive.
/// </summary>
public class PowderCatalogUpsertService : IPowderCatalogUpsertService
{
private readonly IUnitOfWork _unitOfWork;
private readonly ILogger<PowderCatalogUpsertService> _logger;
public PowderCatalogUpsertService(IUnitOfWork unitOfWork, ILogger<PowderCatalogUpsertService> logger)
{
_unitOfWork = unitOfWork;
_logger = logger;
}
/// <inheritdoc />
public async Task<PowderCatalogUpsertResult> UpsertAsync(
IReadOnlyList<PowderCatalogItem> incoming,
DateTime runTimestamp,
CancellationToken cancellationToken = default)
{
var result = new PowderCatalogUpsertResult();
// Load existing rows for just the vendors we're touching, keyed by (vendor|sku) lower-cased.
var vendorNames = incoming
.Select(i => i.VendorName)
.Where(v => !string.IsNullOrWhiteSpace(v))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
var existing = (await _unitOfWork.PowderCatalog.FindAsync(p => vendorNames.Contains(p.VendorName)))
.ToDictionary(KeyOf, StringComparer.OrdinalIgnoreCase);
var toAdd = new List<PowderCatalogItem>();
foreach (var item in incoming)
{
if (string.IsNullOrWhiteSpace(item.Sku) || string.IsNullOrWhiteSpace(item.ColorName))
{
result.Skipped++;
continue;
}
if (existing.TryGetValue(KeyOf(item), out var record))
{
if (ApplyFeedFields(record, item))
{
record.UpdatedAt = runTimestamp;
record.LastSyncedAt = runTimestamp;
await _unitOfWork.PowderCatalog.UpdateAsync(record);
result.Updated++;
}
else
{
result.Unchanged++;
}
}
else
{
item.CreatedAt = runTimestamp;
item.LastSyncedAt = runTimestamp;
toAdd.Add(item);
result.Inserted++;
}
}
if (toAdd.Count > 0)
await _unitOfWork.PowderCatalog.AddRangeAsync(toAdd);
await _unitOfWork.CompleteAsync();
_logger.LogInformation(
"Powder catalog upsert: {Inserted} inserted, {Updated} updated, {Unchanged} unchanged, {Skipped} skipped.",
result.Inserted, result.Updated, result.Unchanged, result.Skipped);
return result;
}
private static string KeyOf(PowderCatalogItem p) => $"{p.VendorName}|{p.Sku}";
/// <summary>
/// Copies feed-sourced fields from <paramref name="src"/> onto <paramref name="dest"/> and
/// returns true if anything changed. Deliberately leaves enrichment fields (SpecificGravity,
/// CoverageSqFtPerLb, TransferEfficiency, Finish) and lifecycle flags untouched — those are
/// owned by lazy TDS/AI enrichment and the discontinuation sweep, not the feed.
/// </summary>
private static bool ApplyFeedFields(PowderCatalogItem dest, PowderCatalogItem src)
{
var changed = false;
changed |= Set(() => dest.ColorName, v => dest.ColorName = v, src.ColorName);
changed |= Set(() => dest.Description, v => dest.Description = v, src.Description);
changed |= dest.UnitPrice != src.UnitPrice && Assign(() => dest.UnitPrice = src.UnitPrice);
changed |= Set(() => dest.PriceTiersJson, v => dest.PriceTiersJson = v, src.PriceTiersJson);
changed |= Set(() => dest.ImageUrl, v => dest.ImageUrl = v, src.ImageUrl);
changed |= Set(() => dest.SdsUrl, v => dest.SdsUrl = v, src.SdsUrl);
changed |= Set(() => dest.TdsUrl, v => dest.TdsUrl = v, src.TdsUrl);
changed |= Set(() => dest.ApplicationGuideUrl, v => dest.ApplicationGuideUrl = v, src.ApplicationGuideUrl);
changed |= Set(() => dest.ProductUrl, v => dest.ProductUrl = v, src.ProductUrl);
changed |= Set(() => dest.ChemistryType, v => dest.ChemistryType = v, src.ChemistryType);
changed |= Set(() => dest.MilThickness, v => dest.MilThickness = v, src.MilThickness);
changed |= Set(() => dest.CureScheduleText, v => dest.CureScheduleText = v, src.CureScheduleText);
changed |= Set(() => dest.CureCurvesJson, v => dest.CureCurvesJson = v, src.CureCurvesJson);
changed |= dest.CureTemperatureF != src.CureTemperatureF && Assign(() => dest.CureTemperatureF = src.CureTemperatureF);
changed |= dest.CureTimeMinutes != src.CureTimeMinutes && Assign(() => dest.CureTimeMinutes = src.CureTimeMinutes);
changed |= dest.RequiresClearCoat != src.RequiresClearCoat && Assign(() => dest.RequiresClearCoat = src.RequiresClearCoat);
changed |= Set(() => dest.ColorFamilies, v => dest.ColorFamilies = v, src.ColorFamilies);
changed |= Set(() => dest.FormulationChanges, v => dest.FormulationChanges = v, src.FormulationChanges);
changed |= Set(() => dest.Category, v => dest.Category = v, src.Category);
changed |= Set(() => dest.Source, v => dest.Source = v, src.Source);
return changed;
}
/// <summary>Sets a nullable-string property when it differs (ordinal compare); returns whether it changed.</summary>
private static bool Set(Func<string?> get, Action<string?> set, string? newValue)
{
if (!string.Equals(get(), newValue, StringComparison.Ordinal))
{
set(newValue);
return true;
}
return false;
}
/// <summary>Helper so a value assignment can participate in a boolean OR chain.</summary>
private static bool Assign(Action assign)
{
assign();
return true;
}
}
+3
View File
@@ -11,6 +11,7 @@ using PowderCoating.Core.Interfaces.Services;
using PowderCoating.Infrastructure.Data;
using PowderCoating.Infrastructure.Repositories;
using PowderCoating.Infrastructure.Services;
using PowderCoating.Infrastructure.Services.Columbia;
using PowderCoating.Application.Interfaces;
using PowderCoating.Application.Services;
using PowderCoating.Application.Configuration;
@@ -223,6 +224,8 @@ builder.Services.AddScoped<ICustomFormulaAiService, CustomFormulaAiService>();
builder.Services.AddScoped<IFormulaLibraryService, FormulaLibraryService>();
builder.Services.AddHttpClient();
builder.Services.AddScoped<IColumbiaCoatingsApiClient, ColumbiaCoatingsApiClient>();
builder.Services.AddScoped<IPowderCatalogUpsertService, PowderCatalogUpsertService>();
builder.Services.AddScoped<IColumbiaCatalogSyncService, ColumbiaCatalogSyncService>();
builder.Services.AddScoped<ICompanyLogoService, CompanyLogoService>();
builder.Services.AddScoped<IEquipmentManualService, EquipmentManualService>();
builder.Services.AddScoped<IPricingCalculationService, PricingCalculationService>();
@@ -0,0 +1,245 @@
using System.Text.Json;
using PowderCoating.Application.DTOs.Columbia;
using PowderCoating.Infrastructure.Services.Columbia;
namespace PowderCoating.UnitTests;
/// <summary>
/// Tests for the Columbia catalog mapper, focused on the fields that are tricky in the real feed:
/// the free-text cure parser (multiple glyphs, multi-curve, partial-cure), manufacturer derivation
/// from a multi-brand distributor, pricing across simple/variable products, and HTML stripping.
/// Cases mirror records captured from the live API.
/// </summary>
public class ColumbiaCatalogMapperTests
{
// ── Cure schedule parsing ─────────────────────────────────────────────
[Fact]
public void ParseCureCurves_SimpleSchedule_ReturnsSingleCurve()
{
var curves = ColumbiaCatalogMapper.ParseCureCurves("10 minutes @ 400°F");
Assert.Single(curves);
Assert.Equal(400, curves[0].TempF);
Assert.Equal(10, curves[0].Minutes);
}
[Fact]
public void ParseCureCurves_MetalTemperaturePrefixWithCelsius_ParsesFahrenheitOnly()
{
var curves = ColumbiaCatalogMapper.ParseCureCurves("Metal Temperature: 10 minutes at 400°F (204°C)");
Assert.Single(curves);
Assert.Equal(400, curves[0].TempF);
Assert.Equal(10, curves[0].Minutes);
}
[Theory]
[InlineData("10 minutes @ 400˚F (204˚C)")] // U+02DA ring above
[InlineData("10 minutes @ 400ºF (204ºC)")] // U+00BA masculine ordinal
[InlineData("Metal temperature: 10 minutes @ 400F (204C)")] // no degree glyph at all
public void ParseCureCurves_DegreeGlyphVariants_AllParse(string schedule)
{
var curves = ColumbiaCatalogMapper.ParseCureCurves(schedule);
Assert.Single(curves);
Assert.Equal(400, curves[0].TempF);
}
[Fact]
public void ParseCureCurves_MultiCurve_CapturesAllInOrder_PrimaryIsFirst()
{
// Real record S1790085-55: standard high-temp curve first, low-temp alternates after.
var schedule = "Metal Temperature: 5 minutes at 400°F (204°C) -or- 10 minutes at 360°F (182°C)* 15 minutes at 340°F (171°C)* *Low-Temp cure curve";
var curves = ColumbiaCatalogMapper.ParseCureCurves(schedule);
Assert.Equal(3, curves.Count);
Assert.Equal(new ColumbiaCatalogMapper.CureCurve(400, 5), curves[0]); // primary
Assert.Equal(new ColumbiaCatalogMapper.CureCurve(360, 10), curves[1]); // alternate
Assert.Equal(new ColumbiaCatalogMapper.CureCurve(340, 15), curves[2]); // alternate
}
[Fact]
public void ParseCureCurves_PartialCureInstructions_ReturnsEmpty()
{
// Illusion powder F1697027 — multi-step, no single temp/time pair.
var schedule = "(1) - Apply a basecoat and partial cure. (2) - Apply this powder and partial cure (3). - Apply a clear coat and fully cure.";
var curves = ColumbiaCatalogMapper.ParseCureCurves(schedule);
Assert.Empty(curves);
}
[Fact]
public void Map_MultiCurve_SetsPrimaryTempTimeAndStoresAllCurvesJson()
{
var product = new ColumbiaProduct
{
Sku = "S1790085-55",
Name = "Multi Cure",
CureSchedule = "5 minutes at 400°F -or- 10 minutes at 360°F",
};
var item = ColumbiaCatalogMapper.Map(product);
Assert.Equal(400m, item.CureTemperatureF);
Assert.Equal(5, item.CureTimeMinutes);
Assert.False(string.IsNullOrEmpty(item.CureCurvesJson));
Assert.Equal("5 minutes at 400°F -or- 10 minutes at 360°F", item.CureScheduleText);
var curves = JsonSerializer.Deserialize<List<ColumbiaCatalogMapper.CureCurve>>(item.CureCurvesJson!);
Assert.Equal(2, curves!.Count);
}
// ── Manufacturer derivation ───────────────────────────────────────────
[Fact]
public void DeriveManufacturer_AddSkuPrefix_IsKpPigments()
{
var p = new ColumbiaProduct { Sku = "ADD-BRBDS", Name = "Barbados Blue ColorShift Pearl" };
Assert.Equal("KP Pigments", ColumbiaCatalogMapper.DeriveManufacturer(p));
Assert.True(ColumbiaCatalogMapper.IsAdditive(p));
}
[Fact]
public void DeriveManufacturer_PpgCategory_IsPpg()
{
var p = new ColumbiaProduct
{
Sku = "PCU75139",
Name = "PPG Chrome Shadow",
Categories = { new ColumbiaNamed { Name = "PPG Powders" } },
};
Assert.Equal("PPG", ColumbiaCatalogMapper.DeriveManufacturer(p));
}
[Fact]
public void DeriveManufacturer_HouseBrand_IsColumbia()
{
var p = new ColumbiaProduct
{
Sku = "X5004124",
Name = "Blue Wrinkle",
Categories = { new ColumbiaNamed { Name = "Powders" }, new ColumbiaNamed { Name = "New Releases" } },
};
Assert.Equal("Columbia Coatings", ColumbiaCatalogMapper.DeriveManufacturer(p));
Assert.False(ColumbiaCatalogMapper.IsAdditive(p));
}
// ── Pricing ───────────────────────────────────────────────────────────
[Fact]
public void ParseBasePrice_SimpleProduct_UsesTopLevelPrice()
{
var p = new ColumbiaProduct { Price = "18.85", RegularPrice = "18.85" };
Assert.Equal(18.85m, ColumbiaCatalogMapper.ParseBasePrice(p));
}
[Fact]
public void ParseBasePrice_VariableProductWithZeroRegular_FallsBackToPriceThenVariants()
{
// Variable parent: price carries the lead variant, regular_price is "0".
var p = new ColumbiaProduct
{
Price = "18.85",
RegularPrice = "0",
VariationPricing = new List<ColumbiaVariationPricing>
{
new() { Sku = "X-B", Price = "18.85" },
new() { Sku = "X-P", Price = "18.85" },
},
};
Assert.Equal(18.85m, ColumbiaCatalogMapper.ParseBasePrice(p));
}
[Fact]
public void BuildPriceTiersJson_VariableProduct_SerializesVariationPricing()
{
var p = new ColumbiaProduct
{
VariationPricing = new List<ColumbiaVariationPricing>
{
new() { Sku = "X-B", Price = "18.85" },
},
};
var json = ColumbiaCatalogMapper.BuildPriceTiersJson(p);
Assert.NotNull(json);
Assert.Contains("X-B", json);
}
[Fact]
public void BuildPriceTiersJson_EmptyTieredPricingArray_ReturnsNull()
{
// Variable products carry tiered_pricing as an empty array.
var p = new ColumbiaProduct { TieredPricing = JsonDocument.Parse("[]").RootElement };
Assert.Null(ColumbiaCatalogMapper.BuildPriceTiersJson(p));
}
// ── Chemistry, color, HTML ────────────────────────────────────────────
[Theory]
[InlineData("Polyester/TGIC", "Polyester/TGIC")]
[InlineData("Polyester TGIC", "Polyester/TGIC")]
[InlineData("TGIC Polyester", "Polyester/TGIC")]
[InlineData("TGIC", "TGIC")]
[InlineData("", null)]
public void NormalizeChemistry_CollapsesPolyesterTgicVariants(string input, string? expected)
{
Assert.Equal(expected, ColumbiaCatalogMapper.NormalizeChemistry(input));
}
[Fact]
public void BuildColorFamilies_JoinsColorGroupNames()
{
var p = new ColumbiaProduct
{
PaColorGroup = { new ColumbiaNamed { Name = "Blue" }, new ColumbiaNamed { Name = "Green" } },
};
Assert.Equal("Blue,Green", ColumbiaCatalogMapper.BuildColorFamilies(p));
}
[Fact]
public void BuildColorFamilies_FallsBackToColorGroupAttribute()
{
var p = new ColumbiaProduct
{
Attributes =
{
new ColumbiaAttribute
{
Name = "Color Group",
Options = { new ColumbiaNamed { Name = "Black" } },
},
},
};
Assert.Equal("Black", ColumbiaCatalogMapper.BuildColorFamilies(p));
}
[Fact]
public void StripHtml_RemovesTagsEntitiesAndCollapsesWhitespace()
{
var html = "<strong>Blue Wrinkle</strong>\r\n\r\nThis is a vibrant &amp; bright coating.";
var text = ColumbiaCatalogMapper.StripHtml(html);
Assert.Equal("Blue Wrinkle This is a vibrant & bright coating.", text);
}
[Fact]
public void DetectRequiresClearCoat_IllusionDescription_IsTrue()
{
var p = new ColumbiaProduct
{
Description = "This Illusion powder requires a clear coat to activate the effect.",
};
Assert.True(ColumbiaCatalogMapper.DetectRequiresClearCoat(p));
}
// ── End-to-end mapping invariants ─────────────────────────────────────
[Fact]
public void Map_AlwaysStampsSource_AndLeavesEnrichmentFieldsNull()
{
var p = new ColumbiaProduct { Sku = "X5004124", Name = "Blue Wrinkle", Price = "18.85" };
var item = ColumbiaCatalogMapper.Map(p);
Assert.Equal("Columbia Coatings API", item.Source);
// Enrichment fields are not in the feed and must stay null for lazy TDS/AI enrichment.
Assert.Null(item.SpecificGravity);
Assert.Null(item.CoverageSqFtPerLb);
Assert.Null(item.TransferEfficiency);
}
}