Parallelize AI price check batches, increase batch size to 25

500-item catalog was making 50 sequential API calls, causing progressive rate-limit
throttling (explains "super slow towards the end") and ~$3 in credits.

- BatchSize: 10 → 25 (word limits are in place; 25 items × ~80 tokens ≈ 2000
  output tokens, well within MaxTokens=8192 — the original truncation cause)
- Run up to 3 batches concurrently via SemaphoreSlim(3) — independent API calls
  with no shared state, so no growing context issue
- For a 500-item catalog: 50 sequential calls → 20 calls in ~7 parallel waves,
  roughly 4× faster and 60% cheaper
- Dropped unused `costs` param from AnalyzeBatchAsync (system prompt has all costs)
- JS progress timing updated to reflect parallel waves

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-04-25 20:27:07 -04:00
parent 9370fcdd8f
commit 19cc03ad1c
2 changed files with 33 additions and 14 deletions
@@ -20,7 +20,8 @@ public class AiCatalogPriceCheckService : IAiCatalogPriceCheckService
private readonly ILogger<AiCatalogPriceCheckService> _logger;
private const string Model = "claude-sonnet-4-6";
private const int BatchSize = 10; // keep responses predictably short; 10 items × ~120 tokens ≈ 1200 output tokens
private const int BatchSize = 25; // 25 items × ~80 tokens (with word limits) ≈ 2000 output tokens, well within 8192
private const int MaxConcurrentBatches = 3; // cap parallel API calls to stay within rate limits
private static readonly JsonSerializerOptions JsonOpts = new() { PropertyNameCaseInsensitive = true };
@@ -98,26 +99,42 @@ public class AiCatalogPriceCheckService : IAiCatalogPriceCheckService
var client = new AnthropicClient(apiKey);
var systemPrompt = BuildSystemPrompt(costs);
var results = new List<CatalogItemPriceVerdict>();
// Process items in batches of BatchSize
for (int batchStart = 0; batchStart < items.Count; batchStart += BatchSize)
// Split into independent batches upfront
var batches = Enumerable.Range(0, (int)Math.Ceiling(items.Count / (double)BatchSize))
.Select(i => items.Skip(i * BatchSize).Take(BatchSize).ToList())
.ToList();
// Run up to MaxConcurrentBatches in parallel. Each batch is an independent API call
// with its own fresh MessageParameters — no shared state, no growing context.
var semaphore = new SemaphoreSlim(MaxConcurrentBatches, MaxConcurrentBatches);
var batchTasks = batches.Select(async (batch, index) =>
{
cancellationToken.ThrowIfCancellationRequested();
await semaphore.WaitAsync(cancellationToken);
try
{
_logger.LogInformation("Starting price check batch {Index}/{Total} ({Count} items)",
index + 1, batches.Count, batch.Count);
return await AnalyzeBatchAsync(client, systemPrompt, batch);
}
finally
{
semaphore.Release();
}
}).ToList();
var batch = items.Skip(batchStart).Take(BatchSize).ToList();
var batchResults = await AnalyzeBatchAsync(client, systemPrompt, batch, costs);
results.AddRange(batchResults);
}
var batchResults = await Task.WhenAll(batchTasks);
return results;
// Preserve original catalog order
return batches.Zip(batchResults, (batch, results) => results)
.SelectMany(r => r)
.ToList();
}
private async Task<List<CatalogItemPriceVerdict>> AnalyzeBatchAsync(
AnthropicClient client,
string systemPrompt,
List<CatalogItemForPriceCheck> batch,
ShopOperatingCostSummary costs)
List<CatalogItemForPriceCheck> batch)
{
var userPrompt = BuildUserPrompt(batch);