Parallelize AI price check batches, increase batch size to 25
500-item catalog was making 50 sequential API calls, causing progressive rate-limit throttling (explains "super slow towards the end") and ~$3 in credits. - BatchSize: 10 → 25 (word limits are in place; 25 items × ~80 tokens ≈ 2000 output tokens, well within MaxTokens=8192 — the original truncation cause) - Run up to 3 batches concurrently via SemaphoreSlim(3) — independent API calls with no shared state, so no growing context issue - For a 500-item catalog: 50 sequential calls → 20 calls in ~7 parallel waves, roughly 4× faster and 60% cheaper - Dropped unused `costs` param from AnalyzeBatchAsync (system prompt has all costs) - JS progress timing updated to reflect parallel waves Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -20,7 +20,8 @@ public class AiCatalogPriceCheckService : IAiCatalogPriceCheckService
|
||||
private readonly ILogger<AiCatalogPriceCheckService> _logger;
|
||||
|
||||
private const string Model = "claude-sonnet-4-6";
|
||||
private const int BatchSize = 10; // keep responses predictably short; 10 items × ~120 tokens ≈ 1200 output tokens
|
||||
private const int BatchSize = 25; // 25 items × ~80 tokens (with word limits) ≈ 2000 output tokens, well within 8192
|
||||
private const int MaxConcurrentBatches = 3; // cap parallel API calls to stay within rate limits
|
||||
|
||||
private static readonly JsonSerializerOptions JsonOpts = new() { PropertyNameCaseInsensitive = true };
|
||||
|
||||
@@ -98,26 +99,42 @@ public class AiCatalogPriceCheckService : IAiCatalogPriceCheckService
|
||||
|
||||
var client = new AnthropicClient(apiKey);
|
||||
var systemPrompt = BuildSystemPrompt(costs);
|
||||
var results = new List<CatalogItemPriceVerdict>();
|
||||
|
||||
// Process items in batches of BatchSize
|
||||
for (int batchStart = 0; batchStart < items.Count; batchStart += BatchSize)
|
||||
// Split into independent batches upfront
|
||||
var batches = Enumerable.Range(0, (int)Math.Ceiling(items.Count / (double)BatchSize))
|
||||
.Select(i => items.Skip(i * BatchSize).Take(BatchSize).ToList())
|
||||
.ToList();
|
||||
|
||||
// Run up to MaxConcurrentBatches in parallel. Each batch is an independent API call
|
||||
// with its own fresh MessageParameters — no shared state, no growing context.
|
||||
var semaphore = new SemaphoreSlim(MaxConcurrentBatches, MaxConcurrentBatches);
|
||||
var batchTasks = batches.Select(async (batch, index) =>
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
await semaphore.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("Starting price check batch {Index}/{Total} ({Count} items)",
|
||||
index + 1, batches.Count, batch.Count);
|
||||
return await AnalyzeBatchAsync(client, systemPrompt, batch);
|
||||
}
|
||||
finally
|
||||
{
|
||||
semaphore.Release();
|
||||
}
|
||||
}).ToList();
|
||||
|
||||
var batch = items.Skip(batchStart).Take(BatchSize).ToList();
|
||||
var batchResults = await AnalyzeBatchAsync(client, systemPrompt, batch, costs);
|
||||
results.AddRange(batchResults);
|
||||
}
|
||||
var batchResults = await Task.WhenAll(batchTasks);
|
||||
|
||||
return results;
|
||||
// Preserve original catalog order
|
||||
return batches.Zip(batchResults, (batch, results) => results)
|
||||
.SelectMany(r => r)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private async Task<List<CatalogItemPriceVerdict>> AnalyzeBatchAsync(
|
||||
AnthropicClient client,
|
||||
string systemPrompt,
|
||||
List<CatalogItemForPriceCheck> batch,
|
||||
ShopOperatingCostSummary costs)
|
||||
List<CatalogItemForPriceCheck> batch)
|
||||
{
|
||||
var userPrompt = BuildUserPrompt(batch);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user