Merge branch 'dev'
This commit is contained in:
@@ -142,7 +142,36 @@
|
||||
"PowerShell(dotnet build *)",
|
||||
"PowerShell(New-Item *)",
|
||||
"PowerShell(& \"Y:\\\\PCC\\\\PowderCoatingApp\\\\scripts\\\\generate-migration-script.ps1\")",
|
||||
"PowerShell(if \\(Test-Path \"Y:\\\\pcc\\\\deployment\\\\migrations.sql\"\\) { $f = Get-Item \"Y:\\\\pcc\\\\deployment\\\\migrations.sql\"; Write-Host \"File exists: $\\($f.Length\\) bytes\" } else { Write-Host \"File not created\" })"
|
||||
"PowerShell(if \\(Test-Path \"Y:\\\\pcc\\\\deployment\\\\migrations.sql\"\\) { $f = Get-Item \"Y:\\\\pcc\\\\deployment\\\\migrations.sql\"; Write-Host \"File exists: $\\($f.Length\\) bytes\" } else { Write-Host \"File not created\" })",
|
||||
"Bash(git add *)",
|
||||
"Bash(git commit -m ' *)",
|
||||
"Bash(git push *)",
|
||||
"Bash(git commit *)",
|
||||
"Bash(git checkout *)",
|
||||
"Bash(git merge *)",
|
||||
"Bash(dotnet package *)",
|
||||
"Bash(dotnet test *)",
|
||||
"Bash(git rm *)",
|
||||
"Bash(git stash *)",
|
||||
"Bash(dotnet ef *)",
|
||||
"Bash(sqlcmd -S \".\\\\SQLEXPRESS\" -d PowderCoatingDb -Q \"SELECT Id, DisplayName, IsCoating, IsActive FROM InventoryCategoryLookups ORDER BY DisplayOrder\" -W)",
|
||||
"Skill(schedule)",
|
||||
"Bash(git -C \"//192.168.0.37/SCPSoftware/tmp/PowderCoatingApp-dev-perf\" log --oneline -10)",
|
||||
"Bash(git -C \"//192.168.0.37/SCPSoftware/tmp/PowderCoatingApp-dev-perf\" status --short)",
|
||||
"Bash(git *)",
|
||||
"Bash(get-childitem -Recurse -Filter \"QuotesController.cs\")",
|
||||
"Bash(Select-Object -ExpandProperty FullName)",
|
||||
"Bash(dotnet user-secrets *)",
|
||||
"Bash(Get-ChildItem -Path \"Y:\\\\PCC\\\\PowderCoatingApp\" -Directory)",
|
||||
"Bash(Select-Object Name)",
|
||||
"Bash(Get-Content *)",
|
||||
"Bash(python -c \"import json; data=json.load\\(open\\('prismatic_powders.json','r',encoding='utf-8'\\)\\); print\\(f'Total records: {len\\(data\\)}'\\); print\\('First record:'\\); print\\(json.dumps\\(data[0], indent=2\\)\\)\")",
|
||||
"Bash(python -c \"import json; data=json.load\\(open\\('prismatic_powders.json','r',encoding='utf-8'\\)\\); keys=list\\(data.keys\\(\\)\\); print\\('Top-level keys:', keys[:10]\\); first=data[keys[0]]; print\\('First record key:', keys[0]\\); print\\(json.dumps\\(first, indent=2\\)\\)\")",
|
||||
"PowerShell(Get-ChildItem *)",
|
||||
"PowerShell(Select-String *)",
|
||||
"Bash(Select-Object -First 20)",
|
||||
"PowerShell(node -e \"require\\('fs'\\).existsSync\\(require\\('path'\\).join\\(process.cwd\\(\\), 'node_modules', 'sharp'\\)\\) ? console.log\\('sharp ok'\\) : console.log\\('no sharp'\\)\")",
|
||||
"WebFetch(domain:www.powdercoatinglogix.com)"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+2
-6
@@ -6,10 +6,6 @@ pipeline {
|
||||
timestamps()
|
||||
}
|
||||
|
||||
environment {
|
||||
PATH = "C:\\Program Files\\Microsoft SDKs\\Azure\\CLI2\\wbin;${env.PATH}"
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Checkout') {
|
||||
steps {
|
||||
@@ -35,13 +31,13 @@ pipeline {
|
||||
|
||||
stage('Publish') {
|
||||
steps {
|
||||
bat 'dotnet publish src\\PowderCoating.Web\\PowderCoating.Web.csproj -c Release -r linux-x64 --self-contained false -o publish'
|
||||
bat 'dotnet publish src\\PowderCoating.Web\\PowderCoating.Web.csproj -c Release --no-build -o publish'
|
||||
}
|
||||
}
|
||||
|
||||
stage('Deploy to Azure') {
|
||||
steps {
|
||||
bat 'powershell -Command "Add-Type -Assembly System.IO.Compression.FileSystem; if (Test-Path deploy.zip) { Remove-Item deploy.zip }; [System.IO.Compression.ZipFile]::CreateFromDirectory(\'publish\', \'deploy.zip\')"'
|
||||
bat 'powershell -Command "Compress-Archive -Path publish\\* -DestinationPath deploy.zip -Force"'
|
||||
withCredentials([azureServicePrincipal(
|
||||
credentialsId: 'azure-pcl',
|
||||
subscriptionIdVariable: 'AZ_SUB_ID',
|
||||
|
||||
@@ -1,15 +1,7 @@
|
||||
Shop Management App TO DO List
|
||||
==============================
|
||||
-Look into possibly having AI scan a product catalog and suggest prices for items.
|
||||
-Add images to product catalog items for easily identification of parts
|
||||
-AI Company Lookup (similar to inventory lookup)
|
||||
|
||||
|
||||
-Add ability to save a quoted item to the product catalog either from an AI Photo Quote or from the calculated item
|
||||
|
||||
-Google review request email after a job
|
||||
-Check my ChatGPT chat about surface area for a few solid ideas for the system
|
||||
|
||||
-Add SMS capabilities
|
||||
-Fix up approve/decline messages between customer and user on quote approval feature
|
||||
|
||||
Done and need testing
|
||||
@@ -178,7 +170,17 @@ AI Agent item where we upload a picture and it will calculate the approximate sq
|
||||
-Make sure we're tracking logins. I see a user logged on, but the company health page states they have never logged in.
|
||||
-Allow printing blank work orders (model after the SCP Powder Coating blank work order)
|
||||
-IDEA: Print powders to use on work order with their QR code so they can be scanned right from there and usage recorded.
|
||||
|
||||
-Add ability to save a quoted item to the product catalog either from an AI Photo Quote or from the calculated item
|
||||
-Add images to product catalog items for easily identification of parts
|
||||
-Look into possibly having AI scan a product catalog and suggest prices for items.
|
||||
-Add Oven and Add Blasting Setup don't work in Setup Wizard
|
||||
-When scanning inventory QR Code, there is no cancel button
|
||||
-Bug: When scanning Inventory QR Code, if not logged in...it takes you to the dashboard after login, not our inventory scanning screen
|
||||
-Add SMS capabilities
|
||||
-Lookup not working 100% correct. If I type columbia as the manufacturer and a color name....it's finding blackmamba from prismatic incorrectly.
|
||||
-Lookup Modal not showing ALL matches. Maybe make scrollable
|
||||
-Pickup cure information from TDS Sheet if not found by AI Search
|
||||
-ON AI Photo Quote page, when the AI info comes back we should scroll the modal window down so it's visible. It's not clear that new info has been added to the modal for all customers
|
||||
|
||||
Ideas Removed
|
||||
=======================
|
||||
|
||||
+12
-4
@@ -1,9 +1,11 @@
|
||||
Shop Management App TO DO List
|
||||
==============================
|
||||
-Add ability to save a quoted item to the product catalog either from an AI Photo Quote or from the calculated item
|
||||
-Lookup not working 100% correct. If I type columbia as the manufacturer and a color name....it's finding blackmamba from prismatic incorrectly.
|
||||
-Lookup Modal not showing ALL matches. Maybe make scrollable
|
||||
-Pickup cure information from TDS Sheet if not found by AI Search
|
||||
-ON AI Photo Quote page, when the AI info comes back we should scroll the modal window down so it's visible. It's not clear that new info has been added to the modal for all customers
|
||||
-Google review request email after a job
|
||||
-Check my ChatGPT chat about surface area for a few solid ideas for the system
|
||||
|
||||
-Add SMS capabilities
|
||||
-Fix up approve/decline messages between customer and user on quote approval feature
|
||||
|
||||
Done and need testing
|
||||
@@ -172,7 +174,13 @@ AI Agent item where we upload a picture and it will calculate the approximate sq
|
||||
-Make sure we're tracking logins. I see a user logged on, but the company health page states they have never logged in.
|
||||
-Allow printing blank work orders (model after the SCP Powder Coating blank work order)
|
||||
-IDEA: Print powders to use on work order with their QR code so they can be scanned right from there and usage recorded.
|
||||
|
||||
-Add ability to save a quoted item to the product catalog either from an AI Photo Quote or from the calculated item
|
||||
-Add images to product catalog items for easily identification of parts
|
||||
-Look into possibly having AI scan a product catalog and suggest prices for items.
|
||||
-Add Oven and Add Blasting Setup don't work in Setup Wizard
|
||||
-When scanning inventory QR Code, there is no cancel button
|
||||
-Bug: When scanning Inventory QR Code, if not logged in...it takes you to the dashboard after login, not our inventory scanning screen
|
||||
-Add SMS capabilities
|
||||
|
||||
Ideas Removed
|
||||
=======================
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
# Guided Activation Flow Feature Spec
|
||||
|
||||
## Overview
|
||||
|
||||
This feature introduces a **post-setup guided activation flow** for new companies.
|
||||
|
||||
After completing the setup wizard, users should be guided through their **first real workflow** so they understand how to use the system immediately.
|
||||
|
||||
This is NOT a tooltip tour.
|
||||
|
||||
This is a **guided outcome flow using real system actions** (quotes, jobs, invoices).
|
||||
|
||||
---
|
||||
|
||||
## Problem
|
||||
|
||||
Current behavior:
|
||||
- Users complete setup wizard
|
||||
- Land on dashboard
|
||||
- Do not create quotes, jobs, or invoices
|
||||
- Drop off
|
||||
|
||||
Goal:
|
||||
- Ensure users complete at least ONE real workflow
|
||||
- Create an "aha moment" within first session
|
||||
|
||||
---
|
||||
|
||||
## Business Workflows
|
||||
|
||||
### 1. Quote-First Workflow
|
||||
- Create Quote
|
||||
- Send to customer
|
||||
- Convert Quote → Job
|
||||
- Process Job
|
||||
- Create Invoice
|
||||
- Customer Pays
|
||||
|
||||
### 2. Job-First Workflow (Walk-in)
|
||||
- Create Job directly
|
||||
- Process Job
|
||||
- Create Invoice
|
||||
- Customer Pays
|
||||
|
||||
---
|
||||
|
||||
## Feature Behavior
|
||||
|
||||
### Trigger Condition
|
||||
|
||||
IF:
|
||||
- setup wizard is completed
|
||||
- AND firstWorkflowCompleted == false
|
||||
|
||||
THEN:
|
||||
→ redirect user to guided activation flow
|
||||
|
||||
---
|
||||
|
||||
## Step 1: Workflow Selection
|
||||
|
||||
Display full-screen page:
|
||||
|
||||
### Title:
|
||||
"Your shop is set up. Let’s run your first workflow."
|
||||
|
||||
### Subtitle:
|
||||
"Choose how jobs usually start for your shop and we’ll guide you through it."
|
||||
|
||||
### Question:
|
||||
"How do jobs usually start for your shop?"
|
||||
|
||||
### Options:
|
||||
|
||||
#### Option A:
|
||||
Title: "I send a quote first"
|
||||
Description: "Create a quote, convert it to a job, then invoice when work is complete."
|
||||
|
||||
#### Option B:
|
||||
Title: "I start with a job"
|
||||
Description: "For walk-ins or approved work where you start immediately."
|
||||
|
||||
---
|
||||
|
||||
### On Selection:
|
||||
Save:
|
||||
- onboardingPath = "quote_first" | "job_first"
|
||||
|
||||
Then continue into guided flow
|
||||
|
||||
---
|
||||
|
||||
## Step 2: Guided Flow
|
||||
|
||||
### Path A — Quote First
|
||||
|
||||
#### Step A1: Create Quote
|
||||
- Use existing quote creation logic
|
||||
- Pre-fill fields:
|
||||
- Customer: "Sample Customer"
|
||||
- Item: "Wheel Set"
|
||||
- Quantity: 4
|
||||
- Notes: "Sample onboarding quote"
|
||||
- Allow editing before submit
|
||||
|
||||
#### Step A2: Show Quote Created
|
||||
Message:
|
||||
"This is the quote you would send to your customer."
|
||||
|
||||
CTA:
|
||||
"Convert to Job"
|
||||
|
||||
#### Step A3: Convert Quote → Job
|
||||
- Use existing conversion logic
|
||||
|
||||
#### Step A4: Show Job
|
||||
Message:
|
||||
"This job is now tracked in your workflow."
|
||||
|
||||
CTA:
|
||||
"Create Invoice" (if supported)
|
||||
|
||||
#### Step A5: Create Invoice (optional)
|
||||
- Use existing invoice logic
|
||||
|
||||
#### Completion:
|
||||
Set:
|
||||
- firstWorkflowCompleted = true
|
||||
|
||||
---
|
||||
|
||||
### Path B — Job First
|
||||
|
||||
#### Step B1: Create Job
|
||||
- Use existing job creation logic
|
||||
- Pre-fill:
|
||||
- Customer: "Walk-in Customer"
|
||||
- Item: "Wheel Set"
|
||||
- Quantity: 4
|
||||
- Notes: "Sample onboarding job"
|
||||
|
||||
#### Step B2: Show Job
|
||||
Message:
|
||||
"This job is now in your workflow."
|
||||
|
||||
CTA:
|
||||
"Create Invoice" (optional)
|
||||
|
||||
#### Step B3: Create Invoice (optional)
|
||||
|
||||
#### Completion:
|
||||
Set:
|
||||
- firstWorkflowCompleted = true
|
||||
|
||||
---
|
||||
|
||||
## Skipping
|
||||
|
||||
Provide "Skip for now" option.
|
||||
|
||||
If skipped:
|
||||
- DO NOT set firstWorkflowCompleted
|
||||
- Redirect to dashboard
|
||||
- Continue showing activation banner
|
||||
|
||||
---
|
||||
|
||||
## Dashboard Behavior
|
||||
|
||||
If:
|
||||
- setup complete
|
||||
- AND firstWorkflowCompleted == false
|
||||
|
||||
Show persistent banner:
|
||||
|
||||
Title:
|
||||
"Create your first job or quote"
|
||||
|
||||
Text:
|
||||
"Run a quick 2-minute workflow to see how the system works."
|
||||
|
||||
CTA:
|
||||
"Start first workflow"
|
||||
|
||||
---
|
||||
|
||||
## Data Model Changes
|
||||
|
||||
Add to Company or User:
|
||||
|
||||
- onboardingPath: string | null
|
||||
- firstWorkflowCompleted: boolean
|
||||
|
||||
Optional:
|
||||
- firstQuoteCreatedAt: datetime
|
||||
- firstJobCreatedAt: datetime
|
||||
- firstInvoiceCreatedAt: datetime
|
||||
|
||||
---
|
||||
|
||||
## Events / Tracking (if system exists)
|
||||
|
||||
Track:
|
||||
- onboarding_path_selected
|
||||
- first_quote_created
|
||||
- first_job_created
|
||||
- first_invoice_created
|
||||
- first_workflow_completed
|
||||
- first_workflow_skipped
|
||||
|
||||
---
|
||||
|
||||
## Implementation Constraints
|
||||
|
||||
- MUST reuse existing quote/job/invoice logic
|
||||
- DO NOT duplicate business logic
|
||||
- DO NOT create separate fake systems
|
||||
- Use existing forms and APIs where possible
|
||||
- Keep UI minimal and fast
|
||||
- Pre-fill as much as possible
|
||||
|
||||
---
|
||||
|
||||
## UX Requirements
|
||||
|
||||
- No tooltip tours
|
||||
- Linear guided flow only
|
||||
- One action at a time
|
||||
- Minimize user effort
|
||||
- Show immediate visual feedback
|
||||
|
||||
---
|
||||
|
||||
## Developer Instructions
|
||||
|
||||
Before coding:
|
||||
1. Inspect setup wizard completion logic
|
||||
2. Identify routing after setup
|
||||
3. Identify quote/job/invoice creation flows
|
||||
4. Identify data model structure
|
||||
|
||||
Then:
|
||||
5. Propose implementation plan
|
||||
6. Wait for approval
|
||||
7. Implement incrementally
|
||||
8. Summarize changes
|
||||
9. Provide manual QA steps
|
||||
|
||||
---
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- % of users creating first job increases significantly
|
||||
- Users complete at least one workflow during onboarding
|
||||
- Reduced drop-off after setup wizard
|
||||
|
||||
Target:
|
||||
≥ 30% of new users create at least one job or quote
|
||||
@@ -0,0 +1,173 @@
|
||||
Add a dashboard progress widget for post-onboarding activation.
|
||||
|
||||
Context:
|
||||
This is a powder coating shop management app. We recently shortened the setup wizard and added a guided activation flow. Some setup items are intentionally deferred so users can evaluate the system quickly before fully configuring everything.
|
||||
|
||||
Goal:
|
||||
Create a dashboard widget that helps users “get the most out of their shop” without making it feel like unfinished homework.
|
||||
|
||||
Do NOT call it “Complete setup.”
|
||||
|
||||
Recommended title:
|
||||
“Get the most out of your shop”
|
||||
|
||||
Purpose:
|
||||
Show progress based on real usage/configuration milestones and give users clear next actions.
|
||||
|
||||
Requirements:
|
||||
|
||||
1. Inspect existing dashboard structure
|
||||
|
||||
* Locate the dashboard controller/view/components.
|
||||
* Reuse existing card, alert, progress bar, and button styles.
|
||||
* Follow existing UI conventions.
|
||||
|
||||
2. Widget visibility
|
||||
Show the widget for companies that:
|
||||
|
||||
* Have completed the setup wizard
|
||||
* Are not yet meaningfully activated OR still have recommended setup tasks incomplete
|
||||
|
||||
It is okay to keep showing it until all tasks are complete.
|
||||
|
||||
3. Progress calculation
|
||||
Create a checklist of 5–6 items max.
|
||||
|
||||
Suggested items:
|
||||
|
||||
A. Create your first quote or job
|
||||
Complete when:
|
||||
|
||||
* company has at least one quote OR at least one job
|
||||
|
||||
CTA:
|
||||
|
||||
* “Create quote/job” or “Start workflow”
|
||||
|
||||
B. Move a job through your workflow
|
||||
Complete when:
|
||||
|
||||
* at least one job has had a status/stage change
|
||||
* If there is no existing way to detect this, use the closest available activity/history/status timestamp
|
||||
|
||||
CTA:
|
||||
|
||||
* “Open daily board”
|
||||
|
||||
C. Create your first invoice
|
||||
Complete when:
|
||||
|
||||
* company has at least one invoice
|
||||
|
||||
CTA:
|
||||
|
||||
* “Create invoice”
|
||||
|
||||
D. Invite your team
|
||||
Complete when:
|
||||
|
||||
* company has more than one active user/team member
|
||||
|
||||
CTA:
|
||||
|
||||
* “Invite team”
|
||||
|
||||
E. Customize pricing
|
||||
Complete when:
|
||||
|
||||
* company has configured pricing tiers/custom pricing settings beyond defaults
|
||||
* If this is hard to detect reliably, make this optional or use a simple existing flag/count
|
||||
|
||||
CTA:
|
||||
|
||||
* “Customize pricing”
|
||||
|
||||
F. Review payment terms
|
||||
Complete when:
|
||||
|
||||
* company has customized payment terms from default
|
||||
* If this is hard to detect reliably, make this optional or use a simple existing flag/value comparison
|
||||
|
||||
CTA:
|
||||
|
||||
* “Review terms”
|
||||
|
||||
4. UX copy
|
||||
Use friendly, value-focused language.
|
||||
|
||||
Widget title:
|
||||
“Get the most out of your shop”
|
||||
|
||||
Subtitle:
|
||||
“Complete a few quick steps to unlock the full workflow.”
|
||||
|
||||
Progress text:
|
||||
“X of Y complete”
|
||||
|
||||
Avoid wording like:
|
||||
|
||||
* “Incomplete setup”
|
||||
* “Missing configuration”
|
||||
* “Required steps”
|
||||
|
||||
5. Visual design
|
||||
|
||||
* Use a card-style widget near the top of the dashboard.
|
||||
* Include a progress bar.
|
||||
* Show checklist rows with completed and incomplete states.
|
||||
* Completed items should feel rewarding.
|
||||
* Incomplete items should have one clear CTA.
|
||||
* Keep it compact and non-annoying.
|
||||
|
||||
6. Behavior
|
||||
|
||||
* Each checklist item should link to the most relevant existing page or action.
|
||||
* Do not build new duplicate workflows.
|
||||
* Reuse existing guided activation route for “Create your first quote or job” if available.
|
||||
* If a task cannot be detected reliably yet, implement it conservatively or leave a TODO comment explaining why.
|
||||
|
||||
7. Data/query logic
|
||||
|
||||
* Prefer calculating progress server-side in the dashboard view model.
|
||||
* Avoid expensive queries.
|
||||
* Reuse existing repositories/services if available.
|
||||
* Keep the logic readable and testable.
|
||||
|
||||
8. Dismissal behavior
|
||||
Add optional dismissal if easy:
|
||||
|
||||
* Let user collapse or dismiss the widget.
|
||||
* If dismissed, do not permanently hide it forever unless all tasks are complete.
|
||||
* Prefer “collapse” over full dismissal.
|
||||
* Store dismissal/collapse state only if there is already a simple place to store dashboard preferences.
|
||||
|
||||
9. Important product guidance
|
||||
This widget should guide users from evaluation into real adoption.
|
||||
|
||||
The emotional framing should be:
|
||||
“You’re already making progress — here are the next valuable things to try.”
|
||||
|
||||
Not:
|
||||
“You failed to finish setup.”
|
||||
|
||||
10. Implementation style
|
||||
Before coding:
|
||||
|
||||
* Inspect relevant dashboard, setup wizard, guided activation, company preference, quote, job, invoice, user/team, pricing, and payment term structures.
|
||||
* Propose a concise implementation plan.
|
||||
* Then implement incrementally.
|
||||
|
||||
After coding:
|
||||
|
||||
* Summarize changed files.
|
||||
* Explain how progress is calculated.
|
||||
* Provide manual QA steps.
|
||||
|
||||
Manual QA scenarios:
|
||||
|
||||
* Brand new company after setup wizard
|
||||
* Company with first quote/job created
|
||||
* Company with moved job/status change
|
||||
* Company with invoice created
|
||||
* Company with invited team member
|
||||
* Company with all tasks complete
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,49 @@
|
||||
BEGIN TRANSACTION;
|
||||
GO
|
||||
|
||||
ALTER TABLE [CompanyPreferences] ADD [FirstInvoiceCreatedAt] datetime2 NULL;
|
||||
GO
|
||||
|
||||
ALTER TABLE [CompanyPreferences] ADD [FirstJobCreatedAt] datetime2 NULL;
|
||||
GO
|
||||
|
||||
ALTER TABLE [CompanyPreferences] ADD [FirstQuoteCreatedAt] datetime2 NULL;
|
||||
GO
|
||||
|
||||
ALTER TABLE [CompanyPreferences] ADD [FirstWorkflowCompleted] bit NOT NULL DEFAULT CAST(0 AS bit);
|
||||
GO
|
||||
|
||||
ALTER TABLE [CompanyPreferences] ADD [FirstWorkflowCompletedAt] datetime2 NULL;
|
||||
GO
|
||||
|
||||
ALTER TABLE [CompanyPreferences] ADD [GuidedActivationDismissedAt] datetime2 NULL;
|
||||
GO
|
||||
|
||||
ALTER TABLE [CompanyPreferences] ADD [OnboardingPath] nvarchar(max) NULL;
|
||||
GO
|
||||
|
||||
UPDATE [PricingTiers] SET [CreatedAt] = '2026-04-28T16:40:22.3595055Z'
|
||||
WHERE [Id] = 1;
|
||||
SELECT @@ROWCOUNT;
|
||||
|
||||
GO
|
||||
|
||||
UPDATE [PricingTiers] SET [CreatedAt] = '2026-04-28T16:40:22.3595063Z'
|
||||
WHERE [Id] = 2;
|
||||
SELECT @@ROWCOUNT;
|
||||
|
||||
GO
|
||||
|
||||
UPDATE [PricingTiers] SET [CreatedAt] = '2026-04-28T16:40:22.3595065Z'
|
||||
WHERE [Id] = 3;
|
||||
SELECT @@ROWCOUNT;
|
||||
|
||||
GO
|
||||
|
||||
INSERT INTO [__EFMigrationsHistory] ([MigrationId], [ProductVersion])
|
||||
VALUES (N'20260428164026_AddGuidedActivationFields', N'8.0.11');
|
||||
GO
|
||||
|
||||
COMMIT;
|
||||
GO
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,319 @@
|
||||
# Discover-Prismatic-Product-Urls-By-ColorParam.ps1
|
||||
#
|
||||
# Discovers Prismatic Powders product URLs by visiting color filter URLs like:
|
||||
# https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_red
|
||||
#
|
||||
# Outputs:
|
||||
# .\product-urls.txt
|
||||
# .\color-discovery-log.json
|
||||
#
|
||||
# First-time setup:
|
||||
# Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass
|
||||
# .\Discover-Prismatic-Product-Urls-By-ColorParam.ps1 -InstallPlaywright -Headed
|
||||
#
|
||||
# Normal run:
|
||||
# .\Discover-Prismatic-Product-Urls-By-ColorParam.ps1
|
||||
#
|
||||
# Watch browser:
|
||||
# .\Discover-Prismatic-Product-Urls-By-ColorParam.ps1 -Headed
|
||||
|
||||
param(
|
||||
[switch]$InstallPlaywright,
|
||||
[switch]$Headed,
|
||||
[int]$MaxScrollsPerColor = 180,
|
||||
[int]$StopAfterNoNewScrolls = 10
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Ensure-NodeAvailable {
|
||||
if (-not (Get-Command node -ErrorAction SilentlyContinue)) {
|
||||
throw "Node.js is required. Install Node.js LTS from https://nodejs.org/"
|
||||
}
|
||||
|
||||
if (-not (Get-Command npm -ErrorAction SilentlyContinue)) {
|
||||
throw "npm is required. It usually comes with Node.js."
|
||||
}
|
||||
}
|
||||
|
||||
function Install-PlaywrightIfNeeded {
|
||||
param([bool]$Requested)
|
||||
|
||||
Ensure-NodeAvailable
|
||||
|
||||
if ($Requested -or -not (Test-Path ".\node_modules\playwright")) {
|
||||
Write-Host "Installing Playwright package locally..."
|
||||
npm init -y | Out-Null
|
||||
npm install playwright | Out-Null
|
||||
|
||||
Write-Host "Installing Playwright Chromium browser..."
|
||||
npx playwright install chromium
|
||||
}
|
||||
}
|
||||
|
||||
function Write-NodeDiscoveryScript {
|
||||
$js = @'
|
||||
const fs = require("fs");
|
||||
const { chromium } = require("playwright");
|
||||
|
||||
const headed = process.argv.includes("--headed");
|
||||
|
||||
function getArgValue(name, defaultValue) {
|
||||
const prefix = `--${name}=`;
|
||||
const found = process.argv.find(x => x.startsWith(prefix));
|
||||
return found ? found.slice(prefix.length) : defaultValue;
|
||||
}
|
||||
|
||||
const maxScrollsPerColor = parseInt(getArgValue("max-scrolls-per-color", "180"), 10);
|
||||
const stopAfterNoNewScrolls = parseInt(getArgValue("stop-after-no-new-scrolls", "10"), 10);
|
||||
|
||||
const baseUrl = "https://www.prismaticpowders.com/shop/powder-coating-colors";
|
||||
const outputFile = "product-urls.txt";
|
||||
const logFile = "color-discovery-log.json";
|
||||
|
||||
// Update this list if you find more color params in the site HTML.
|
||||
const colorParams = [
|
||||
"pris_black",
|
||||
"pris_blue",
|
||||
"pris_bronze",
|
||||
"pris_brown",
|
||||
"pris_clear",
|
||||
"pris_copper",
|
||||
"pris_gold",
|
||||
"pris_gray",
|
||||
"pris_green",
|
||||
"pris_orange",
|
||||
"pris_pink",
|
||||
"pris_purple",
|
||||
"pris_red",
|
||||
"pris_silver",
|
||||
"pris_tan",
|
||||
"pris_white",
|
||||
"pris_yellow"
|
||||
];
|
||||
|
||||
function cleanUrl(url) {
|
||||
return (url || "").split("?")[0].split("#")[0].trim();
|
||||
}
|
||||
|
||||
function isProductUrl(url) {
|
||||
return /\/shop\/powder-coating-colors\/[A-Z0-9-]+\//i.test(url || "");
|
||||
}
|
||||
|
||||
function readExistingUrls() {
|
||||
if (!fs.existsSync(outputFile)) return [];
|
||||
|
||||
return fs.readFileSync(outputFile, "utf8")
|
||||
.split(/\r?\n/)
|
||||
.map(cleanUrl)
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function writeUrls(urls) {
|
||||
const sorted = [...urls].sort();
|
||||
fs.writeFileSync(outputFile, sorted.join("\r\n") + "\r\n", "utf8");
|
||||
}
|
||||
|
||||
function readLog() {
|
||||
if (!fs.existsSync(logFile)) {
|
||||
return {
|
||||
completed_colors: {},
|
||||
runs: []
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(logFile, "utf8"));
|
||||
} catch {
|
||||
return {
|
||||
completed_colors: {},
|
||||
runs: []
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function writeLog(log) {
|
||||
fs.writeFileSync(logFile, JSON.stringify(log, null, 2), "utf8");
|
||||
}
|
||||
|
||||
async function collectProductLinks(page) {
|
||||
const links = await page.locator("a").evaluateAll(anchors =>
|
||||
anchors
|
||||
.map(a => a.href)
|
||||
.filter(Boolean)
|
||||
.filter(h => /\/shop\/powder-coating-colors\/[A-Z0-9-]+\//i.test(h))
|
||||
);
|
||||
|
||||
return links.map(cleanUrl).filter(Boolean);
|
||||
}
|
||||
|
||||
async function scrollAndCollect(page, urls, label) {
|
||||
let noNewScrolls = 0;
|
||||
let totalAddedForThisColor = 0;
|
||||
|
||||
for (let i = 0; i < maxScrollsPerColor; i++) {
|
||||
const before = urls.size;
|
||||
|
||||
for (const link of await collectProductLinks(page)) {
|
||||
urls.add(link);
|
||||
}
|
||||
|
||||
const after = urls.size;
|
||||
const added = after - before;
|
||||
totalAddedForThisColor += added;
|
||||
|
||||
if (added === 0) {
|
||||
noNewScrolls++;
|
||||
} else {
|
||||
noNewScrolls = 0;
|
||||
}
|
||||
|
||||
writeUrls(urls);
|
||||
|
||||
console.log(`[${label}] Scroll ${i + 1}/${maxScrollsPerColor}: +${added}, total ${after}, no-new ${noNewScrolls}`);
|
||||
|
||||
if (noNewScrolls >= stopAfterNoNewScrolls) {
|
||||
break;
|
||||
}
|
||||
|
||||
await page.mouse.wheel(0, 2500);
|
||||
await page.waitForTimeout(1500);
|
||||
}
|
||||
|
||||
return totalAddedForThisColor;
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const existingUrls = readExistingUrls();
|
||||
const urls = new Set(existingUrls);
|
||||
const log = readLog();
|
||||
|
||||
console.log(`Existing URLs in ${outputFile}: ${existingUrls.length}`);
|
||||
|
||||
const browser = await chromium.launch({ headless: !headed });
|
||||
|
||||
const context = await browser.newContext({
|
||||
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
viewport: { width: 1365, height: 900 },
|
||||
locale: "en-US",
|
||||
timezoneId: "America/New_York"
|
||||
});
|
||||
|
||||
const page = await context.newPage();
|
||||
|
||||
const runRecord = {
|
||||
started_at: new Date().toISOString(),
|
||||
existing_at_start: existingUrls.length,
|
||||
colors_attempted: []
|
||||
};
|
||||
|
||||
for (const color of colorParams) {
|
||||
if (log.completed_colors[color]) {
|
||||
console.log(`Skipping completed color: ${color}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const url = `${baseUrl}?color=${encodeURIComponent(color)}`;
|
||||
console.log("");
|
||||
console.log(`Opening color filter: ${color}`);
|
||||
console.log(url);
|
||||
|
||||
try {
|
||||
const response = await page.goto(url, {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: 60000
|
||||
});
|
||||
|
||||
const status = response ? response.status() : "unknown";
|
||||
console.log(`HTTP status: ${status}`);
|
||||
|
||||
await page.waitForTimeout(5000);
|
||||
|
||||
const before = urls.size;
|
||||
const addedDuringScroll = await scrollAndCollect(page, urls, color);
|
||||
const after = urls.size;
|
||||
const netAdded = after - before;
|
||||
|
||||
log.completed_colors[color] = {
|
||||
url,
|
||||
http_status: status,
|
||||
added: netAdded,
|
||||
added_during_scroll: addedDuringScroll,
|
||||
total_after: after,
|
||||
completed_at: new Date().toISOString()
|
||||
};
|
||||
|
||||
runRecord.colors_attempted.push({
|
||||
color,
|
||||
url,
|
||||
http_status: status,
|
||||
added: netAdded,
|
||||
total_after: after
|
||||
});
|
||||
|
||||
writeLog(log);
|
||||
writeUrls(urls);
|
||||
|
||||
console.log(`Color complete: ${color}; added ${netAdded}; total ${after}`);
|
||||
|
||||
// Polite pause between filters.
|
||||
await page.waitForTimeout(3000);
|
||||
} catch (err) {
|
||||
console.log(`Color failed: ${color}; ${err.message}`);
|
||||
|
||||
runRecord.colors_attempted.push({
|
||||
color,
|
||||
url,
|
||||
added: 0,
|
||||
error: err.message
|
||||
});
|
||||
|
||||
writeLog(log);
|
||||
}
|
||||
}
|
||||
|
||||
runRecord.finished_at = new Date().toISOString();
|
||||
runRecord.final_total = urls.size;
|
||||
runRecord.new_this_run = urls.size - existingUrls.length;
|
||||
|
||||
log.runs.push(runRecord);
|
||||
writeLog(log);
|
||||
writeUrls(urls);
|
||||
|
||||
console.log("");
|
||||
console.log("Color-param discovery complete.");
|
||||
console.log(`Existing at start: ${existingUrls.length}`);
|
||||
console.log(`Final total: ${urls.size}`);
|
||||
console.log(`New this run: ${urls.size - existingUrls.length}`);
|
||||
console.log(`Output: ${outputFile}`);
|
||||
console.log(`Log: ${logFile}`);
|
||||
|
||||
await browser.close();
|
||||
})();
|
||||
'@
|
||||
|
||||
Set-Content -Path ".\discover-prismatic-by-color-param.js" -Value $js -Encoding UTF8
|
||||
}
|
||||
|
||||
try {
|
||||
Install-PlaywrightIfNeeded -Requested:$InstallPlaywright
|
||||
Write-NodeDiscoveryScript
|
||||
|
||||
Write-Host "Running color-param URL discovery..."
|
||||
|
||||
$nodeArgs = @(
|
||||
".\discover-prismatic-by-color-param.js",
|
||||
"--max-scrolls-per-color=$MaxScrollsPerColor",
|
||||
"--stop-after-no-new-scrolls=$StopAfterNoNewScrolls"
|
||||
)
|
||||
|
||||
if ($Headed) {
|
||||
$nodeArgs += "--headed"
|
||||
}
|
||||
|
||||
node @nodeArgs
|
||||
}
|
||||
catch {
|
||||
Write-Error $_.Exception.Message
|
||||
exit 1
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
# Get-Product-Info-Resumable.ps1
|
||||
#
|
||||
# Resumable, slow/polite Prismatic Powders product scraper.
|
||||
#
|
||||
# Inputs:
|
||||
# .\product-urls.txt
|
||||
#
|
||||
# Outputs:
|
||||
# .\prismatic_powders.json
|
||||
# .\prismatic-scrape-progress.log
|
||||
#
|
||||
# First-time setup:
|
||||
# Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass
|
||||
# .\Get-Product-Info-Resumable.ps1 -InstallPlaywright -Headed -MaxProducts 5
|
||||
#
|
||||
# Normal full run:
|
||||
# .\Get-Product-Info-Resumable.ps1
|
||||
#
|
||||
# Test first 25 remaining:
|
||||
# .\Get-Product-Info-Resumable.ps1 -MaxProducts 25 -Headed
|
||||
#
|
||||
# Retry failed URLs too:
|
||||
# .\Get-Product-Info-Resumable.ps1 -RetryErrors
|
||||
#
|
||||
# Slow it down more:
|
||||
# .\Get-Product-Info-Resumable.ps1 -MinDelaySeconds 12 -MaxDelaySeconds 25
|
||||
|
||||
param(
|
||||
[switch]$InstallPlaywright,
|
||||
[switch]$Headed,
|
||||
|
||||
[string]$InputFile = ".\product-urls.txt",
|
||||
[string]$OutputJson = ".\prismatic_powders.json",
|
||||
[string]$ProgressLog = ".\prismatic-scrape-progress.log",
|
||||
|
||||
[int]$MinDelaySeconds = 4,
|
||||
[int]$MaxDelaySeconds = 10,
|
||||
[int]$PageSettleSeconds = 4,
|
||||
|
||||
# 0 means no limit.
|
||||
[int]$MaxProducts = 0,
|
||||
|
||||
# By default, URLs in errors are skipped on resume.
|
||||
# Use -RetryErrors to try failed URLs again.
|
||||
[switch]$RetryErrors
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Ensure-NodeAvailable {
|
||||
if (-not (Get-Command node -ErrorAction SilentlyContinue)) {
|
||||
throw "Node.js is required. Install Node.js LTS from https://nodejs.org/"
|
||||
}
|
||||
|
||||
if (-not (Get-Command npm -ErrorAction SilentlyContinue)) {
|
||||
throw "npm is required. It usually comes with Node.js."
|
||||
}
|
||||
}
|
||||
|
||||
function Install-PlaywrightIfNeeded {
|
||||
param([bool]$Requested)
|
||||
|
||||
Ensure-NodeAvailable
|
||||
|
||||
if ($Requested -or -not (Test-Path ".\node_modules\playwright")) {
|
||||
Write-Host "Installing Playwright package locally..."
|
||||
npm init -y | Out-Null
|
||||
npm install playwright | Out-Null
|
||||
|
||||
Write-Host "Installing Playwright Chromium browser..."
|
||||
npx playwright install chromium
|
||||
}
|
||||
}
|
||||
|
||||
function Write-NodeScraper {
|
||||
$js = @'
|
||||
const fs = require("fs");
|
||||
const { chromium } = require("playwright");
|
||||
|
||||
const headed = process.argv.includes("--headed");
|
||||
const retryErrors = process.argv.includes("--retry-errors");
|
||||
|
||||
function getArgValue(name, defaultValue) {
|
||||
const prefix = `--${name}=`;
|
||||
const found = process.argv.find(x => x.startsWith(prefix));
|
||||
return found ? found.slice(prefix.length) : defaultValue;
|
||||
}
|
||||
|
||||
const inputFile = getArgValue("input-file", "product-urls.txt");
|
||||
const outputJson = getArgValue("output-json", "prismatic_powders.json");
|
||||
const progressLog = getArgValue("progress-log", "prismatic-scrape-progress.log");
|
||||
|
||||
const minDelaySeconds = parseInt(getArgValue("min-delay-seconds", "8"), 10);
|
||||
const maxDelaySeconds = parseInt(getArgValue("max-delay-seconds", "18"), 10);
|
||||
const pageSettleSeconds = parseInt(getArgValue("page-settle-seconds", "4"), 10);
|
||||
const maxProducts = parseInt(getArgValue("max-products", "0"), 10);
|
||||
|
||||
function clean(text) {
|
||||
return (text || "").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function cleanUrl(url) {
|
||||
return (url || "").split("?")[0].split("#")[0].trim();
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function randomDelayMs() {
|
||||
const minMs = Math.max(0, minDelaySeconds * 1000);
|
||||
const maxMs = Math.max(minMs, maxDelaySeconds * 1000);
|
||||
return Math.floor(minMs + Math.random() * (maxMs - minMs + 1));
|
||||
}
|
||||
|
||||
function logLine(message) {
|
||||
const line = `[${new Date().toISOString()}] ${message}`;
|
||||
console.log(line);
|
||||
fs.appendFileSync(progressLog, line + "\r\n", "utf8");
|
||||
}
|
||||
|
||||
function absoluteUrl(baseUrl, maybeUrl) {
|
||||
if (!maybeUrl) return "";
|
||||
try {
|
||||
return new URL(maybeUrl, baseUrl).href;
|
||||
} catch {
|
||||
return maybeUrl;
|
||||
}
|
||||
}
|
||||
|
||||
function loadInputUrls() {
|
||||
if (!fs.existsSync(inputFile)) {
|
||||
throw new Error(`Input file not found: ${inputFile}`);
|
||||
}
|
||||
|
||||
const urls = fs.readFileSync(inputFile, "utf8")
|
||||
.split(/\r?\n/)
|
||||
.map(cleanUrl)
|
||||
.filter(Boolean)
|
||||
.filter(x => !x.startsWith("#"))
|
||||
.filter(x => /\/shop\/powder-coating-colors\/[A-Z0-9-]+\//i.test(x));
|
||||
|
||||
return [...new Set(urls)];
|
||||
}
|
||||
|
||||
function loadOutput() {
|
||||
if (!fs.existsSync(outputJson)) {
|
||||
return { results: [], errors: [] };
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(fs.readFileSync(outputJson, "utf8"));
|
||||
|
||||
if (Array.isArray(parsed)) {
|
||||
return { results: parsed, errors: [] };
|
||||
}
|
||||
|
||||
return {
|
||||
results: Array.isArray(parsed.results) ? parsed.results : [],
|
||||
errors: Array.isArray(parsed.errors) ? parsed.errors : []
|
||||
};
|
||||
} catch (err) {
|
||||
const backup = `${outputJson}.invalid-${Date.now()}.bak`;
|
||||
fs.copyFileSync(outputJson, backup);
|
||||
throw new Error(`Could not parse existing ${outputJson}. Backed it up to ${backup}. Error: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function saveOutput(data) {
|
||||
const tempFile = `${outputJson}.tmp`;
|
||||
fs.writeFileSync(tempFile, JSON.stringify(data, null, 2), "utf8");
|
||||
fs.renameSync(tempFile, outputJson);
|
||||
}
|
||||
|
||||
function parsePriceTiers(plainText) {
|
||||
const priceMatches = [...plainText.matchAll(/(\d+\s*-\s*\d+\s*lbs|\d+\s*\+\s*lbs)\s*\$([\d.]+)/gi)];
|
||||
|
||||
return priceMatches.map(m => {
|
||||
const rangeText = clean(m[1]);
|
||||
const price = parseFloat(m[2]);
|
||||
|
||||
let min = null;
|
||||
let max = null;
|
||||
|
||||
const rangeMatch = rangeText.match(/(\d+)\s*-\s*(\d+)/);
|
||||
if (rangeMatch) {
|
||||
min = parseInt(rangeMatch[1], 10);
|
||||
max = parseInt(rangeMatch[2], 10);
|
||||
}
|
||||
|
||||
const plusMatch = rangeText.match(/(\d+)\s*\+/);
|
||||
if (plusMatch) {
|
||||
min = parseInt(plusMatch[1], 10);
|
||||
max = null;
|
||||
}
|
||||
|
||||
return { min, max, price };
|
||||
});
|
||||
}
|
||||
|
||||
async function getLinkByText(page, patterns) {
|
||||
const links = await page.locator("a").evaluateAll((anchors) =>
|
||||
anchors.map(a => ({
|
||||
text: (a.innerText || a.textContent || "").replace(/\s+/g, " ").trim(),
|
||||
href: a.getAttribute("href") || ""
|
||||
}))
|
||||
);
|
||||
|
||||
for (const link of links) {
|
||||
if (patterns.some(p => new RegExp(p, "i").test(link.text))) {
|
||||
return absoluteUrl(page.url(), link.href);
|
||||
}
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
async function getSampleImageUrl(page) {
|
||||
const imageUrls = await page.locator("img").evaluateAll((imgs) =>
|
||||
imgs.map(img =>
|
||||
img.currentSrc ||
|
||||
img.src ||
|
||||
img.getAttribute("src") ||
|
||||
img.getAttribute("data-src") ||
|
||||
""
|
||||
).filter(Boolean)
|
||||
);
|
||||
|
||||
return (
|
||||
imageUrls.find(src => /images\.nicindustries\.com/i.test(src) && !/thumbnail/i.test(src)) ||
|
||||
imageUrls.find(src => /images\.nicindustries\.com/i.test(src)) ||
|
||||
imageUrls.find(src => /prismatic|powder|color/i.test(src)) ||
|
||||
""
|
||||
);
|
||||
}
|
||||
|
||||
async function parseProduct(page, url) {
|
||||
logLine(`Scraping ${url}`);
|
||||
|
||||
const response = await page.goto(url, {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: 60000
|
||||
});
|
||||
|
||||
await page.waitForTimeout(pageSettleSeconds * 1000);
|
||||
|
||||
const status = response ? response.status() : 0;
|
||||
const pageTitle = clean(await page.title().catch(() => ""));
|
||||
const plainText = clean(await page.locator("body").innerText().catch(() => ""));
|
||||
|
||||
logLine(`HTTP status ${status}; title "${pageTitle}"`);
|
||||
|
||||
if (status === 403 || /^403 Forbidden$/i.test(pageTitle) || /^403 Forbidden$/i.test(plainText)) {
|
||||
throw new Error("403 Forbidden returned by site.");
|
||||
}
|
||||
|
||||
if (status === 404 || /404|Page Not Found/i.test(pageTitle)) {
|
||||
throw new Error("404 Not Found returned by site.");
|
||||
}
|
||||
|
||||
const title = clean(await page.locator("h1").first().innerText().catch(() => ""));
|
||||
|
||||
const skuMatch = plainText.match(/Item:\s*([A-Z0-9-]+)/i);
|
||||
const sku = skuMatch ? skuMatch[1] : "";
|
||||
|
||||
if (!sku && !title) {
|
||||
throw new Error("Could not find SKU or title on product page.");
|
||||
}
|
||||
|
||||
const descMatch = plainText.match(/Description:\s*(.*?)(WARNING:|What does this match\?|$)/is);
|
||||
const description = descMatch ? clean(descMatch[1]) : "";
|
||||
|
||||
const priceTiers = parsePriceTiers(plainText);
|
||||
|
||||
const safetyDataSheetUrl = await getLinkByText(page, ["Safety Data Sheet", "\\bSDS\\b"]);
|
||||
const applicationGuideUrl = await getLinkByText(page, ["Application Guide"]);
|
||||
const technicalDataSheetUrl = await getLinkByText(page, ["Tech Data Sheet", "Technical Data Sheet", "\\bTDS\\b"]);
|
||||
const sampleImageUrl = await getSampleImageUrl(page);
|
||||
|
||||
return {
|
||||
sku,
|
||||
color_name: title,
|
||||
description,
|
||||
price_tiers: priceTiers,
|
||||
safety_data_sheet_url: safetyDataSheetUrl,
|
||||
technical_data_sheet_url: technicalDataSheetUrl,
|
||||
application_guide_url: applicationGuideUrl,
|
||||
sample_image_url: sampleImageUrl,
|
||||
product_url: url,
|
||||
scraped_at: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const allUrls = loadInputUrls();
|
||||
const data = loadOutput();
|
||||
|
||||
const completedUrls = new Set(data.results.map(r => cleanUrl(r.product_url)).filter(Boolean));
|
||||
const errorUrls = new Set(data.errors.map(e => cleanUrl(e.product_url)).filter(Boolean));
|
||||
|
||||
let remainingUrls = allUrls.filter(url => {
|
||||
if (completedUrls.has(url)) return false;
|
||||
if (!retryErrors && errorUrls.has(url)) return false;
|
||||
return true;
|
||||
});
|
||||
|
||||
if (maxProducts > 0) {
|
||||
remainingUrls = remainingUrls.slice(0, maxProducts);
|
||||
}
|
||||
|
||||
logLine(`Input URLs: ${allUrls.length}`);
|
||||
logLine(`Already scraped: ${completedUrls.size}`);
|
||||
logLine(`Existing errors: ${errorUrls.size}`);
|
||||
logLine(`Retry errors: ${retryErrors ? "yes" : "no"}`);
|
||||
logLine(`This run target count: ${remainingUrls.length}`);
|
||||
logLine(`Delay range: ${minDelaySeconds}-${maxDelaySeconds} seconds; page settle: ${pageSettleSeconds} seconds`);
|
||||
|
||||
if (remainingUrls.length === 0) {
|
||||
logLine("Nothing to scrape. Done.");
|
||||
saveOutput(data);
|
||||
return;
|
||||
}
|
||||
|
||||
const browser = await chromium.launch({
|
||||
headless: !headed
|
||||
});
|
||||
|
||||
const context = await browser.newContext({
|
||||
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
viewport: { width: 1365, height: 900 },
|
||||
locale: "en-US",
|
||||
timezoneId: "America/New_York"
|
||||
});
|
||||
|
||||
const page = await context.newPage();
|
||||
|
||||
let processedThisRun = 0;
|
||||
|
||||
for (const url of remainingUrls) {
|
||||
try {
|
||||
const row = await parseProduct(page, url);
|
||||
|
||||
// If retrying an old error, keep the old error history but avoid duplicate successful result.
|
||||
if (!completedUrls.has(url)) {
|
||||
data.results.push(row);
|
||||
completedUrls.add(url);
|
||||
}
|
||||
|
||||
processedThisRun++;
|
||||
saveOutput(data);
|
||||
|
||||
logLine(`Saved result ${processedThisRun}/${remainingUrls.length}: ${row.sku || "(no sku)"} ${row.color_name || ""}`);
|
||||
} catch (err) {
|
||||
const errorRecord = {
|
||||
product_url: url,
|
||||
error: err.message,
|
||||
scraped_at: new Date().toISOString()
|
||||
};
|
||||
|
||||
data.errors.push(errorRecord);
|
||||
saveOutput(data);
|
||||
|
||||
logLine(`ERROR ${url}: ${err.message}`);
|
||||
}
|
||||
|
||||
const delay = randomDelayMs();
|
||||
logLine(`Waiting ${(delay / 1000).toFixed(1)} seconds before next product...`);
|
||||
await sleep(delay);
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
|
||||
logLine(`Done. Results: ${data.results.length}; Errors: ${data.errors.length}; Output: ${outputJson}`);
|
||||
})();
|
||||
'@
|
||||
|
||||
Set-Content -Path ".\prismatic-browser-scraper.js" -Value $js -Encoding UTF8
|
||||
}
|
||||
|
||||
try {
|
||||
Install-PlaywrightIfNeeded -Requested:$InstallPlaywright
|
||||
Write-NodeScraper
|
||||
|
||||
Write-Host "Running resumable browser scraper..."
|
||||
|
||||
$nodeArgs = @(
|
||||
".\prismatic-browser-scraper.js",
|
||||
"--input-file=$InputFile",
|
||||
"--output-json=$OutputJson",
|
||||
"--progress-log=$ProgressLog",
|
||||
"--min-delay-seconds=$MinDelaySeconds",
|
||||
"--max-delay-seconds=$MaxDelaySeconds",
|
||||
"--page-settle-seconds=$PageSettleSeconds",
|
||||
"--max-products=$MaxProducts"
|
||||
)
|
||||
|
||||
if ($Headed) {
|
||||
$nodeArgs += "--headed"
|
||||
}
|
||||
|
||||
if ($RetryErrors) {
|
||||
$nodeArgs += "--retry-errors"
|
||||
}
|
||||
|
||||
node @nodeArgs
|
||||
}
|
||||
catch {
|
||||
Write-Error $_.Exception.Message
|
||||
exit 1
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
# Get-Product-Info-Resumable.ps1
|
||||
#
|
||||
# Resumable, slow/polite Prismatic Powders product scraper.
|
||||
#
|
||||
# Inputs:
|
||||
# .\product-urls.txt
|
||||
#
|
||||
# Outputs:
|
||||
# .\prismatic_powders.json
|
||||
# .\prismatic-scrape-progress.log
|
||||
#
|
||||
# First-time setup:
|
||||
# Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass
|
||||
# .\Get-Product-Info-Resumable.ps1 -InstallPlaywright -Headed -MaxProducts 5
|
||||
#
|
||||
# Normal full run:
|
||||
# .\Get-Product-Info-Resumable.ps1
|
||||
#
|
||||
# Test first 25 remaining:
|
||||
# .\Get-Product-Info-Resumable.ps1 -MaxProducts 25 -Headed
|
||||
#
|
||||
# Retry failed URLs too:
|
||||
# .\Get-Product-Info-Resumable.ps1 -RetryErrors
|
||||
#
|
||||
# Slow it down more:
|
||||
# .\Get-Product-Info-Resumable.ps1 -MinDelaySeconds 12 -MaxDelaySeconds 25
|
||||
|
||||
param(
|
||||
[switch]$InstallPlaywright,
|
||||
[switch]$Headed,
|
||||
|
||||
[string]$InputFile = ".\product-urls.txt",
|
||||
[string]$OutputJson = ".\prismatic_powders.json",
|
||||
[string]$ProgressLog = ".\prismatic-scrape-progress.log",
|
||||
|
||||
[int]$MinDelaySeconds = 8,
|
||||
[int]$MaxDelaySeconds = 18,
|
||||
[int]$PageSettleSeconds = 4,
|
||||
|
||||
# 0 means no limit.
|
||||
[int]$MaxProducts = 0,
|
||||
|
||||
# By default, URLs in errors are skipped on resume.
|
||||
# Use -RetryErrors to try failed URLs again.
|
||||
[switch]$RetryErrors
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Ensure-NodeAvailable {
|
||||
if (-not (Get-Command node -ErrorAction SilentlyContinue)) {
|
||||
throw "Node.js is required. Install Node.js LTS from https://nodejs.org/"
|
||||
}
|
||||
|
||||
if (-not (Get-Command npm -ErrorAction SilentlyContinue)) {
|
||||
throw "npm is required. It usually comes with Node.js."
|
||||
}
|
||||
}
|
||||
|
||||
function Install-PlaywrightIfNeeded {
|
||||
param([bool]$Requested)
|
||||
|
||||
Ensure-NodeAvailable
|
||||
|
||||
if ($Requested -or -not (Test-Path ".\node_modules\playwright")) {
|
||||
Write-Host "Installing Playwright package locally..."
|
||||
npm init -y | Out-Null
|
||||
npm install playwright | Out-Null
|
||||
|
||||
Write-Host "Installing Playwright Chromium browser..."
|
||||
npx playwright install chromium
|
||||
}
|
||||
}
|
||||
|
||||
function Write-NodeScraper {
|
||||
$js = @'
|
||||
const fs = require("fs");
|
||||
const { chromium } = require("playwright");
|
||||
|
||||
const headed = process.argv.includes("--headed");
|
||||
const retryErrors = process.argv.includes("--retry-errors");
|
||||
|
||||
function getArgValue(name, defaultValue) {
|
||||
const prefix = `--${name}=`;
|
||||
const found = process.argv.find(x => x.startsWith(prefix));
|
||||
return found ? found.slice(prefix.length) : defaultValue;
|
||||
}
|
||||
|
||||
const inputFile = getArgValue("input-file", "product-urls.txt");
|
||||
const outputJson = getArgValue("output-json", "prismatic_powders.json");
|
||||
const progressLog = getArgValue("progress-log", "prismatic-scrape-progress.log");
|
||||
|
||||
const minDelaySeconds = parseInt(getArgValue("min-delay-seconds", "8"), 10);
|
||||
const maxDelaySeconds = parseInt(getArgValue("max-delay-seconds", "18"), 10);
|
||||
const pageSettleSeconds = parseInt(getArgValue("page-settle-seconds", "4"), 10);
|
||||
const maxProducts = parseInt(getArgValue("max-products", "0"), 10);
|
||||
|
||||
function clean(text) {
|
||||
return (text || "").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function cleanUrl(url) {
|
||||
return (url || "").split("?")[0].split("#")[0].trim();
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function randomDelayMs() {
|
||||
const minMs = Math.max(0, minDelaySeconds * 1000);
|
||||
const maxMs = Math.max(minMs, maxDelaySeconds * 1000);
|
||||
return Math.floor(minMs + Math.random() * (maxMs - minMs + 1));
|
||||
}
|
||||
|
||||
function logLine(message) {
|
||||
const line = `[${new Date().toISOString()}] ${message}`;
|
||||
console.log(line);
|
||||
fs.appendFileSync(progressLog, line + "\r\n", "utf8");
|
||||
}
|
||||
|
||||
function absoluteUrl(baseUrl, maybeUrl) {
|
||||
if (!maybeUrl) return "";
|
||||
try {
|
||||
return new URL(maybeUrl, baseUrl).href;
|
||||
} catch {
|
||||
return maybeUrl;
|
||||
}
|
||||
}
|
||||
|
||||
function loadInputUrls() {
|
||||
if (!fs.existsSync(inputFile)) {
|
||||
throw new Error(`Input file not found: ${inputFile}`);
|
||||
}
|
||||
|
||||
const urls = fs.readFileSync(inputFile, "utf8")
|
||||
.split(/\r?\n/)
|
||||
.map(cleanUrl)
|
||||
.filter(Boolean)
|
||||
.filter(x => !x.startsWith("#"))
|
||||
.filter(x => /\/shop\/powder-coating-colors\/[A-Z0-9-]+\//i.test(x));
|
||||
|
||||
return [...new Set(urls)];
|
||||
}
|
||||
|
||||
function loadOutput() {
|
||||
if (!fs.existsSync(outputJson)) {
|
||||
return { results: [], errors: [] };
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(fs.readFileSync(outputJson, "utf8"));
|
||||
|
||||
if (Array.isArray(parsed)) {
|
||||
return { results: parsed, errors: [] };
|
||||
}
|
||||
|
||||
return {
|
||||
results: Array.isArray(parsed.results) ? parsed.results : [],
|
||||
errors: Array.isArray(parsed.errors) ? parsed.errors : []
|
||||
};
|
||||
} catch (err) {
|
||||
const backup = `${outputJson}.invalid-${Date.now()}.bak`;
|
||||
fs.copyFileSync(outputJson, backup);
|
||||
throw new Error(`Could not parse existing ${outputJson}. Backed it up to ${backup}. Error: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function saveOutput(data) {
|
||||
const tempFile = `${outputJson}.tmp`;
|
||||
fs.writeFileSync(tempFile, JSON.stringify(data, null, 2), "utf8");
|
||||
fs.renameSync(tempFile, outputJson);
|
||||
}
|
||||
|
||||
function parsePriceTiers(plainText) {
|
||||
const priceMatches = [...plainText.matchAll(/(\d+\s*-\s*\d+\s*lbs|\d+\s*\+\s*lbs)\s*\$([\d.]+)/gi)];
|
||||
|
||||
return priceMatches.map(m => {
|
||||
const rangeText = clean(m[1]);
|
||||
const price = parseFloat(m[2]);
|
||||
|
||||
let min = null;
|
||||
let max = null;
|
||||
|
||||
const rangeMatch = rangeText.match(/(\d+)\s*-\s*(\d+)/);
|
||||
if (rangeMatch) {
|
||||
min = parseInt(rangeMatch[1], 10);
|
||||
max = parseInt(rangeMatch[2], 10);
|
||||
}
|
||||
|
||||
const plusMatch = rangeText.match(/(\d+)\s*\+/);
|
||||
if (plusMatch) {
|
||||
min = parseInt(plusMatch[1], 10);
|
||||
max = null;
|
||||
}
|
||||
|
||||
return { min, max, price };
|
||||
});
|
||||
}
|
||||
|
||||
async function getLinkByText(page, patterns) {
|
||||
const links = await page.locator("a").evaluateAll((anchors) =>
|
||||
anchors.map(a => ({
|
||||
text: (a.innerText || a.textContent || "").replace(/\s+/g, " ").trim(),
|
||||
href: a.getAttribute("href") || ""
|
||||
}))
|
||||
);
|
||||
|
||||
for (const link of links) {
|
||||
if (patterns.some(p => new RegExp(p, "i").test(link.text))) {
|
||||
return absoluteUrl(page.url(), link.href);
|
||||
}
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
async function getSampleImageUrl(page) {
|
||||
const imageUrls = await page.locator("img").evaluateAll((imgs) =>
|
||||
imgs.map(img =>
|
||||
img.currentSrc ||
|
||||
img.src ||
|
||||
img.getAttribute("src") ||
|
||||
img.getAttribute("data-src") ||
|
||||
""
|
||||
).filter(Boolean)
|
||||
);
|
||||
|
||||
return (
|
||||
imageUrls.find(src => /images\.nicindustries\.com/i.test(src) && !/thumbnail/i.test(src)) ||
|
||||
imageUrls.find(src => /images\.nicindustries\.com/i.test(src)) ||
|
||||
imageUrls.find(src => /prismatic|powder|color/i.test(src)) ||
|
||||
""
|
||||
);
|
||||
}
|
||||
|
||||
async function parseProduct(page, url) {
|
||||
logLine(`Scraping ${url}`);
|
||||
|
||||
const response = await page.goto(url, {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: 60000
|
||||
});
|
||||
|
||||
await page.waitForTimeout(pageSettleSeconds * 1000);
|
||||
|
||||
const status = response ? response.status() : 0;
|
||||
const pageTitle = clean(await page.title().catch(() => ""));
|
||||
const plainText = clean(await page.locator("body").innerText().catch(() => ""));
|
||||
|
||||
logLine(`HTTP status ${status}; title "${pageTitle}"`);
|
||||
|
||||
if (status === 403 || /^403 Forbidden$/i.test(pageTitle) || /^403 Forbidden$/i.test(plainText)) {
|
||||
throw new Error("403 Forbidden returned by site.");
|
||||
}
|
||||
|
||||
if (status === 404 || /404|Page Not Found/i.test(pageTitle)) {
|
||||
throw new Error("404 Not Found returned by site.");
|
||||
}
|
||||
|
||||
const title = clean(await page.locator("h1").first().innerText().catch(() => ""));
|
||||
|
||||
const skuMatch = plainText.match(/Item:\s*([A-Z0-9-]+)/i);
|
||||
const sku = skuMatch ? skuMatch[1] : "";
|
||||
|
||||
if (!sku && !title) {
|
||||
throw new Error("Could not find SKU or title on product page.");
|
||||
}
|
||||
|
||||
const descMatch = plainText.match(/Description:\s*(.*?)(WARNING:|What does this match\?|$)/is);
|
||||
const description = descMatch ? clean(descMatch[1]) : "";
|
||||
|
||||
const priceTiers = parsePriceTiers(plainText);
|
||||
|
||||
const safetyDataSheetUrl = await getLinkByText(page, ["Safety Data Sheet", "\\bSDS\\b"]);
|
||||
const applicationGuideUrl = await getLinkByText(page, ["Application Guide"]);
|
||||
const technicalDataSheetUrl = await getLinkByText(page, ["Tech Data Sheet", "Technical Data Sheet", "\\bTDS\\b"]);
|
||||
const sampleImageUrl = await getSampleImageUrl(page);
|
||||
|
||||
return {
|
||||
sku,
|
||||
color_name: title,
|
||||
description,
|
||||
price_tiers: priceTiers,
|
||||
safety_data_sheet_url: safetyDataSheetUrl,
|
||||
technical_data_sheet_url: technicalDataSheetUrl,
|
||||
application_guide_url: applicationGuideUrl,
|
||||
sample_image_url: sampleImageUrl,
|
||||
product_url: url,
|
||||
scraped_at: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const allUrls = loadInputUrls();
|
||||
const data = loadOutput();
|
||||
|
||||
const completedUrls = new Set(data.results.map(r => cleanUrl(r.product_url)).filter(Boolean));
|
||||
const errorUrls = new Set(data.errors.map(e => cleanUrl(e.product_url)).filter(Boolean));
|
||||
|
||||
let remainingUrls = allUrls.filter(url => {
|
||||
if (completedUrls.has(url)) return false;
|
||||
if (!retryErrors && errorUrls.has(url)) return false;
|
||||
return true;
|
||||
});
|
||||
|
||||
if (maxProducts > 0) {
|
||||
remainingUrls = remainingUrls.slice(0, maxProducts);
|
||||
}
|
||||
|
||||
logLine(`Input URLs: ${allUrls.length}`);
|
||||
logLine(`Already scraped: ${completedUrls.size}`);
|
||||
logLine(`Existing errors: ${errorUrls.size}`);
|
||||
logLine(`Retry errors: ${retryErrors ? "yes" : "no"}`);
|
||||
logLine(`This run target count: ${remainingUrls.length}`);
|
||||
logLine(`Delay range: ${minDelaySeconds}-${maxDelaySeconds} seconds; page settle: ${pageSettleSeconds} seconds`);
|
||||
|
||||
if (remainingUrls.length === 0) {
|
||||
logLine("Nothing to scrape. Done.");
|
||||
saveOutput(data);
|
||||
return;
|
||||
}
|
||||
|
||||
const browser = await chromium.launch({
|
||||
headless: !headed
|
||||
});
|
||||
|
||||
const context = await browser.newContext({
|
||||
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
viewport: { width: 1365, height: 900 },
|
||||
locale: "en-US",
|
||||
timezoneId: "America/New_York"
|
||||
});
|
||||
|
||||
const page = await context.newPage();
|
||||
|
||||
let processedThisRun = 0;
|
||||
|
||||
for (const url of remainingUrls) {
|
||||
try {
|
||||
const row = await parseProduct(page, url);
|
||||
|
||||
// If retrying an old error, keep the old error history but avoid duplicate successful result.
|
||||
if (!completedUrls.has(url)) {
|
||||
data.results.push(row);
|
||||
completedUrls.add(url);
|
||||
}
|
||||
|
||||
processedThisRun++;
|
||||
saveOutput(data);
|
||||
|
||||
logLine(`Saved result ${processedThisRun}/${remainingUrls.length}: ${row.sku || "(no sku)"} ${row.color_name || ""}`);
|
||||
} catch (err) {
|
||||
const errorRecord = {
|
||||
product_url: url,
|
||||
error: err.message,
|
||||
scraped_at: new Date().toISOString()
|
||||
};
|
||||
|
||||
data.errors.push(errorRecord);
|
||||
saveOutput(data);
|
||||
|
||||
logLine(`ERROR ${url}: ${err.message}`);
|
||||
}
|
||||
|
||||
const delay = randomDelayMs();
|
||||
logLine(`Waiting ${(delay / 1000).toFixed(1)} seconds before next product...`);
|
||||
await sleep(delay);
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
|
||||
logLine(`Done. Results: ${data.results.length}; Errors: ${data.errors.length}; Output: ${outputJson}`);
|
||||
})();
|
||||
'@
|
||||
|
||||
Set-Content -Path ".\prismatic-browser-scraper.js" -Value $js -Encoding UTF8
|
||||
}
|
||||
|
||||
try {
|
||||
Install-PlaywrightIfNeeded -Requested:$InstallPlaywright
|
||||
Write-NodeScraper
|
||||
|
||||
Write-Host "Running resumable browser scraper..."
|
||||
|
||||
$nodeArgs = @(
|
||||
".\prismatic-browser-scraper.js",
|
||||
"--input-file=$InputFile",
|
||||
"--output-json=$OutputJson",
|
||||
"--progress-log=$ProgressLog",
|
||||
"--min-delay-seconds=$MinDelaySeconds",
|
||||
"--max-delay-seconds=$MaxDelaySeconds",
|
||||
"--page-settle-seconds=$PageSettleSeconds",
|
||||
"--max-products=$MaxProducts"
|
||||
)
|
||||
|
||||
if ($Headed) {
|
||||
$nodeArgs += "--headed"
|
||||
}
|
||||
|
||||
if ($RetryErrors) {
|
||||
$nodeArgs += "--retry-errors"
|
||||
}
|
||||
|
||||
node @nodeArgs
|
||||
}
|
||||
catch {
|
||||
Write-Error $_.Exception.Message
|
||||
exit 1
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
# Crawl and Index Prismatic Colors - Known-Good Style JSON.ps1
|
||||
#
|
||||
# Rollback to the earlier working browser pattern:
|
||||
# - Playwright Chromium
|
||||
# - Full Chrome-style User-Agent
|
||||
# - JSON output
|
||||
# - Structured price tiers
|
||||
# - Color matches from #collection-list
|
||||
#
|
||||
# First-time setup:
|
||||
# Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass
|
||||
# .\Crawl-and-Index-Prismatic-colors-known-good-json.ps1 -InstallPlaywright
|
||||
#
|
||||
# Normal run:
|
||||
# .\Crawl-and-Index-Prismatic-colors-known-good-json.ps1
|
||||
#
|
||||
# Watch browser:
|
||||
# .\Crawl-and-Index-Prismatic-colors-known-good-json.ps1 -Headed
|
||||
|
||||
param(
|
||||
[switch]$InstallPlaywright,
|
||||
[switch]$Headed
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Ensure-NodeAvailable {
|
||||
if (-not (Get-Command node -ErrorAction SilentlyContinue)) {
|
||||
throw "Node.js is required. Install Node.js LTS from https://nodejs.org/"
|
||||
}
|
||||
|
||||
if (-not (Get-Command npm -ErrorAction SilentlyContinue)) {
|
||||
throw "npm is required. It usually comes with Node.js."
|
||||
}
|
||||
}
|
||||
|
||||
function Install-PlaywrightIfNeeded {
|
||||
param([bool]$Requested)
|
||||
|
||||
Ensure-NodeAvailable
|
||||
|
||||
if ($Requested -or -not (Test-Path ".\node_modules\playwright")) {
|
||||
Write-Host "Installing Playwright package locally..."
|
||||
npm init -y | Out-Null
|
||||
npm install playwright | Out-Null
|
||||
|
||||
Write-Host "Installing Playwright Chromium browser..."
|
||||
npx playwright install chromium
|
||||
}
|
||||
}
|
||||
|
||||
function Write-NodeScraper {
|
||||
# Single-quoted here-string prevents PowerShell from interpreting JavaScript regex/template strings.
|
||||
$js = @'
|
||||
const fs = require("fs");
|
||||
const { chromium } = require("playwright");
|
||||
|
||||
const headed = process.argv.includes("--headed");
|
||||
|
||||
const productUrls = [
|
||||
"https://www.prismaticpowders.com/shop/powder-coating-colors/PSS-11248/high-gloss-black"
|
||||
];
|
||||
|
||||
const outputJson = "prismatic_powders.json";
|
||||
|
||||
function clean(text) {
|
||||
return (text || "").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function absoluteUrl(baseUrl, maybeUrl) {
|
||||
if (!maybeUrl) return "";
|
||||
try {
|
||||
return new URL(maybeUrl, baseUrl).href;
|
||||
} catch {
|
||||
return maybeUrl;
|
||||
}
|
||||
}
|
||||
|
||||
function unique(items) {
|
||||
return [...new Set(items.filter(Boolean).map(clean).filter(Boolean))];
|
||||
}
|
||||
|
||||
async function getLinkByText(page, patterns) {
|
||||
const links = await page.locator("a").evaluateAll((anchors) =>
|
||||
anchors.map(a => ({
|
||||
text: (a.innerText || a.textContent || "").replace(/\s+/g, " ").trim(),
|
||||
href: a.getAttribute("href") || ""
|
||||
}))
|
||||
);
|
||||
|
||||
for (const link of links) {
|
||||
if (patterns.some(p => new RegExp(p, "i").test(link.text))) {
|
||||
return absoluteUrl(page.url(), link.href);
|
||||
}
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
function parsePriceTiers(plainText) {
|
||||
const priceMatches = [...plainText.matchAll(/(\d+\s*-\s*\d+\s*lbs|\d+\s*\+\s*lbs)\s*\$([\d.]+)/gi)];
|
||||
|
||||
return priceMatches.map(m => {
|
||||
const rangeText = clean(m[1]);
|
||||
const price = parseFloat(m[2]);
|
||||
|
||||
let min = null;
|
||||
let max = null;
|
||||
|
||||
const rangeMatch = rangeText.match(/(\d+)\s*-\s*(\d+)/);
|
||||
if (rangeMatch) {
|
||||
min = parseInt(rangeMatch[1], 10);
|
||||
max = parseInt(rangeMatch[2], 10);
|
||||
}
|
||||
|
||||
const plusMatch = rangeText.match(/(\d+)\s*\+/);
|
||||
if (plusMatch) {
|
||||
min = parseInt(plusMatch[1], 10);
|
||||
max = null;
|
||||
}
|
||||
|
||||
return {
|
||||
min,
|
||||
max,
|
||||
price
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function getSampleImageUrl(page) {
|
||||
const imageUrls = await page.locator("img").evaluateAll((imgs) =>
|
||||
imgs.map(img =>
|
||||
img.currentSrc ||
|
||||
img.src ||
|
||||
img.getAttribute("src") ||
|
||||
img.getAttribute("data-src") ||
|
||||
""
|
||||
).filter(Boolean)
|
||||
);
|
||||
|
||||
return (
|
||||
imageUrls.find(src => /images\.nicindustries\.com/i.test(src) && !/thumbnail/i.test(src)) ||
|
||||
imageUrls.find(src => /images\.nicindustries\.com/i.test(src)) ||
|
||||
imageUrls.find(src => /prismatic|powder|color/i.test(src)) ||
|
||||
""
|
||||
);
|
||||
}
|
||||
|
||||
async function parseProduct(page, url) {
|
||||
console.log(`Scraping ${url}`);
|
||||
|
||||
const response = await page.goto(url, {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: 60000
|
||||
});
|
||||
|
||||
await page.waitForTimeout(3000);
|
||||
|
||||
const status = response ? response.status() : 0;
|
||||
const pageTitle = clean(await page.title().catch(() => ""));
|
||||
const plainText = clean(await page.locator("body").innerText().catch(() => ""));
|
||||
|
||||
console.log(`HTTP status: ${status}`);
|
||||
console.log(`Page title: ${pageTitle}`);
|
||||
|
||||
// Do not silently output a fake product if blocked.
|
||||
if (status === 403 || /^403 Forbidden$/i.test(pageTitle) || /^403 Forbidden$/i.test(plainText)) {
|
||||
throw new Error("403 Forbidden returned by site.");
|
||||
}
|
||||
|
||||
const title = clean(await page.locator("h1").first().innerText().catch(() => ""));
|
||||
|
||||
const skuMatch = plainText.match(/Item:\s*([A-Z0-9-]+)/i);
|
||||
const sku = skuMatch ? skuMatch[1] : "";
|
||||
|
||||
const descMatch = plainText.match(/Description:\s*(.*?)(WARNING:|What does this match\?|$)/is);
|
||||
const description = descMatch ? clean(descMatch[1]) : "";
|
||||
|
||||
const priceTiers = parsePriceTiers(plainText);
|
||||
|
||||
const safetyDataSheetUrl = await getLinkByText(page, ["Safety Data Sheet", "\\bSDS\\b"]);
|
||||
const applicationGuideUrl = await getLinkByText(page, ["Application Guide"]);
|
||||
const technicalDataSheetUrl = await getLinkByText(page, ["Tech Data Sheet", "Technical Data Sheet", "\\bTDS\\b"]);
|
||||
const sampleImageUrl = await getSampleImageUrl(page);
|
||||
|
||||
return {
|
||||
sku,
|
||||
color_name: title,
|
||||
description,
|
||||
price_tiers: priceTiers,
|
||||
safety_data_sheet_url: safetyDataSheetUrl,
|
||||
technical_data_sheet_url: technicalDataSheetUrl,
|
||||
application_guide_url: applicationGuideUrl,
|
||||
sample_image_url: sampleImageUrl,
|
||||
product_url: url,
|
||||
scraped_at: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const browser = await chromium.launch({
|
||||
headless: !headed
|
||||
});
|
||||
|
||||
const context = await browser.newContext({
|
||||
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
viewport: { width: 1365, height: 900 },
|
||||
locale: "en-US",
|
||||
timezoneId: "America/New_York"
|
||||
});
|
||||
|
||||
const page = await context.newPage();
|
||||
|
||||
const results = [];
|
||||
const errors = [];
|
||||
|
||||
for (const url of productUrls) {
|
||||
try {
|
||||
const row = await parseProduct(page, url);
|
||||
results.push(row);
|
||||
await page.waitForTimeout(3000);
|
||||
} catch (err) {
|
||||
console.warn(`Failed ${url}: ${err.message}`);
|
||||
errors.push({
|
||||
product_url: url,
|
||||
error: err.message,
|
||||
scraped_at: new Date().toISOString()
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
|
||||
// If you prefer only the array, change this to JSON.stringify(results, null, 2)
|
||||
const output = {
|
||||
results,
|
||||
errors
|
||||
};
|
||||
|
||||
fs.writeFileSync(outputJson, JSON.stringify(output, null, 2), "utf8");
|
||||
|
||||
console.log(`Done. Output: ${outputJson}`);
|
||||
})();
|
||||
'@
|
||||
|
||||
Set-Content -Path ".\prismatic-browser-scraper.js" -Value $js -Encoding UTF8
|
||||
}
|
||||
|
||||
try {
|
||||
Install-PlaywrightIfNeeded -Requested:$InstallPlaywright
|
||||
Write-NodeScraper
|
||||
|
||||
Write-Host "Running browser scraper..."
|
||||
|
||||
if ($Headed) {
|
||||
node .\prismatic-browser-scraper.js --headed
|
||||
}
|
||||
else {
|
||||
node .\prismatic-browser-scraper.js
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Write-Error $_.Exception.Message
|
||||
exit 1
|
||||
}
|
||||
@@ -0,0 +1,319 @@
|
||||
# Discover-Prismatic-Product-Urls-By-ColorParam.ps1
|
||||
#
|
||||
# Discovers Prismatic Powders product URLs by visiting color filter URLs like:
|
||||
# https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_red
|
||||
#
|
||||
# Outputs:
|
||||
# .\product-urls.txt
|
||||
# .\color-discovery-log.json
|
||||
#
|
||||
# First-time setup:
|
||||
# Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass
|
||||
# .\Discover-Prismatic-Product-Urls-By-ColorParam.ps1 -InstallPlaywright -Headed
|
||||
#
|
||||
# Normal run:
|
||||
# .\Discover-Prismatic-Product-Urls-By-ColorParam.ps1
|
||||
#
|
||||
# Watch browser:
|
||||
# .\Discover-Prismatic-Product-Urls-By-ColorParam.ps1 -Headed
|
||||
|
||||
param(
|
||||
[switch]$InstallPlaywright,
|
||||
[switch]$Headed,
|
||||
[int]$MaxScrollsPerColor = 180,
|
||||
[int]$StopAfterNoNewScrolls = 10
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Ensure-NodeAvailable {
|
||||
if (-not (Get-Command node -ErrorAction SilentlyContinue)) {
|
||||
throw "Node.js is required. Install Node.js LTS from https://nodejs.org/"
|
||||
}
|
||||
|
||||
if (-not (Get-Command npm -ErrorAction SilentlyContinue)) {
|
||||
throw "npm is required. It usually comes with Node.js."
|
||||
}
|
||||
}
|
||||
|
||||
function Install-PlaywrightIfNeeded {
|
||||
param([bool]$Requested)
|
||||
|
||||
Ensure-NodeAvailable
|
||||
|
||||
if ($Requested -or -not (Test-Path ".\node_modules\playwright")) {
|
||||
Write-Host "Installing Playwright package locally..."
|
||||
npm init -y | Out-Null
|
||||
npm install playwright | Out-Null
|
||||
|
||||
Write-Host "Installing Playwright Chromium browser..."
|
||||
npx playwright install chromium
|
||||
}
|
||||
}
|
||||
|
||||
function Write-NodeDiscoveryScript {
|
||||
$js = @'
|
||||
const fs = require("fs");
|
||||
const { chromium } = require("playwright");
|
||||
|
||||
const headed = process.argv.includes("--headed");
|
||||
|
||||
function getArgValue(name, defaultValue) {
|
||||
const prefix = `--${name}=`;
|
||||
const found = process.argv.find(x => x.startsWith(prefix));
|
||||
return found ? found.slice(prefix.length) : defaultValue;
|
||||
}
|
||||
|
||||
const maxScrollsPerColor = parseInt(getArgValue("max-scrolls-per-color", "180"), 10);
|
||||
const stopAfterNoNewScrolls = parseInt(getArgValue("stop-after-no-new-scrolls", "10"), 10);
|
||||
|
||||
const baseUrl = "https://www.prismaticpowders.com/shop/powder-coating-colors";
|
||||
const outputFile = "product-urls.txt";
|
||||
const logFile = "color-discovery-log.json";
|
||||
|
||||
// Update this list if you find more color params in the site HTML.
|
||||
const colorParams = [
|
||||
"pris_black",
|
||||
"pris_blue",
|
||||
"pris_bronze",
|
||||
"pris_brown",
|
||||
"pris_clear",
|
||||
"pris_copper",
|
||||
"pris_gold",
|
||||
"pris_gray",
|
||||
"pris_green",
|
||||
"pris_orange",
|
||||
"pris_pink",
|
||||
"pris_purple",
|
||||
"pris_red",
|
||||
"pris_silver",
|
||||
"pris_tan",
|
||||
"pris_white",
|
||||
"pris_yellow"
|
||||
];
|
||||
|
||||
function cleanUrl(url) {
|
||||
return (url || "").split("?")[0].split("#")[0].trim();
|
||||
}
|
||||
|
||||
function isProductUrl(url) {
|
||||
return /\/shop\/powder-coating-colors\/[A-Z0-9-]+\//i.test(url || "");
|
||||
}
|
||||
|
||||
function readExistingUrls() {
|
||||
if (!fs.existsSync(outputFile)) return [];
|
||||
|
||||
return fs.readFileSync(outputFile, "utf8")
|
||||
.split(/\r?\n/)
|
||||
.map(cleanUrl)
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function writeUrls(urls) {
|
||||
const sorted = [...urls].sort();
|
||||
fs.writeFileSync(outputFile, sorted.join("\r\n") + "\r\n", "utf8");
|
||||
}
|
||||
|
||||
function readLog() {
|
||||
if (!fs.existsSync(logFile)) {
|
||||
return {
|
||||
completed_colors: {},
|
||||
runs: []
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(logFile, "utf8"));
|
||||
} catch {
|
||||
return {
|
||||
completed_colors: {},
|
||||
runs: []
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function writeLog(log) {
|
||||
fs.writeFileSync(logFile, JSON.stringify(log, null, 2), "utf8");
|
||||
}
|
||||
|
||||
async function collectProductLinks(page) {
|
||||
const links = await page.locator("a").evaluateAll(anchors =>
|
||||
anchors
|
||||
.map(a => a.href)
|
||||
.filter(Boolean)
|
||||
.filter(h => /\/shop\/powder-coating-colors\/[A-Z0-9-]+\//i.test(h))
|
||||
);
|
||||
|
||||
return links.map(cleanUrl).filter(Boolean);
|
||||
}
|
||||
|
||||
async function scrollAndCollect(page, urls, label) {
|
||||
let noNewScrolls = 0;
|
||||
let totalAddedForThisColor = 0;
|
||||
|
||||
for (let i = 0; i < maxScrollsPerColor; i++) {
|
||||
const before = urls.size;
|
||||
|
||||
for (const link of await collectProductLinks(page)) {
|
||||
urls.add(link);
|
||||
}
|
||||
|
||||
const after = urls.size;
|
||||
const added = after - before;
|
||||
totalAddedForThisColor += added;
|
||||
|
||||
if (added === 0) {
|
||||
noNewScrolls++;
|
||||
} else {
|
||||
noNewScrolls = 0;
|
||||
}
|
||||
|
||||
writeUrls(urls);
|
||||
|
||||
console.log(`[${label}] Scroll ${i + 1}/${maxScrollsPerColor}: +${added}, total ${after}, no-new ${noNewScrolls}`);
|
||||
|
||||
if (noNewScrolls >= stopAfterNoNewScrolls) {
|
||||
break;
|
||||
}
|
||||
|
||||
await page.mouse.wheel(0, 2500);
|
||||
await page.waitForTimeout(1500);
|
||||
}
|
||||
|
||||
return totalAddedForThisColor;
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const existingUrls = readExistingUrls();
|
||||
const urls = new Set(existingUrls);
|
||||
const log = readLog();
|
||||
|
||||
console.log(`Existing URLs in ${outputFile}: ${existingUrls.length}`);
|
||||
|
||||
const browser = await chromium.launch({ headless: !headed });
|
||||
|
||||
const context = await browser.newContext({
|
||||
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
viewport: { width: 1365, height: 900 },
|
||||
locale: "en-US",
|
||||
timezoneId: "America/New_York"
|
||||
});
|
||||
|
||||
const page = await context.newPage();
|
||||
|
||||
const runRecord = {
|
||||
started_at: new Date().toISOString(),
|
||||
existing_at_start: existingUrls.length,
|
||||
colors_attempted: []
|
||||
};
|
||||
|
||||
for (const color of colorParams) {
|
||||
if (log.completed_colors[color]) {
|
||||
console.log(`Skipping completed color: ${color}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const url = `${baseUrl}?color=${encodeURIComponent(color)}`;
|
||||
console.log("");
|
||||
console.log(`Opening color filter: ${color}`);
|
||||
console.log(url);
|
||||
|
||||
try {
|
||||
const response = await page.goto(url, {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: 60000
|
||||
});
|
||||
|
||||
const status = response ? response.status() : "unknown";
|
||||
console.log(`HTTP status: ${status}`);
|
||||
|
||||
await page.waitForTimeout(5000);
|
||||
|
||||
const before = urls.size;
|
||||
const addedDuringScroll = await scrollAndCollect(page, urls, color);
|
||||
const after = urls.size;
|
||||
const netAdded = after - before;
|
||||
|
||||
log.completed_colors[color] = {
|
||||
url,
|
||||
http_status: status,
|
||||
added: netAdded,
|
||||
added_during_scroll: addedDuringScroll,
|
||||
total_after: after,
|
||||
completed_at: new Date().toISOString()
|
||||
};
|
||||
|
||||
runRecord.colors_attempted.push({
|
||||
color,
|
||||
url,
|
||||
http_status: status,
|
||||
added: netAdded,
|
||||
total_after: after
|
||||
});
|
||||
|
||||
writeLog(log);
|
||||
writeUrls(urls);
|
||||
|
||||
console.log(`Color complete: ${color}; added ${netAdded}; total ${after}`);
|
||||
|
||||
// Polite pause between filters.
|
||||
await page.waitForTimeout(3000);
|
||||
} catch (err) {
|
||||
console.log(`Color failed: ${color}; ${err.message}`);
|
||||
|
||||
runRecord.colors_attempted.push({
|
||||
color,
|
||||
url,
|
||||
added: 0,
|
||||
error: err.message
|
||||
});
|
||||
|
||||
writeLog(log);
|
||||
}
|
||||
}
|
||||
|
||||
runRecord.finished_at = new Date().toISOString();
|
||||
runRecord.final_total = urls.size;
|
||||
runRecord.new_this_run = urls.size - existingUrls.length;
|
||||
|
||||
log.runs.push(runRecord);
|
||||
writeLog(log);
|
||||
writeUrls(urls);
|
||||
|
||||
console.log("");
|
||||
console.log("Color-param discovery complete.");
|
||||
console.log(`Existing at start: ${existingUrls.length}`);
|
||||
console.log(`Final total: ${urls.size}`);
|
||||
console.log(`New this run: ${urls.size - existingUrls.length}`);
|
||||
console.log(`Output: ${outputFile}`);
|
||||
console.log(`Log: ${logFile}`);
|
||||
|
||||
await browser.close();
|
||||
})();
|
||||
'@
|
||||
|
||||
Set-Content -Path ".\discover-prismatic-by-color-param.js" -Value $js -Encoding UTF8
|
||||
}
|
||||
|
||||
try {
|
||||
Install-PlaywrightIfNeeded -Requested:$InstallPlaywright
|
||||
Write-NodeDiscoveryScript
|
||||
|
||||
Write-Host "Running color-param URL discovery..."
|
||||
|
||||
$nodeArgs = @(
|
||||
".\discover-prismatic-by-color-param.js",
|
||||
"--max-scrolls-per-color=$MaxScrollsPerColor",
|
||||
"--stop-after-no-new-scrolls=$StopAfterNoNewScrolls"
|
||||
)
|
||||
|
||||
if ($Headed) {
|
||||
$nodeArgs += "--headed"
|
||||
}
|
||||
|
||||
node @nodeArgs
|
||||
}
|
||||
catch {
|
||||
Write-Error $_.Exception.Message
|
||||
exit 1
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
# Crawl and Index Prismatic Colors - Known-Good Style JSON.ps1
|
||||
#
|
||||
# Rollback to the earlier working browser pattern:
|
||||
# - Playwright Chromium
|
||||
# - Full Chrome-style User-Agent
|
||||
# - JSON output
|
||||
# - Structured price tiers
|
||||
# - Color matches from #collection-list
|
||||
#
|
||||
# First-time setup:
|
||||
# Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass
|
||||
# .\Crawl-and-Index-Prismatic-colors-known-good-json.ps1 -InstallPlaywright
|
||||
#
|
||||
# Normal run:
|
||||
# .\Crawl-and-Index-Prismatic-colors-known-good-json.ps1
|
||||
#
|
||||
# Watch browser:
|
||||
# .\Crawl-and-Index-Prismatic-colors-known-good-json.ps1 -Headed
|
||||
|
||||
param(
|
||||
[switch]$InstallPlaywright,
|
||||
[switch]$Headed
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Ensure-NodeAvailable {
|
||||
if (-not (Get-Command node -ErrorAction SilentlyContinue)) {
|
||||
throw "Node.js is required. Install Node.js LTS from https://nodejs.org/"
|
||||
}
|
||||
|
||||
if (-not (Get-Command npm -ErrorAction SilentlyContinue)) {
|
||||
throw "npm is required. It usually comes with Node.js."
|
||||
}
|
||||
}
|
||||
|
||||
function Install-PlaywrightIfNeeded {
|
||||
param([bool]$Requested)
|
||||
|
||||
Ensure-NodeAvailable
|
||||
|
||||
if ($Requested -or -not (Test-Path ".\node_modules\playwright")) {
|
||||
Write-Host "Installing Playwright package locally..."
|
||||
npm init -y | Out-Null
|
||||
npm install playwright | Out-Null
|
||||
|
||||
Write-Host "Installing Playwright Chromium browser..."
|
||||
npx playwright install chromium
|
||||
}
|
||||
}
|
||||
|
||||
function Write-NodeScraper {
|
||||
# Single-quoted here-string prevents PowerShell from interpreting JavaScript regex/template strings.
|
||||
$js = @'
|
||||
const fs = require("fs");
|
||||
const { chromium } = require("playwright");
|
||||
|
||||
const headed = process.argv.includes("--headed");
|
||||
|
||||
const productUrls = [
|
||||
"https://www.prismaticpowders.com/shop/powder-coating-colors/PSS-11248/high-gloss-black"
|
||||
];
|
||||
|
||||
const outputJson = "prismatic_powders.json";
|
||||
|
||||
function clean(text) {
|
||||
return (text || "").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function absoluteUrl(baseUrl, maybeUrl) {
|
||||
if (!maybeUrl) return "";
|
||||
try {
|
||||
return new URL(maybeUrl, baseUrl).href;
|
||||
} catch {
|
||||
return maybeUrl;
|
||||
}
|
||||
}
|
||||
|
||||
function unique(items) {
|
||||
return [...new Set(items.filter(Boolean).map(clean).filter(Boolean))];
|
||||
}
|
||||
|
||||
async function getLinkByText(page, patterns) {
|
||||
const links = await page.locator("a").evaluateAll((anchors) =>
|
||||
anchors.map(a => ({
|
||||
text: (a.innerText || a.textContent || "").replace(/\s+/g, " ").trim(),
|
||||
href: a.getAttribute("href") || ""
|
||||
}))
|
||||
);
|
||||
|
||||
for (const link of links) {
|
||||
if (patterns.some(p => new RegExp(p, "i").test(link.text))) {
|
||||
return absoluteUrl(page.url(), link.href);
|
||||
}
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
function parsePriceTiers(plainText) {
|
||||
const priceMatches = [...plainText.matchAll(/(\d+\s*-\s*\d+\s*lbs|\d+\s*\+\s*lbs)\s*\$([\d.]+)/gi)];
|
||||
|
||||
return priceMatches.map(m => {
|
||||
const rangeText = clean(m[1]);
|
||||
const price = parseFloat(m[2]);
|
||||
|
||||
let min = null;
|
||||
let max = null;
|
||||
|
||||
const rangeMatch = rangeText.match(/(\d+)\s*-\s*(\d+)/);
|
||||
if (rangeMatch) {
|
||||
min = parseInt(rangeMatch[1], 10);
|
||||
max = parseInt(rangeMatch[2], 10);
|
||||
}
|
||||
|
||||
const plusMatch = rangeText.match(/(\d+)\s*\+/);
|
||||
if (plusMatch) {
|
||||
min = parseInt(plusMatch[1], 10);
|
||||
max = null;
|
||||
}
|
||||
|
||||
return {
|
||||
min,
|
||||
max,
|
||||
price
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function getSampleImageUrl(page) {
|
||||
const imageUrls = await page.locator("img").evaluateAll((imgs) =>
|
||||
imgs.map(img =>
|
||||
img.currentSrc ||
|
||||
img.src ||
|
||||
img.getAttribute("src") ||
|
||||
img.getAttribute("data-src") ||
|
||||
""
|
||||
).filter(Boolean)
|
||||
);
|
||||
|
||||
return (
|
||||
imageUrls.find(src => /images\.nicindustries\.com/i.test(src) && !/thumbnail/i.test(src)) ||
|
||||
imageUrls.find(src => /images\.nicindustries\.com/i.test(src)) ||
|
||||
imageUrls.find(src => /prismatic|powder|color/i.test(src)) ||
|
||||
""
|
||||
);
|
||||
}
|
||||
|
||||
async function parseProduct(page, url) {
|
||||
console.log(`Scraping ${url}`);
|
||||
|
||||
const response = await page.goto(url, {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: 60000
|
||||
});
|
||||
|
||||
await page.waitForTimeout(3000);
|
||||
|
||||
const status = response ? response.status() : 0;
|
||||
const pageTitle = clean(await page.title().catch(() => ""));
|
||||
const plainText = clean(await page.locator("body").innerText().catch(() => ""));
|
||||
|
||||
console.log(`HTTP status: ${status}`);
|
||||
console.log(`Page title: ${pageTitle}`);
|
||||
|
||||
// Do not silently output a fake product if blocked.
|
||||
if (status === 403 || /^403 Forbidden$/i.test(pageTitle) || /^403 Forbidden$/i.test(plainText)) {
|
||||
throw new Error("403 Forbidden returned by site.");
|
||||
}
|
||||
|
||||
const title = clean(await page.locator("h1").first().innerText().catch(() => ""));
|
||||
|
||||
const skuMatch = plainText.match(/Item:\s*([A-Z0-9-]+)/i);
|
||||
const sku = skuMatch ? skuMatch[1] : "";
|
||||
|
||||
const descMatch = plainText.match(/Description:\s*(.*?)(WARNING:|What does this match\?|$)/is);
|
||||
const description = descMatch ? clean(descMatch[1]) : "";
|
||||
|
||||
const priceTiers = parsePriceTiers(plainText);
|
||||
|
||||
const safetyDataSheetUrl = await getLinkByText(page, ["Safety Data Sheet", "\\bSDS\\b"]);
|
||||
const applicationGuideUrl = await getLinkByText(page, ["Application Guide"]);
|
||||
const technicalDataSheetUrl = await getLinkByText(page, ["Tech Data Sheet", "Technical Data Sheet", "\\bTDS\\b"]);
|
||||
const sampleImageUrl = await getSampleImageUrl(page);
|
||||
|
||||
return {
|
||||
sku,
|
||||
color_name: title,
|
||||
description,
|
||||
price_tiers: priceTiers,
|
||||
safety_data_sheet_url: safetyDataSheetUrl,
|
||||
technical_data_sheet_url: technicalDataSheetUrl,
|
||||
application_guide_url: applicationGuideUrl,
|
||||
sample_image_url: sampleImageUrl,
|
||||
product_url: url,
|
||||
scraped_at: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const browser = await chromium.launch({
|
||||
headless: !headed
|
||||
});
|
||||
|
||||
const context = await browser.newContext({
|
||||
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
viewport: { width: 1365, height: 900 },
|
||||
locale: "en-US",
|
||||
timezoneId: "America/New_York"
|
||||
});
|
||||
|
||||
const page = await context.newPage();
|
||||
|
||||
const results = [];
|
||||
const errors = [];
|
||||
|
||||
for (const url of productUrls) {
|
||||
try {
|
||||
const row = await parseProduct(page, url);
|
||||
results.push(row);
|
||||
await page.waitForTimeout(3000);
|
||||
} catch (err) {
|
||||
console.warn(`Failed ${url}: ${err.message}`);
|
||||
errors.push({
|
||||
product_url: url,
|
||||
error: err.message,
|
||||
scraped_at: new Date().toISOString()
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
|
||||
// If you prefer only the array, change this to JSON.stringify(results, null, 2)
|
||||
const output = {
|
||||
results,
|
||||
errors
|
||||
};
|
||||
|
||||
fs.writeFileSync(outputJson, JSON.stringify(output, null, 2), "utf8");
|
||||
|
||||
console.log(`Done. Output: ${outputJson}`);
|
||||
})();
|
||||
'@
|
||||
|
||||
Set-Content -Path ".\prismatic-browser-scraper.js" -Value $js -Encoding UTF8
|
||||
}
|
||||
|
||||
try {
|
||||
Install-PlaywrightIfNeeded -Requested:$InstallPlaywright
|
||||
Write-NodeScraper
|
||||
|
||||
Write-Host "Running browser scraper..."
|
||||
|
||||
if ($Headed) {
|
||||
node .\prismatic-browser-scraper.js --headed
|
||||
}
|
||||
else {
|
||||
node .\prismatic-browser-scraper.js
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Write-Error $_.Exception.Message
|
||||
exit 1
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,270 @@
|
||||
{
|
||||
"completed_colors": {
|
||||
"pris_black": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_black",
|
||||
"http_status": 200,
|
||||
"added": 472,
|
||||
"added_during_scroll": 472,
|
||||
"total_after": 472,
|
||||
"completed_at": "2026-04-30T00:47:46.289Z"
|
||||
},
|
||||
"pris_blue": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_blue",
|
||||
"http_status": 200,
|
||||
"added": 948,
|
||||
"added_during_scroll": 948,
|
||||
"total_after": 1420,
|
||||
"completed_at": "2026-04-30T00:49:25.145Z"
|
||||
},
|
||||
"pris_bronze": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_bronze",
|
||||
"http_status": 200,
|
||||
"added": 358,
|
||||
"added_during_scroll": 358,
|
||||
"total_after": 1778,
|
||||
"completed_at": "2026-04-30T00:50:18.466Z"
|
||||
},
|
||||
"pris_brown": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_brown",
|
||||
"http_status": 200,
|
||||
"added": 373,
|
||||
"added_during_scroll": 373,
|
||||
"total_after": 2151,
|
||||
"completed_at": "2026-04-30T00:51:18.033Z"
|
||||
},
|
||||
"pris_clear": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_clear",
|
||||
"http_status": 200,
|
||||
"added": 19,
|
||||
"added_during_scroll": 19,
|
||||
"total_after": 2170,
|
||||
"completed_at": "2026-04-30T00:51:42.889Z"
|
||||
},
|
||||
"pris_copper": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_copper",
|
||||
"http_status": 200,
|
||||
"added": 1094,
|
||||
"added_during_scroll": 1094,
|
||||
"total_after": 3264,
|
||||
"completed_at": "2026-04-30T00:56:34.934Z"
|
||||
},
|
||||
"pris_gold": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_gold",
|
||||
"http_status": 200,
|
||||
"added": 152,
|
||||
"added_during_scroll": 152,
|
||||
"total_after": 3416,
|
||||
"completed_at": "2026-04-30T00:57:26.775Z"
|
||||
},
|
||||
"pris_gray": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_gray",
|
||||
"http_status": 200,
|
||||
"added": 0,
|
||||
"added_during_scroll": 0,
|
||||
"total_after": 3416,
|
||||
"completed_at": "2026-04-30T00:57:49.624Z"
|
||||
},
|
||||
"pris_green": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_green",
|
||||
"http_status": 200,
|
||||
"added": 0,
|
||||
"added_during_scroll": 0,
|
||||
"total_after": 3416,
|
||||
"completed_at": "2026-04-30T00:58:12.277Z"
|
||||
},
|
||||
"pris_orange": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_orange",
|
||||
"http_status": 200,
|
||||
"added": 233,
|
||||
"added_during_scroll": 233,
|
||||
"total_after": 3649,
|
||||
"completed_at": "2026-04-30T00:59:06.776Z"
|
||||
},
|
||||
"pris_pink": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_pink",
|
||||
"http_status": 200,
|
||||
"added": 169,
|
||||
"added_during_scroll": 169,
|
||||
"total_after": 3818,
|
||||
"completed_at": "2026-04-30T00:59:49.323Z"
|
||||
},
|
||||
"pris_purple": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_purple",
|
||||
"http_status": 200,
|
||||
"added": 182,
|
||||
"added_during_scroll": 182,
|
||||
"total_after": 4000,
|
||||
"completed_at": "2026-04-30T01:00:38.111Z"
|
||||
},
|
||||
"pris_red": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_red",
|
||||
"http_status": 200,
|
||||
"added": 346,
|
||||
"added_during_scroll": 346,
|
||||
"total_after": 4346,
|
||||
"completed_at": "2026-04-30T01:01:51.910Z"
|
||||
},
|
||||
"pris_silver": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_silver",
|
||||
"http_status": 200,
|
||||
"added": 210,
|
||||
"added_during_scroll": 210,
|
||||
"total_after": 4556,
|
||||
"completed_at": "2026-04-30T01:02:51.835Z"
|
||||
},
|
||||
"pris_tan": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_tan",
|
||||
"http_status": 200,
|
||||
"added": 219,
|
||||
"added_during_scroll": 219,
|
||||
"total_after": 4775,
|
||||
"completed_at": "2026-04-30T01:03:43.244Z"
|
||||
},
|
||||
"pris_white": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_white",
|
||||
"http_status": 200,
|
||||
"added": 218,
|
||||
"added_during_scroll": 218,
|
||||
"total_after": 4993,
|
||||
"completed_at": "2026-04-30T01:04:39.931Z"
|
||||
},
|
||||
"pris_yellow": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_yellow",
|
||||
"http_status": 200,
|
||||
"added": 199,
|
||||
"added_during_scroll": 199,
|
||||
"total_after": 5192,
|
||||
"completed_at": "2026-04-30T01:05:31.945Z"
|
||||
}
|
||||
},
|
||||
"runs": [
|
||||
{
|
||||
"started_at": "2026-04-30T00:46:47.692Z",
|
||||
"existing_at_start": 0,
|
||||
"colors_attempted": [
|
||||
{
|
||||
"color": "pris_black",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_black",
|
||||
"http_status": 200,
|
||||
"added": 472,
|
||||
"total_after": 472
|
||||
},
|
||||
{
|
||||
"color": "pris_blue",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_blue",
|
||||
"http_status": 200,
|
||||
"added": 948,
|
||||
"total_after": 1420
|
||||
},
|
||||
{
|
||||
"color": "pris_bronze",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_bronze",
|
||||
"http_status": 200,
|
||||
"added": 358,
|
||||
"total_after": 1778
|
||||
},
|
||||
{
|
||||
"color": "pris_brown",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_brown",
|
||||
"http_status": 200,
|
||||
"added": 373,
|
||||
"total_after": 2151
|
||||
},
|
||||
{
|
||||
"color": "pris_clear",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_clear",
|
||||
"http_status": 200,
|
||||
"added": 19,
|
||||
"total_after": 2170
|
||||
},
|
||||
{
|
||||
"color": "pris_copper",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_copper",
|
||||
"http_status": 200,
|
||||
"added": 1094,
|
||||
"total_after": 3264
|
||||
},
|
||||
{
|
||||
"color": "pris_gold",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_gold",
|
||||
"http_status": 200,
|
||||
"added": 152,
|
||||
"total_after": 3416
|
||||
},
|
||||
{
|
||||
"color": "pris_gray",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_gray",
|
||||
"http_status": 200,
|
||||
"added": 0,
|
||||
"total_after": 3416
|
||||
},
|
||||
{
|
||||
"color": "pris_green",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_green",
|
||||
"http_status": 200,
|
||||
"added": 0,
|
||||
"total_after": 3416
|
||||
},
|
||||
{
|
||||
"color": "pris_orange",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_orange",
|
||||
"http_status": 200,
|
||||
"added": 233,
|
||||
"total_after": 3649
|
||||
},
|
||||
{
|
||||
"color": "pris_pink",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_pink",
|
||||
"http_status": 200,
|
||||
"added": 169,
|
||||
"total_after": 3818
|
||||
},
|
||||
{
|
||||
"color": "pris_purple",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_purple",
|
||||
"http_status": 200,
|
||||
"added": 182,
|
||||
"total_after": 4000
|
||||
},
|
||||
{
|
||||
"color": "pris_red",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_red",
|
||||
"http_status": 200,
|
||||
"added": 346,
|
||||
"total_after": 4346
|
||||
},
|
||||
{
|
||||
"color": "pris_silver",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_silver",
|
||||
"http_status": 200,
|
||||
"added": 210,
|
||||
"total_after": 4556
|
||||
},
|
||||
{
|
||||
"color": "pris_tan",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_tan",
|
||||
"http_status": 200,
|
||||
"added": 219,
|
||||
"total_after": 4775
|
||||
},
|
||||
{
|
||||
"color": "pris_white",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_white",
|
||||
"http_status": 200,
|
||||
"added": 218,
|
||||
"total_after": 4993
|
||||
},
|
||||
{
|
||||
"color": "pris_yellow",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_yellow",
|
||||
"http_status": 200,
|
||||
"added": 199,
|
||||
"total_after": 5192
|
||||
}
|
||||
],
|
||||
"finished_at": "2026-04-30T01:05:34.987Z",
|
||||
"final_total": 5192,
|
||||
"new_this_run": 5192
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
const fs = require("fs");
|
||||
const { chromium } = require("playwright");
|
||||
|
||||
const headed = process.argv.includes("--headed");
|
||||
|
||||
function getArgValue(name, defaultValue) {
|
||||
const prefix = `--${name}=`;
|
||||
const found = process.argv.find(x => x.startsWith(prefix));
|
||||
return found ? found.slice(prefix.length) : defaultValue;
|
||||
}
|
||||
|
||||
const maxScrollsPerColor = parseInt(getArgValue("max-scrolls-per-color", "180"), 10);
|
||||
const stopAfterNoNewScrolls = parseInt(getArgValue("stop-after-no-new-scrolls", "10"), 10);
|
||||
|
||||
const baseUrl = "https://www.prismaticpowders.com/shop/powder-coating-colors";
|
||||
const outputFile = "product-urls.txt";
|
||||
const logFile = "color-discovery-log.json";
|
||||
|
||||
// Update this list if you find more color params in the site HTML.
|
||||
const colorParams = [
|
||||
"pris_black",
|
||||
"pris_blue",
|
||||
"pris_bronze",
|
||||
"pris_brown",
|
||||
"pris_clear",
|
||||
"pris_copper",
|
||||
"pris_gold",
|
||||
"pris_gray",
|
||||
"pris_green",
|
||||
"pris_orange",
|
||||
"pris_pink",
|
||||
"pris_purple",
|
||||
"pris_red",
|
||||
"pris_silver",
|
||||
"pris_tan",
|
||||
"pris_white",
|
||||
"pris_yellow"
|
||||
];
|
||||
|
||||
function cleanUrl(url) {
|
||||
return (url || "").split("?")[0].split("#")[0].trim();
|
||||
}
|
||||
|
||||
function isProductUrl(url) {
|
||||
return /\/shop\/powder-coating-colors\/[A-Z0-9-]+\//i.test(url || "");
|
||||
}
|
||||
|
||||
function readExistingUrls() {
|
||||
if (!fs.existsSync(outputFile)) return [];
|
||||
|
||||
return fs.readFileSync(outputFile, "utf8")
|
||||
.split(/\r?\n/)
|
||||
.map(cleanUrl)
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function writeUrls(urls) {
|
||||
const sorted = [...urls].sort();
|
||||
fs.writeFileSync(outputFile, sorted.join("\r\n") + "\r\n", "utf8");
|
||||
}
|
||||
|
||||
function readLog() {
|
||||
if (!fs.existsSync(logFile)) {
|
||||
return {
|
||||
completed_colors: {},
|
||||
runs: []
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(logFile, "utf8"));
|
||||
} catch {
|
||||
return {
|
||||
completed_colors: {},
|
||||
runs: []
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function writeLog(log) {
|
||||
fs.writeFileSync(logFile, JSON.stringify(log, null, 2), "utf8");
|
||||
}
|
||||
|
||||
async function collectProductLinks(page) {
|
||||
const links = await page.locator("a").evaluateAll(anchors =>
|
||||
anchors
|
||||
.map(a => a.href)
|
||||
.filter(Boolean)
|
||||
.filter(h => /\/shop\/powder-coating-colors\/[A-Z0-9-]+\//i.test(h))
|
||||
);
|
||||
|
||||
return links.map(cleanUrl).filter(Boolean);
|
||||
}
|
||||
|
||||
async function scrollAndCollect(page, urls, label) {
|
||||
let noNewScrolls = 0;
|
||||
let totalAddedForThisColor = 0;
|
||||
|
||||
for (let i = 0; i < maxScrollsPerColor; i++) {
|
||||
const before = urls.size;
|
||||
|
||||
for (const link of await collectProductLinks(page)) {
|
||||
urls.add(link);
|
||||
}
|
||||
|
||||
const after = urls.size;
|
||||
const added = after - before;
|
||||
totalAddedForThisColor += added;
|
||||
|
||||
if (added === 0) {
|
||||
noNewScrolls++;
|
||||
} else {
|
||||
noNewScrolls = 0;
|
||||
}
|
||||
|
||||
writeUrls(urls);
|
||||
|
||||
console.log(`[${label}] Scroll ${i + 1}/${maxScrollsPerColor}: +${added}, total ${after}, no-new ${noNewScrolls}`);
|
||||
|
||||
if (noNewScrolls >= stopAfterNoNewScrolls) {
|
||||
break;
|
||||
}
|
||||
|
||||
await page.mouse.wheel(0, 2500);
|
||||
await page.waitForTimeout(1500);
|
||||
}
|
||||
|
||||
return totalAddedForThisColor;
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const existingUrls = readExistingUrls();
|
||||
const urls = new Set(existingUrls);
|
||||
const log = readLog();
|
||||
|
||||
console.log(`Existing URLs in ${outputFile}: ${existingUrls.length}`);
|
||||
|
||||
const browser = await chromium.launch({ headless: !headed });
|
||||
|
||||
const context = await browser.newContext({
|
||||
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
viewport: { width: 1365, height: 900 },
|
||||
locale: "en-US",
|
||||
timezoneId: "America/New_York"
|
||||
});
|
||||
|
||||
const page = await context.newPage();
|
||||
|
||||
const runRecord = {
|
||||
started_at: new Date().toISOString(),
|
||||
existing_at_start: existingUrls.length,
|
||||
colors_attempted: []
|
||||
};
|
||||
|
||||
for (const color of colorParams) {
|
||||
if (log.completed_colors[color]) {
|
||||
console.log(`Skipping completed color: ${color}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const url = `${baseUrl}?color=${encodeURIComponent(color)}`;
|
||||
console.log("");
|
||||
console.log(`Opening color filter: ${color}`);
|
||||
console.log(url);
|
||||
|
||||
try {
|
||||
const response = await page.goto(url, {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: 60000
|
||||
});
|
||||
|
||||
const status = response ? response.status() : "unknown";
|
||||
console.log(`HTTP status: ${status}`);
|
||||
|
||||
await page.waitForTimeout(5000);
|
||||
|
||||
const before = urls.size;
|
||||
const addedDuringScroll = await scrollAndCollect(page, urls, color);
|
||||
const after = urls.size;
|
||||
const netAdded = after - before;
|
||||
|
||||
log.completed_colors[color] = {
|
||||
url,
|
||||
http_status: status,
|
||||
added: netAdded,
|
||||
added_during_scroll: addedDuringScroll,
|
||||
total_after: after,
|
||||
completed_at: new Date().toISOString()
|
||||
};
|
||||
|
||||
runRecord.colors_attempted.push({
|
||||
color,
|
||||
url,
|
||||
http_status: status,
|
||||
added: netAdded,
|
||||
total_after: after
|
||||
});
|
||||
|
||||
writeLog(log);
|
||||
writeUrls(urls);
|
||||
|
||||
console.log(`Color complete: ${color}; added ${netAdded}; total ${after}`);
|
||||
|
||||
// Polite pause between filters.
|
||||
await page.waitForTimeout(3000);
|
||||
} catch (err) {
|
||||
console.log(`Color failed: ${color}; ${err.message}`);
|
||||
|
||||
runRecord.colors_attempted.push({
|
||||
color,
|
||||
url,
|
||||
added: 0,
|
||||
error: err.message
|
||||
});
|
||||
|
||||
writeLog(log);
|
||||
}
|
||||
}
|
||||
|
||||
runRecord.finished_at = new Date().toISOString();
|
||||
runRecord.final_total = urls.size;
|
||||
runRecord.new_this_run = urls.size - existingUrls.length;
|
||||
|
||||
log.runs.push(runRecord);
|
||||
writeLog(log);
|
||||
writeUrls(urls);
|
||||
|
||||
console.log("");
|
||||
console.log("Color-param discovery complete.");
|
||||
console.log(`Existing at start: ${existingUrls.length}`);
|
||||
console.log(`Final total: ${urls.size}`);
|
||||
console.log(`New this run: ${urls.size - existingUrls.length}`);
|
||||
console.log(`Output: ${outputFile}`);
|
||||
console.log(`Log: ${logFile}`);
|
||||
|
||||
await browser.close();
|
||||
})();
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"name": "web-scraping",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "web-scraping",
|
||||
"version": "1.0.0",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"playwright": "^1.59.1"
|
||||
}
|
||||
},
|
||||
"node_modules/fsevents": {
|
||||
"version": "2.3.2",
|
||||
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
|
||||
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/playwright": {
|
||||
"version": "1.59.1",
|
||||
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz",
|
||||
"integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"playwright-core": "1.59.1"
|
||||
},
|
||||
"bin": {
|
||||
"playwright": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"fsevents": "2.3.2"
|
||||
}
|
||||
},
|
||||
"node_modules/playwright-core": {
|
||||
"version": "1.59.1",
|
||||
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz",
|
||||
"integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==",
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"playwright-core": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"name": "web-scraping",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"type": "commonjs",
|
||||
"dependencies": {
|
||||
"playwright": "^1.59.1"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
const fs = require("fs");
|
||||
const { chromium } = require("playwright");
|
||||
|
||||
const headed = process.argv.includes("--headed");
|
||||
|
||||
const productUrls = [
|
||||
"https://www.prismaticpowders.com/shop/powder-coating-colors/PSS-11248/high-gloss-black"
|
||||
];
|
||||
|
||||
const outputJson = "prismatic_powders.json";
|
||||
|
||||
function clean(text) {
|
||||
return (text || "").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function absoluteUrl(baseUrl, maybeUrl) {
|
||||
if (!maybeUrl) return "";
|
||||
try {
|
||||
return new URL(maybeUrl, baseUrl).href;
|
||||
} catch {
|
||||
return maybeUrl;
|
||||
}
|
||||
}
|
||||
|
||||
function unique(items) {
|
||||
return [...new Set(items.filter(Boolean).map(clean).filter(Boolean))];
|
||||
}
|
||||
|
||||
async function getLinkByText(page, patterns) {
|
||||
const links = await page.locator("a").evaluateAll((anchors) =>
|
||||
anchors.map(a => ({
|
||||
text: (a.innerText || a.textContent || "").replace(/\s+/g, " ").trim(),
|
||||
href: a.getAttribute("href") || ""
|
||||
}))
|
||||
);
|
||||
|
||||
for (const link of links) {
|
||||
if (patterns.some(p => new RegExp(p, "i").test(link.text))) {
|
||||
return absoluteUrl(page.url(), link.href);
|
||||
}
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
function parsePriceTiers(plainText) {
|
||||
const priceMatches = [...plainText.matchAll(/(\d+\s*-\s*\d+\s*lbs|\d+\s*\+\s*lbs)\s*\$([\d.]+)/gi)];
|
||||
|
||||
return priceMatches.map(m => {
|
||||
const rangeText = clean(m[1]);
|
||||
const price = parseFloat(m[2]);
|
||||
|
||||
let min = null;
|
||||
let max = null;
|
||||
|
||||
const rangeMatch = rangeText.match(/(\d+)\s*-\s*(\d+)/);
|
||||
if (rangeMatch) {
|
||||
min = parseInt(rangeMatch[1], 10);
|
||||
max = parseInt(rangeMatch[2], 10);
|
||||
}
|
||||
|
||||
const plusMatch = rangeText.match(/(\d+)\s*\+/);
|
||||
if (plusMatch) {
|
||||
min = parseInt(plusMatch[1], 10);
|
||||
max = null;
|
||||
}
|
||||
|
||||
return {
|
||||
min,
|
||||
max,
|
||||
price
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function getSampleImageUrl(page) {
|
||||
const imageUrls = await page.locator("img").evaluateAll((imgs) =>
|
||||
imgs.map(img =>
|
||||
img.currentSrc ||
|
||||
img.src ||
|
||||
img.getAttribute("src") ||
|
||||
img.getAttribute("data-src") ||
|
||||
""
|
||||
).filter(Boolean)
|
||||
);
|
||||
|
||||
return (
|
||||
imageUrls.find(src => /images\.nicindustries\.com/i.test(src) && !/thumbnail/i.test(src)) ||
|
||||
imageUrls.find(src => /images\.nicindustries\.com/i.test(src)) ||
|
||||
imageUrls.find(src => /prismatic|powder|color/i.test(src)) ||
|
||||
""
|
||||
);
|
||||
}
|
||||
|
||||
async function parseProduct(page, url) {
|
||||
console.log(`Scraping ${url}`);
|
||||
|
||||
const response = await page.goto(url, {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: 60000
|
||||
});
|
||||
|
||||
await page.waitForTimeout(3000);
|
||||
|
||||
const status = response ? response.status() : 0;
|
||||
const pageTitle = clean(await page.title().catch(() => ""));
|
||||
const plainText = clean(await page.locator("body").innerText().catch(() => ""));
|
||||
|
||||
console.log(`HTTP status: ${status}`);
|
||||
console.log(`Page title: ${pageTitle}`);
|
||||
|
||||
// Do not silently output a fake product if blocked.
|
||||
if (status === 403 || /^403 Forbidden$/i.test(pageTitle) || /^403 Forbidden$/i.test(plainText)) {
|
||||
throw new Error("403 Forbidden returned by site.");
|
||||
}
|
||||
|
||||
const title = clean(await page.locator("h1").first().innerText().catch(() => ""));
|
||||
|
||||
const skuMatch = plainText.match(/Item:\s*([A-Z0-9-]+)/i);
|
||||
const sku = skuMatch ? skuMatch[1] : "";
|
||||
|
||||
const descMatch = plainText.match(/Description:\s*(.*?)(WARNING:|What does this match\?|$)/is);
|
||||
const description = descMatch ? clean(descMatch[1]) : "";
|
||||
|
||||
const priceTiers = parsePriceTiers(plainText);
|
||||
|
||||
const safetyDataSheetUrl = await getLinkByText(page, ["Safety Data Sheet", "\\bSDS\\b"]);
|
||||
const applicationGuideUrl = await getLinkByText(page, ["Application Guide"]);
|
||||
const technicalDataSheetUrl = await getLinkByText(page, ["Tech Data Sheet", "Technical Data Sheet", "\\bTDS\\b"]);
|
||||
const sampleImageUrl = await getSampleImageUrl(page);
|
||||
|
||||
return {
|
||||
sku,
|
||||
color_name: title,
|
||||
description,
|
||||
price_tiers: priceTiers,
|
||||
safety_data_sheet_url: safetyDataSheetUrl,
|
||||
technical_data_sheet_url: technicalDataSheetUrl,
|
||||
application_guide_url: applicationGuideUrl,
|
||||
sample_image_url: sampleImageUrl,
|
||||
product_url: url,
|
||||
scraped_at: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const browser = await chromium.launch({
|
||||
headless: !headed
|
||||
});
|
||||
|
||||
const context = await browser.newContext({
|
||||
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
viewport: { width: 1365, height: 900 },
|
||||
locale: "en-US",
|
||||
timezoneId: "America/New_York"
|
||||
});
|
||||
|
||||
const page = await context.newPage();
|
||||
|
||||
const results = [];
|
||||
const errors = [];
|
||||
|
||||
for (const url of productUrls) {
|
||||
try {
|
||||
const row = await parseProduct(page, url);
|
||||
results.push(row);
|
||||
await page.waitForTimeout(3000);
|
||||
} catch (err) {
|
||||
console.warn(`Failed ${url}: ${err.message}`);
|
||||
errors.push({
|
||||
product_url: url,
|
||||
error: err.message,
|
||||
scraped_at: new Date().toISOString()
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
|
||||
// If you prefer only the array, change this to JSON.stringify(results, null, 2)
|
||||
const output = {
|
||||
results,
|
||||
errors
|
||||
};
|
||||
|
||||
fs.writeFileSync(outputJson, JSON.stringify(output, null, 2), "utf8");
|
||||
|
||||
console.log(`Done. Output: ${outputJson}`);
|
||||
})();
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"results": [
|
||||
{
|
||||
"sku": "PSS-11248",
|
||||
"color_name": "High Gloss Black",
|
||||
"description": "Prismatic Powders developed High Gloss Black to be the only high gloss black powder coating you will ever need. It has an incredibly deep, mirror like finish with amazing flow out, yielding the highest gloss, true black finish available in a powder coating. High Gloss Black is a polyester solid tone and is the best option to use as a base coat with all of our clear metallics, because of its mirror-like finish. Gloss Units: 85+ Proposition 65 Warning",
|
||||
"price_tiers": [
|
||||
{
|
||||
"min": 1,
|
||||
"max": 49,
|
||||
"price": 12.47
|
||||
},
|
||||
{
|
||||
"min": 50,
|
||||
"max": 199,
|
||||
"price": 11.85
|
||||
},
|
||||
{
|
||||
"min": 200,
|
||||
"max": null,
|
||||
"price": 11.22
|
||||
}
|
||||
],
|
||||
"safety_data_sheet_url": "https://images.nicindustries.com/prismatic/documents/8099/prismatic-powders-p-series-sds-dt20260126212318272746.pdf?1769462600",
|
||||
"technical_data_sheet_url": "https://images.nicindustries.com/prismatic/documents/5536/pss-11248-high-gloss-black-tds-dt20240111230300658308.pdf?1705014182",
|
||||
"application_guide_url": "https://images.nicindustries.com/prismatic/documents/2274/prismatic-powders-application-guide-dt20230508192819506132.pdf?1683574101",
|
||||
"sample_image_url": "https://images.nicindustries.com/prismatic/products/15027/high-gloss-black-pss-11248-dt20251107212621375559-thumbnail.jpg?size=600",
|
||||
"product_url": "https://www.prismaticpowders.com/shop/powder-coating-colors/PSS-11248/high-gloss-black",
|
||||
"scraped_at": "2026-04-30T12:52:36.244Z"
|
||||
}
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Binary file not shown.
@@ -0,0 +1,270 @@
|
||||
{
|
||||
"completed_colors": {
|
||||
"pris_black": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_black",
|
||||
"http_status": 200,
|
||||
"added": 472,
|
||||
"added_during_scroll": 472,
|
||||
"total_after": 472,
|
||||
"completed_at": "2026-04-30T00:47:46.289Z"
|
||||
},
|
||||
"pris_blue": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_blue",
|
||||
"http_status": 200,
|
||||
"added": 948,
|
||||
"added_during_scroll": 948,
|
||||
"total_after": 1420,
|
||||
"completed_at": "2026-04-30T00:49:25.145Z"
|
||||
},
|
||||
"pris_bronze": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_bronze",
|
||||
"http_status": 200,
|
||||
"added": 358,
|
||||
"added_during_scroll": 358,
|
||||
"total_after": 1778,
|
||||
"completed_at": "2026-04-30T00:50:18.466Z"
|
||||
},
|
||||
"pris_brown": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_brown",
|
||||
"http_status": 200,
|
||||
"added": 373,
|
||||
"added_during_scroll": 373,
|
||||
"total_after": 2151,
|
||||
"completed_at": "2026-04-30T00:51:18.033Z"
|
||||
},
|
||||
"pris_clear": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_clear",
|
||||
"http_status": 200,
|
||||
"added": 19,
|
||||
"added_during_scroll": 19,
|
||||
"total_after": 2170,
|
||||
"completed_at": "2026-04-30T00:51:42.889Z"
|
||||
},
|
||||
"pris_copper": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_copper",
|
||||
"http_status": 200,
|
||||
"added": 1094,
|
||||
"added_during_scroll": 1094,
|
||||
"total_after": 3264,
|
||||
"completed_at": "2026-04-30T00:56:34.934Z"
|
||||
},
|
||||
"pris_gold": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_gold",
|
||||
"http_status": 200,
|
||||
"added": 152,
|
||||
"added_during_scroll": 152,
|
||||
"total_after": 3416,
|
||||
"completed_at": "2026-04-30T00:57:26.775Z"
|
||||
},
|
||||
"pris_gray": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_gray",
|
||||
"http_status": 200,
|
||||
"added": 0,
|
||||
"added_during_scroll": 0,
|
||||
"total_after": 3416,
|
||||
"completed_at": "2026-04-30T00:57:49.624Z"
|
||||
},
|
||||
"pris_green": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_green",
|
||||
"http_status": 200,
|
||||
"added": 0,
|
||||
"added_during_scroll": 0,
|
||||
"total_after": 3416,
|
||||
"completed_at": "2026-04-30T00:58:12.277Z"
|
||||
},
|
||||
"pris_orange": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_orange",
|
||||
"http_status": 200,
|
||||
"added": 233,
|
||||
"added_during_scroll": 233,
|
||||
"total_after": 3649,
|
||||
"completed_at": "2026-04-30T00:59:06.776Z"
|
||||
},
|
||||
"pris_pink": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_pink",
|
||||
"http_status": 200,
|
||||
"added": 169,
|
||||
"added_during_scroll": 169,
|
||||
"total_after": 3818,
|
||||
"completed_at": "2026-04-30T00:59:49.323Z"
|
||||
},
|
||||
"pris_purple": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_purple",
|
||||
"http_status": 200,
|
||||
"added": 182,
|
||||
"added_during_scroll": 182,
|
||||
"total_after": 4000,
|
||||
"completed_at": "2026-04-30T01:00:38.111Z"
|
||||
},
|
||||
"pris_red": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_red",
|
||||
"http_status": 200,
|
||||
"added": 346,
|
||||
"added_during_scroll": 346,
|
||||
"total_after": 4346,
|
||||
"completed_at": "2026-04-30T01:01:51.910Z"
|
||||
},
|
||||
"pris_silver": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_silver",
|
||||
"http_status": 200,
|
||||
"added": 210,
|
||||
"added_during_scroll": 210,
|
||||
"total_after": 4556,
|
||||
"completed_at": "2026-04-30T01:02:51.835Z"
|
||||
},
|
||||
"pris_tan": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_tan",
|
||||
"http_status": 200,
|
||||
"added": 219,
|
||||
"added_during_scroll": 219,
|
||||
"total_after": 4775,
|
||||
"completed_at": "2026-04-30T01:03:43.244Z"
|
||||
},
|
||||
"pris_white": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_white",
|
||||
"http_status": 200,
|
||||
"added": 218,
|
||||
"added_during_scroll": 218,
|
||||
"total_after": 4993,
|
||||
"completed_at": "2026-04-30T01:04:39.931Z"
|
||||
},
|
||||
"pris_yellow": {
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_yellow",
|
||||
"http_status": 200,
|
||||
"added": 199,
|
||||
"added_during_scroll": 199,
|
||||
"total_after": 5192,
|
||||
"completed_at": "2026-04-30T01:05:31.945Z"
|
||||
}
|
||||
},
|
||||
"runs": [
|
||||
{
|
||||
"started_at": "2026-04-30T00:46:47.692Z",
|
||||
"existing_at_start": 0,
|
||||
"colors_attempted": [
|
||||
{
|
||||
"color": "pris_black",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_black",
|
||||
"http_status": 200,
|
||||
"added": 472,
|
||||
"total_after": 472
|
||||
},
|
||||
{
|
||||
"color": "pris_blue",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_blue",
|
||||
"http_status": 200,
|
||||
"added": 948,
|
||||
"total_after": 1420
|
||||
},
|
||||
{
|
||||
"color": "pris_bronze",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_bronze",
|
||||
"http_status": 200,
|
||||
"added": 358,
|
||||
"total_after": 1778
|
||||
},
|
||||
{
|
||||
"color": "pris_brown",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_brown",
|
||||
"http_status": 200,
|
||||
"added": 373,
|
||||
"total_after": 2151
|
||||
},
|
||||
{
|
||||
"color": "pris_clear",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_clear",
|
||||
"http_status": 200,
|
||||
"added": 19,
|
||||
"total_after": 2170
|
||||
},
|
||||
{
|
||||
"color": "pris_copper",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_copper",
|
||||
"http_status": 200,
|
||||
"added": 1094,
|
||||
"total_after": 3264
|
||||
},
|
||||
{
|
||||
"color": "pris_gold",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_gold",
|
||||
"http_status": 200,
|
||||
"added": 152,
|
||||
"total_after": 3416
|
||||
},
|
||||
{
|
||||
"color": "pris_gray",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_gray",
|
||||
"http_status": 200,
|
||||
"added": 0,
|
||||
"total_after": 3416
|
||||
},
|
||||
{
|
||||
"color": "pris_green",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_green",
|
||||
"http_status": 200,
|
||||
"added": 0,
|
||||
"total_after": 3416
|
||||
},
|
||||
{
|
||||
"color": "pris_orange",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_orange",
|
||||
"http_status": 200,
|
||||
"added": 233,
|
||||
"total_after": 3649
|
||||
},
|
||||
{
|
||||
"color": "pris_pink",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_pink",
|
||||
"http_status": 200,
|
||||
"added": 169,
|
||||
"total_after": 3818
|
||||
},
|
||||
{
|
||||
"color": "pris_purple",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_purple",
|
||||
"http_status": 200,
|
||||
"added": 182,
|
||||
"total_after": 4000
|
||||
},
|
||||
{
|
||||
"color": "pris_red",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_red",
|
||||
"http_status": 200,
|
||||
"added": 346,
|
||||
"total_after": 4346
|
||||
},
|
||||
{
|
||||
"color": "pris_silver",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_silver",
|
||||
"http_status": 200,
|
||||
"added": 210,
|
||||
"total_after": 4556
|
||||
},
|
||||
{
|
||||
"color": "pris_tan",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_tan",
|
||||
"http_status": 200,
|
||||
"added": 219,
|
||||
"total_after": 4775
|
||||
},
|
||||
{
|
||||
"color": "pris_white",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_white",
|
||||
"http_status": 200,
|
||||
"added": 218,
|
||||
"total_after": 4993
|
||||
},
|
||||
{
|
||||
"color": "pris_yellow",
|
||||
"url": "https://www.prismaticpowders.com/shop/powder-coating-colors?color=pris_yellow",
|
||||
"http_status": 200,
|
||||
"added": 199,
|
||||
"total_after": 5192
|
||||
}
|
||||
],
|
||||
"finished_at": "2026-04-30T01:05:34.987Z",
|
||||
"final_total": 5192,
|
||||
"new_this_run": 5192
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
const fs = require("fs");
|
||||
const { chromium } = require("playwright");
|
||||
|
||||
const headed = process.argv.includes("--headed");
|
||||
|
||||
function getArgValue(name, defaultValue) {
|
||||
const prefix = `--${name}=`;
|
||||
const found = process.argv.find(x => x.startsWith(prefix));
|
||||
return found ? found.slice(prefix.length) : defaultValue;
|
||||
}
|
||||
|
||||
const maxScrollsPerColor = parseInt(getArgValue("max-scrolls-per-color", "180"), 10);
|
||||
const stopAfterNoNewScrolls = parseInt(getArgValue("stop-after-no-new-scrolls", "10"), 10);
|
||||
|
||||
const baseUrl = "https://www.prismaticpowders.com/shop/powder-coating-colors";
|
||||
const outputFile = "product-urls.txt";
|
||||
const logFile = "color-discovery-log.json";
|
||||
|
||||
// Update this list if you find more color params in the site HTML.
|
||||
const colorParams = [
|
||||
"pris_black",
|
||||
"pris_blue",
|
||||
"pris_bronze",
|
||||
"pris_brown",
|
||||
"pris_clear",
|
||||
"pris_copper",
|
||||
"pris_gold",
|
||||
"pris_gray",
|
||||
"pris_green",
|
||||
"pris_orange",
|
||||
"pris_pink",
|
||||
"pris_purple",
|
||||
"pris_red",
|
||||
"pris_silver",
|
||||
"pris_tan",
|
||||
"pris_white",
|
||||
"pris_yellow"
|
||||
];
|
||||
|
||||
function cleanUrl(url) {
|
||||
return (url || "").split("?")[0].split("#")[0].trim();
|
||||
}
|
||||
|
||||
function isProductUrl(url) {
|
||||
return /\/shop\/powder-coating-colors\/[A-Z0-9-]+\//i.test(url || "");
|
||||
}
|
||||
|
||||
function readExistingUrls() {
|
||||
if (!fs.existsSync(outputFile)) return [];
|
||||
|
||||
return fs.readFileSync(outputFile, "utf8")
|
||||
.split(/\r?\n/)
|
||||
.map(cleanUrl)
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function writeUrls(urls) {
|
||||
const sorted = [...urls].sort();
|
||||
fs.writeFileSync(outputFile, sorted.join("\r\n") + "\r\n", "utf8");
|
||||
}
|
||||
|
||||
function readLog() {
|
||||
if (!fs.existsSync(logFile)) {
|
||||
return {
|
||||
completed_colors: {},
|
||||
runs: []
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(logFile, "utf8"));
|
||||
} catch {
|
||||
return {
|
||||
completed_colors: {},
|
||||
runs: []
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function writeLog(log) {
|
||||
fs.writeFileSync(logFile, JSON.stringify(log, null, 2), "utf8");
|
||||
}
|
||||
|
||||
async function collectProductLinks(page) {
|
||||
const links = await page.locator("a").evaluateAll(anchors =>
|
||||
anchors
|
||||
.map(a => a.href)
|
||||
.filter(Boolean)
|
||||
.filter(h => /\/shop\/powder-coating-colors\/[A-Z0-9-]+\//i.test(h))
|
||||
);
|
||||
|
||||
return links.map(cleanUrl).filter(Boolean);
|
||||
}
|
||||
|
||||
async function scrollAndCollect(page, urls, label) {
|
||||
let noNewScrolls = 0;
|
||||
let totalAddedForThisColor = 0;
|
||||
|
||||
for (let i = 0; i < maxScrollsPerColor; i++) {
|
||||
const before = urls.size;
|
||||
|
||||
for (const link of await collectProductLinks(page)) {
|
||||
urls.add(link);
|
||||
}
|
||||
|
||||
const after = urls.size;
|
||||
const added = after - before;
|
||||
totalAddedForThisColor += added;
|
||||
|
||||
if (added === 0) {
|
||||
noNewScrolls++;
|
||||
} else {
|
||||
noNewScrolls = 0;
|
||||
}
|
||||
|
||||
writeUrls(urls);
|
||||
|
||||
console.log(`[${label}] Scroll ${i + 1}/${maxScrollsPerColor}: +${added}, total ${after}, no-new ${noNewScrolls}`);
|
||||
|
||||
if (noNewScrolls >= stopAfterNoNewScrolls) {
|
||||
break;
|
||||
}
|
||||
|
||||
await page.mouse.wheel(0, 2500);
|
||||
await page.waitForTimeout(1500);
|
||||
}
|
||||
|
||||
return totalAddedForThisColor;
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const existingUrls = readExistingUrls();
|
||||
const urls = new Set(existingUrls);
|
||||
const log = readLog();
|
||||
|
||||
console.log(`Existing URLs in ${outputFile}: ${existingUrls.length}`);
|
||||
|
||||
const browser = await chromium.launch({ headless: !headed });
|
||||
|
||||
const context = await browser.newContext({
|
||||
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
viewport: { width: 1365, height: 900 },
|
||||
locale: "en-US",
|
||||
timezoneId: "America/New_York"
|
||||
});
|
||||
|
||||
const page = await context.newPage();
|
||||
|
||||
const runRecord = {
|
||||
started_at: new Date().toISOString(),
|
||||
existing_at_start: existingUrls.length,
|
||||
colors_attempted: []
|
||||
};
|
||||
|
||||
for (const color of colorParams) {
|
||||
if (log.completed_colors[color]) {
|
||||
console.log(`Skipping completed color: ${color}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const url = `${baseUrl}?color=${encodeURIComponent(color)}`;
|
||||
console.log("");
|
||||
console.log(`Opening color filter: ${color}`);
|
||||
console.log(url);
|
||||
|
||||
try {
|
||||
const response = await page.goto(url, {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: 60000
|
||||
});
|
||||
|
||||
const status = response ? response.status() : "unknown";
|
||||
console.log(`HTTP status: ${status}`);
|
||||
|
||||
await page.waitForTimeout(5000);
|
||||
|
||||
const before = urls.size;
|
||||
const addedDuringScroll = await scrollAndCollect(page, urls, color);
|
||||
const after = urls.size;
|
||||
const netAdded = after - before;
|
||||
|
||||
log.completed_colors[color] = {
|
||||
url,
|
||||
http_status: status,
|
||||
added: netAdded,
|
||||
added_during_scroll: addedDuringScroll,
|
||||
total_after: after,
|
||||
completed_at: new Date().toISOString()
|
||||
};
|
||||
|
||||
runRecord.colors_attempted.push({
|
||||
color,
|
||||
url,
|
||||
http_status: status,
|
||||
added: netAdded,
|
||||
total_after: after
|
||||
});
|
||||
|
||||
writeLog(log);
|
||||
writeUrls(urls);
|
||||
|
||||
console.log(`Color complete: ${color}; added ${netAdded}; total ${after}`);
|
||||
|
||||
// Polite pause between filters.
|
||||
await page.waitForTimeout(3000);
|
||||
} catch (err) {
|
||||
console.log(`Color failed: ${color}; ${err.message}`);
|
||||
|
||||
runRecord.colors_attempted.push({
|
||||
color,
|
||||
url,
|
||||
added: 0,
|
||||
error: err.message
|
||||
});
|
||||
|
||||
writeLog(log);
|
||||
}
|
||||
}
|
||||
|
||||
runRecord.finished_at = new Date().toISOString();
|
||||
runRecord.final_total = urls.size;
|
||||
runRecord.new_this_run = urls.size - existingUrls.length;
|
||||
|
||||
log.runs.push(runRecord);
|
||||
writeLog(log);
|
||||
writeUrls(urls);
|
||||
|
||||
console.log("");
|
||||
console.log("Color-param discovery complete.");
|
||||
console.log(`Existing at start: ${existingUrls.length}`);
|
||||
console.log(`Final total: ${urls.size}`);
|
||||
console.log(`New this run: ${urls.size - existingUrls.length}`);
|
||||
console.log(`Output: ${outputFile}`);
|
||||
console.log(`Log: ${logFile}`);
|
||||
|
||||
await browser.close();
|
||||
})();
|
||||
Generated
+60
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"name": "web-scraping",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "web-scraping",
|
||||
"version": "1.0.0",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"playwright": "^1.59.1"
|
||||
}
|
||||
},
|
||||
"node_modules/fsevents": {
|
||||
"version": "2.3.2",
|
||||
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
|
||||
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/playwright": {
|
||||
"version": "1.59.1",
|
||||
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz",
|
||||
"integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"playwright-core": "1.59.1"
|
||||
},
|
||||
"bin": {
|
||||
"playwright": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"fsevents": "2.3.2"
|
||||
}
|
||||
},
|
||||
"node_modules/playwright-core": {
|
||||
"version": "1.59.1",
|
||||
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz",
|
||||
"integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==",
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"playwright-core": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"name": "web-scraping",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"type": "commonjs",
|
||||
"dependencies": {
|
||||
"playwright": "^1.59.1"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
const fs = require("fs");
|
||||
const { chromium } = require("playwright");
|
||||
|
||||
const headed = process.argv.includes("--headed");
|
||||
const retryErrors = process.argv.includes("--retry-errors");
|
||||
|
||||
function getArgValue(name, defaultValue) {
|
||||
const prefix = `--${name}=`;
|
||||
const found = process.argv.find(x => x.startsWith(prefix));
|
||||
return found ? found.slice(prefix.length) : defaultValue;
|
||||
}
|
||||
|
||||
const inputFile = getArgValue("input-file", "product-urls.txt");
|
||||
const outputJson = getArgValue("output-json", "prismatic_powders.json");
|
||||
const progressLog = getArgValue("progress-log", "prismatic-scrape-progress.log");
|
||||
|
||||
const minDelaySeconds = parseInt(getArgValue("min-delay-seconds", "8"), 10);
|
||||
const maxDelaySeconds = parseInt(getArgValue("max-delay-seconds", "18"), 10);
|
||||
const pageSettleSeconds = parseInt(getArgValue("page-settle-seconds", "4"), 10);
|
||||
const maxProducts = parseInt(getArgValue("max-products", "0"), 10);
|
||||
|
||||
function clean(text) {
|
||||
return (text || "").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function cleanUrl(url) {
|
||||
return (url || "").split("?")[0].split("#")[0].trim();
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function randomDelayMs() {
|
||||
const minMs = Math.max(0, minDelaySeconds * 1000);
|
||||
const maxMs = Math.max(minMs, maxDelaySeconds * 1000);
|
||||
return Math.floor(minMs + Math.random() * (maxMs - minMs + 1));
|
||||
}
|
||||
|
||||
function logLine(message) {
|
||||
const line = `[${new Date().toISOString()}] ${message}`;
|
||||
console.log(line);
|
||||
fs.appendFileSync(progressLog, line + "\r\n", "utf8");
|
||||
}
|
||||
|
||||
function absoluteUrl(baseUrl, maybeUrl) {
|
||||
if (!maybeUrl) return "";
|
||||
try {
|
||||
return new URL(maybeUrl, baseUrl).href;
|
||||
} catch {
|
||||
return maybeUrl;
|
||||
}
|
||||
}
|
||||
|
||||
function loadInputUrls() {
|
||||
if (!fs.existsSync(inputFile)) {
|
||||
throw new Error(`Input file not found: ${inputFile}`);
|
||||
}
|
||||
|
||||
const urls = fs.readFileSync(inputFile, "utf8")
|
||||
.split(/\r?\n/)
|
||||
.map(cleanUrl)
|
||||
.filter(Boolean)
|
||||
.filter(x => !x.startsWith("#"))
|
||||
.filter(x => /\/shop\/powder-coating-colors\/[A-Z0-9-]+\//i.test(x));
|
||||
|
||||
return [...new Set(urls)];
|
||||
}
|
||||
|
||||
function loadOutput() {
|
||||
if (!fs.existsSync(outputJson)) {
|
||||
return { results: [], errors: [] };
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(fs.readFileSync(outputJson, "utf8"));
|
||||
|
||||
if (Array.isArray(parsed)) {
|
||||
return { results: parsed, errors: [] };
|
||||
}
|
||||
|
||||
return {
|
||||
results: Array.isArray(parsed.results) ? parsed.results : [],
|
||||
errors: Array.isArray(parsed.errors) ? parsed.errors : []
|
||||
};
|
||||
} catch (err) {
|
||||
const backup = `${outputJson}.invalid-${Date.now()}.bak`;
|
||||
fs.copyFileSync(outputJson, backup);
|
||||
throw new Error(`Could not parse existing ${outputJson}. Backed it up to ${backup}. Error: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function saveOutput(data) {
|
||||
const tempFile = `${outputJson}.tmp`;
|
||||
fs.writeFileSync(tempFile, JSON.stringify(data, null, 2), "utf8");
|
||||
fs.renameSync(tempFile, outputJson);
|
||||
}
|
||||
|
||||
function parsePriceTiers(plainText) {
|
||||
const priceMatches = [...plainText.matchAll(/(\d+\s*-\s*\d+\s*lbs|\d+\s*\+\s*lbs)\s*\$([\d.]+)/gi)];
|
||||
|
||||
return priceMatches.map(m => {
|
||||
const rangeText = clean(m[1]);
|
||||
const price = parseFloat(m[2]);
|
||||
|
||||
let min = null;
|
||||
let max = null;
|
||||
|
||||
const rangeMatch = rangeText.match(/(\d+)\s*-\s*(\d+)/);
|
||||
if (rangeMatch) {
|
||||
min = parseInt(rangeMatch[1], 10);
|
||||
max = parseInt(rangeMatch[2], 10);
|
||||
}
|
||||
|
||||
const plusMatch = rangeText.match(/(\d+)\s*\+/);
|
||||
if (plusMatch) {
|
||||
min = parseInt(plusMatch[1], 10);
|
||||
max = null;
|
||||
}
|
||||
|
||||
return { min, max, price };
|
||||
});
|
||||
}
|
||||
|
||||
async function getLinkByText(page, patterns) {
|
||||
const links = await page.locator("a").evaluateAll((anchors) =>
|
||||
anchors.map(a => ({
|
||||
text: (a.innerText || a.textContent || "").replace(/\s+/g, " ").trim(),
|
||||
href: a.getAttribute("href") || ""
|
||||
}))
|
||||
);
|
||||
|
||||
for (const link of links) {
|
||||
if (patterns.some(p => new RegExp(p, "i").test(link.text))) {
|
||||
return absoluteUrl(page.url(), link.href);
|
||||
}
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
async function getSampleImageUrl(page) {
|
||||
const imageUrls = await page.locator("img").evaluateAll((imgs) =>
|
||||
imgs.map(img =>
|
||||
img.currentSrc ||
|
||||
img.src ||
|
||||
img.getAttribute("src") ||
|
||||
img.getAttribute("data-src") ||
|
||||
""
|
||||
).filter(Boolean)
|
||||
);
|
||||
|
||||
return (
|
||||
imageUrls.find(src => /images\.nicindustries\.com/i.test(src) && !/thumbnail/i.test(src)) ||
|
||||
imageUrls.find(src => /images\.nicindustries\.com/i.test(src)) ||
|
||||
imageUrls.find(src => /prismatic|powder|color/i.test(src)) ||
|
||||
""
|
||||
);
|
||||
}
|
||||
|
||||
async function parseProduct(page, url) {
|
||||
logLine(`Scraping ${url}`);
|
||||
|
||||
const response = await page.goto(url, {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: 60000
|
||||
});
|
||||
|
||||
await page.waitForTimeout(pageSettleSeconds * 1000);
|
||||
|
||||
const status = response ? response.status() : 0;
|
||||
const pageTitle = clean(await page.title().catch(() => ""));
|
||||
const plainText = clean(await page.locator("body").innerText().catch(() => ""));
|
||||
|
||||
logLine(`HTTP status ${status}; title "${pageTitle}"`);
|
||||
|
||||
if (status === 403 || /^403 Forbidden$/i.test(pageTitle) || /^403 Forbidden$/i.test(plainText)) {
|
||||
throw new Error("403 Forbidden returned by site.");
|
||||
}
|
||||
|
||||
if (status === 404 || /404|Page Not Found/i.test(pageTitle)) {
|
||||
throw new Error("404 Not Found returned by site.");
|
||||
}
|
||||
|
||||
const title = clean(await page.locator("h1").first().innerText().catch(() => ""));
|
||||
|
||||
const skuMatch = plainText.match(/Item:\s*([A-Z0-9-]+)/i);
|
||||
const sku = skuMatch ? skuMatch[1] : "";
|
||||
|
||||
if (!sku && !title) {
|
||||
throw new Error("Could not find SKU or title on product page.");
|
||||
}
|
||||
|
||||
const descMatch = plainText.match(/Description:\s*(.*?)(WARNING:|What does this match\?|$)/is);
|
||||
const description = descMatch ? clean(descMatch[1]) : "";
|
||||
|
||||
const priceTiers = parsePriceTiers(plainText);
|
||||
|
||||
const safetyDataSheetUrl = await getLinkByText(page, ["Safety Data Sheet", "\\bSDS\\b"]);
|
||||
const applicationGuideUrl = await getLinkByText(page, ["Application Guide"]);
|
||||
const technicalDataSheetUrl = await getLinkByText(page, ["Tech Data Sheet", "Technical Data Sheet", "\\bTDS\\b"]);
|
||||
const sampleImageUrl = await getSampleImageUrl(page);
|
||||
|
||||
return {
|
||||
sku,
|
||||
color_name: title,
|
||||
description,
|
||||
price_tiers: priceTiers,
|
||||
safety_data_sheet_url: safetyDataSheetUrl,
|
||||
technical_data_sheet_url: technicalDataSheetUrl,
|
||||
application_guide_url: applicationGuideUrl,
|
||||
sample_image_url: sampleImageUrl,
|
||||
product_url: url,
|
||||
scraped_at: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const allUrls = loadInputUrls();
|
||||
const data = loadOutput();
|
||||
|
||||
const completedUrls = new Set(data.results.map(r => cleanUrl(r.product_url)).filter(Boolean));
|
||||
const errorUrls = new Set(data.errors.map(e => cleanUrl(e.product_url)).filter(Boolean));
|
||||
|
||||
let remainingUrls = allUrls.filter(url => {
|
||||
if (completedUrls.has(url)) return false;
|
||||
if (!retryErrors && errorUrls.has(url)) return false;
|
||||
return true;
|
||||
});
|
||||
|
||||
if (maxProducts > 0) {
|
||||
remainingUrls = remainingUrls.slice(0, maxProducts);
|
||||
}
|
||||
|
||||
logLine(`Input URLs: ${allUrls.length}`);
|
||||
logLine(`Already scraped: ${completedUrls.size}`);
|
||||
logLine(`Existing errors: ${errorUrls.size}`);
|
||||
logLine(`Retry errors: ${retryErrors ? "yes" : "no"}`);
|
||||
logLine(`This run target count: ${remainingUrls.length}`);
|
||||
logLine(`Delay range: ${minDelaySeconds}-${maxDelaySeconds} seconds; page settle: ${pageSettleSeconds} seconds`);
|
||||
|
||||
if (remainingUrls.length === 0) {
|
||||
logLine("Nothing to scrape. Done.");
|
||||
saveOutput(data);
|
||||
return;
|
||||
}
|
||||
|
||||
const browser = await chromium.launch({
|
||||
headless: !headed
|
||||
});
|
||||
|
||||
const context = await browser.newContext({
|
||||
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
viewport: { width: 1365, height: 900 },
|
||||
locale: "en-US",
|
||||
timezoneId: "America/New_York"
|
||||
});
|
||||
|
||||
const page = await context.newPage();
|
||||
|
||||
let processedThisRun = 0;
|
||||
|
||||
for (const url of remainingUrls) {
|
||||
try {
|
||||
const row = await parseProduct(page, url);
|
||||
|
||||
// If retrying an old error, keep the old error history but avoid duplicate successful result.
|
||||
if (!completedUrls.has(url)) {
|
||||
data.results.push(row);
|
||||
completedUrls.add(url);
|
||||
}
|
||||
|
||||
processedThisRun++;
|
||||
saveOutput(data);
|
||||
|
||||
logLine(`Saved result ${processedThisRun}/${remainingUrls.length}: ${row.sku || "(no sku)"} ${row.color_name || ""}`);
|
||||
} catch (err) {
|
||||
const errorRecord = {
|
||||
product_url: url,
|
||||
error: err.message,
|
||||
scraped_at: new Date().toISOString()
|
||||
};
|
||||
|
||||
data.errors.push(errorRecord);
|
||||
saveOutput(data);
|
||||
|
||||
logLine(`ERROR ${url}: ${err.message}`);
|
||||
}
|
||||
|
||||
const delay = randomDelayMs();
|
||||
logLine(`Waiting ${(delay / 1000).toFixed(1)} seconds before next product...`);
|
||||
await sleep(delay);
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
|
||||
logLine(`Done. Results: ${data.results.length}; Errors: ${data.errors.length}; Output: ${outputJson}`);
|
||||
})();
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
@@ -252,11 +252,27 @@ public class PricingCalculationService : IPricingCalculationService
|
||||
};
|
||||
}
|
||||
|
||||
// AI items use ManualUnitPrice directly (set to either the AI estimate or the user's price override).
|
||||
// The AI already factored in all costs — skip the pricing engine entirely.
|
||||
// AI items use ManualUnitPrice as the single-coat base price.
|
||||
// Apply the same additional-coat charge as the calculated-item path so that
|
||||
// adding a 2nd or 3rd coat in step 3 increases the price by AdditionalCoatLaborPercent%
|
||||
// per coat — matching what catalog/calculated items charge.
|
||||
if (item.IsAiItem && item.ManualUnitPrice.HasValue)
|
||||
{
|
||||
var aiUnitPrice = item.ManualUnitPrice.Value;
|
||||
|
||||
int additionalAiCoats = 0;
|
||||
if (item.Coats != null)
|
||||
{
|
||||
for (int i = 1; i < item.Coats.Count; i++)
|
||||
{
|
||||
if (!item.Coats[i].NoExtraLayerCharge)
|
||||
additionalAiCoats++;
|
||||
}
|
||||
}
|
||||
if (additionalAiCoats > 0)
|
||||
aiUnitPrice = Math.Round(
|
||||
aiUnitPrice * (1m + additionalAiCoats * (costs.AdditionalCoatLaborPercent / 100m)), 2);
|
||||
|
||||
var aiTotal = aiUnitPrice * item.Quantity;
|
||||
return new QuoteItemPricingResult
|
||||
{
|
||||
|
||||
@@ -540,6 +540,16 @@ Respond with the JSON object only.";
|
||||
var complexityCharge = subtotalBeforeComplexity * complexityPct;
|
||||
var subtotal = subtotalBeforeComplexity + complexityCharge;
|
||||
|
||||
// Additional coat charge — each coat beyond the first adds AdditionalCoatLaborPercent % of
|
||||
// the subtotal, matching the formula in PricingCalculationService.CalculateQuoteItemPriceAsync.
|
||||
// The coat count here comes from the wizard's step-2 field (aiCoatCount), so the preview
|
||||
// reflects whatever multi-coat configuration the user specified before clicking Analyze.
|
||||
if (request.CoatCount > 1)
|
||||
{
|
||||
var additionalCoatCharge = subtotal * (request.CoatCount - 1) * (costs.AdditionalCoatLaborPercent / 100m);
|
||||
subtotal += additionalCoatCharge;
|
||||
}
|
||||
|
||||
var markupAmount = (materialCost + consumablesSurcharge) * (costs.GeneralMarkupPercentage / 100m);
|
||||
|
||||
// Apply shop minimum
|
||||
|
||||
@@ -1029,12 +1029,13 @@ public class InventoryController : Controller
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Searches the platform-level PowderCatalogItems table by SKU or color name and returns
|
||||
/// up to 10 matches as JSON. Called by the inventory Create/Edit form before falling back
|
||||
/// to the AI Lookup, avoiding unnecessary API calls for known products.
|
||||
/// Searches the platform-level PowderCatalogItems table by SKU or color name.
|
||||
/// Excludes catalog entries already present in the company's inventory (by ManufacturerPartNumber).
|
||||
/// Pass currentId when editing an existing item so its own catalog entry is not filtered out.
|
||||
/// Called by the inventory Create/Edit form before falling back to AI Lookup.
|
||||
/// </summary>
|
||||
[HttpGet]
|
||||
public async Task<IActionResult> CatalogLookup(string? q, string? vendor)
|
||||
public async Task<IActionResult> CatalogLookup(string? q, string? vendor, int? currentId = null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(q) || q.Length < 2)
|
||||
return Json(Array.Empty<object>());
|
||||
@@ -1042,18 +1043,47 @@ public class InventoryController : Controller
|
||||
var term = q.Trim().ToLower();
|
||||
var vendorTerm = vendor?.Trim().ToLower();
|
||||
|
||||
var matches = await _unitOfWork.PowderCatalog.FindAsync(p =>
|
||||
p.Sku.ToLower() == term ||
|
||||
p.ColorName.ToLower().Contains(term) ||
|
||||
p.Sku.ToLower().Contains(term));
|
||||
// Build a set of SKUs already in this company's inventory so we can exclude them.
|
||||
// When editing, the current item's own SKU is re-included so its catalog entry still appears.
|
||||
var existingItems = await _unitOfWork.InventoryItems.GetAllAsync();
|
||||
var existingSkus = existingItems
|
||||
.Where(i => !string.IsNullOrWhiteSpace(i.ManufacturerPartNumber) && i.Id != (currentId ?? 0))
|
||||
.Select(i => i.ManufacturerPartNumber!.Trim().ToLower())
|
||||
.ToHashSet();
|
||||
|
||||
// When a vendor is specified, search vendor-scoped first. Only widen to all vendors
|
||||
// if the scoped search returns nothing — prevents a cross-vendor color match from
|
||||
// being returned as the only result when the user clearly intended a specific manufacturer.
|
||||
IEnumerable<PowderCatalogItem> matches;
|
||||
if (!string.IsNullOrEmpty(vendorTerm))
|
||||
{
|
||||
matches = await _unitOfWork.PowderCatalog.FindAsync(p =>
|
||||
p.VendorName.ToLower().Contains(vendorTerm) && (
|
||||
p.Sku.ToLower() == term ||
|
||||
p.ColorName.ToLower().Contains(term) ||
|
||||
p.Sku.ToLower().Contains(term)));
|
||||
|
||||
// Fall back to all vendors only when the scoped search finds nothing
|
||||
if (!matches.Any())
|
||||
{
|
||||
matches = await _unitOfWork.PowderCatalog.FindAsync(p =>
|
||||
p.Sku.ToLower() == term ||
|
||||
p.ColorName.ToLower().Contains(term) ||
|
||||
p.Sku.ToLower().Contains(term));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
matches = await _unitOfWork.PowderCatalog.FindAsync(p =>
|
||||
p.Sku.ToLower() == term ||
|
||||
p.ColorName.ToLower().Contains(term) ||
|
||||
p.Sku.ToLower().Contains(term));
|
||||
}
|
||||
|
||||
// When a vendor hint is provided, prefer records where VendorName matches,
|
||||
// then fall back to all results so the user still sees cross-vendor options.
|
||||
var results = matches
|
||||
.Where(p => !existingSkus.Contains(p.Sku.ToLower()))
|
||||
.OrderBy(p => p.Sku.ToLower() == term ? 0 : 1)
|
||||
.ThenBy(p => !string.IsNullOrEmpty(vendorTerm) && p.VendorName.ToLower().Contains(vendorTerm) ? 0 : 1)
|
||||
.ThenBy(p => p.ColorName)
|
||||
.Take(10)
|
||||
.Select(p => new
|
||||
{
|
||||
id = p.Id,
|
||||
@@ -1075,7 +1105,8 @@ public class InventoryController : Controller
|
||||
requiresClearCoat = p.RequiresClearCoat,
|
||||
coverageSqFtPerLb = p.CoverageSqFtPerLb,
|
||||
transferEfficiency = p.TransferEfficiency
|
||||
});
|
||||
})
|
||||
.ToList();
|
||||
|
||||
return Json(results);
|
||||
}
|
||||
|
||||
@@ -3393,7 +3393,6 @@ public class QuotesController : Controller
|
||||
/// Returns a tempId that the JS wizard tracks and submits as AiPhotoTempIds[] on form submit.
|
||||
/// </summary>
|
||||
[HttpPost]
|
||||
[ValidateAntiForgeryToken]
|
||||
[EnableRateLimiting(AppConstants.RateLimitPolicies.Ai)]
|
||||
public async Task<IActionResult> UploadAiPhoto(IFormFile? file)
|
||||
{
|
||||
@@ -3431,7 +3430,6 @@ public class QuotesController : Controller
|
||||
/// so the model's estimates are calibrated to this company's pricing and material usage patterns.
|
||||
/// </summary>
|
||||
[HttpPost]
|
||||
[ValidateAntiForgeryToken]
|
||||
[EnableRateLimiting(AppConstants.RateLimitPolicies.Ai)]
|
||||
public async Task<IActionResult> AiAnalyzeItem([FromBody] AiAnalyzeItemRequest request)
|
||||
{
|
||||
@@ -3604,6 +3602,11 @@ public class QuotesController : Controller
|
||||
var complexityCharge = subtotal * complexityPct;
|
||||
subtotal += complexityCharge;
|
||||
|
||||
// Additional coat charge — each coat beyond the first adds AdditionalCoatLaborPercent%,
|
||||
// matching the formula in PricingCalculationService.CalculateQuoteItemPriceAsync.
|
||||
if (request.CoatCount > 1)
|
||||
subtotal += subtotal * (request.CoatCount - 1) * (costs.AdditionalCoatLaborPercent / 100m);
|
||||
|
||||
if (subtotal < costs.ShopMinimumCharge && costs.ShopMinimumCharge > 0)
|
||||
subtotal = costs.ShopMinimumCharge;
|
||||
|
||||
|
||||
@@ -445,6 +445,14 @@ public static class HelpKnowledgeBase
|
||||
- Every scan log is recorded as a JobUsage or Adjustment InventoryTransaction and immediately reduces QuantityOnHand; visible in Inventory Activity ledger
|
||||
- First-time scan on a new device requires login; browser caches the session after that
|
||||
|
||||
**Catalog Lookup & Label Scanner (when adding/editing inventory items):**
|
||||
- When creating or editing an inventory item, click the **Lookup** button next to the SKU/Part Number field to search a built-in platform catalog of thousands of Prismatic Powders and other manufacturer SKUs. Select a match to auto-fill name, manufacturer, color code, finish, coverage rate, SDS/TDS links, and cure specs.
|
||||
- The catalog only shows products not already in the company's inventory (prevents duplicates). When editing, the item's own catalog entry is always shown.
|
||||
- If no catalog match is found, the lookup falls back to **AI Lookup** — Claude searches the web for product specs and fills in what it finds.
|
||||
- If a vendor is selected first, the search is scoped to that vendor; if nothing matches it automatically widens to all vendors.
|
||||
- Click the **camera icon** (next to the Lookup button) to open the **Label Scanner** — point the phone camera at a QR code on a powder bag to identify the product automatically. It checks the platform catalog first; if not found, AI analyzes the label image.
|
||||
- If the scanned product **already exists** in the company's inventory, a modal appears offering to **Add Stock** to the existing item instead — enter quantity received and optional updated cost. No duplicate item is created.
|
||||
|
||||
**Powder Insights:** [/PowderInsights](/PowderInsights) — AI-powered analysis of powder usage trends, efficiency, and cost optimization. Requires at least 10 jobs with powder data; predictive features unlock at 150 jobs.
|
||||
|
||||
---
|
||||
@@ -1148,6 +1156,32 @@ public static class HelpKnowledgeBase
|
||||
|
||||
---
|
||||
|
||||
## MOBILE APP / ADD TO HOME SCREEN
|
||||
|
||||
Powder Coating Logix works in any phone browser but can also be installed as a **home screen app** (Progressive Web App / PWA) for a better experience. Installation is especially important for shop floor workers who use the label scanner or log usage from their phone.
|
||||
|
||||
**Why install it:**
|
||||
- Opens full-screen with no browser address bar — feels like a native app
|
||||
- Camera permission is granted once and remembered permanently (instead of being asked every browser session)
|
||||
- Faster to launch — one tap from the home screen
|
||||
|
||||
**iOS (iPhone / iPad) — must use Safari:**
|
||||
iOS only supports home screen installation from Safari. Installing from Chrome or Firefox on iOS creates a regular browser bookmark that does NOT give the standalone or camera-permission benefits.
|
||||
1. Open the app in **Safari** (not Chrome or Firefox)
|
||||
2. Tap the **Share button** (box with arrow pointing up) at the bottom of Safari
|
||||
3. Scroll down and tap **Add to Home Screen**
|
||||
4. Confirm the name and tap **Add**
|
||||
The app icon now appears on the home screen and launches in full-screen mode.
|
||||
|
||||
**Android — Chrome:**
|
||||
1. Open the app in Chrome
|
||||
2. Chrome may show an **Install App** banner automatically — tap it
|
||||
3. If no banner: tap the **menu (⋮)** in the top-right → **Add to Home Screen** or **Install App**
|
||||
|
||||
**Dashboard install banner:** When a mobile user who has not yet installed the app opens the Dashboard, a banner at the top of the page shows device-specific installation instructions. The banner disappears automatically once the app is running as a standalone home screen app, and can also be dismissed manually — it will not reappear after dismissal.
|
||||
|
||||
---
|
||||
|
||||
## BUG REPORTS
|
||||
|
||||
**Where:** [Bug Report](/BugReport)
|
||||
|
||||
@@ -59,6 +59,23 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@* PWA install banner — rendered by JS only on mobile, hidden once dismissed or already installed *@
|
||||
<div id="pwa-install-banner" class="row mb-4" style="display:none!important;">
|
||||
<div class="col-12">
|
||||
<div class="alert alert-permanent mb-0 d-flex align-items-start gap-3 py-3"
|
||||
style="background:var(--pcl-paper-2);border:1px solid var(--pcl-rule);border-left:4px solid var(--pcl-ember);">
|
||||
<div class="flex-shrink-0 mt-1">
|
||||
<img src="/images/pwa-icon-192.png" alt="" width="36" height="36" style="border-radius:8px;">
|
||||
</div>
|
||||
<div class="flex-grow-1">
|
||||
<div class="fw-semibold mb-1" id="pwa-banner-title">Add to Home Screen</div>
|
||||
<div class="text-muted small" id="pwa-banner-msg"></div>
|
||||
</div>
|
||||
<button type="button" class="btn-close flex-shrink-0" id="pwa-banner-dismiss" aria-label="Dismiss"></button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@if (guidedActivationBanner?.Show == true)
|
||||
{
|
||||
<div class="row mb-4">
|
||||
@@ -1253,6 +1270,61 @@
|
||||
</script>
|
||||
}
|
||||
|
||||
<script>
|
||||
(function () {
|
||||
var DISMISSED_KEY = 'pcl_pwa_banner_dismissed';
|
||||
|
||||
// Already installed as standalone — never show
|
||||
var isStandalone = window.navigator.standalone === true ||
|
||||
window.matchMedia('(display-mode: standalone)').matches;
|
||||
if (isStandalone) return;
|
||||
|
||||
// Already dismissed
|
||||
if (localStorage.getItem(DISMISSED_KEY)) return;
|
||||
|
||||
var ua = navigator.userAgent || '';
|
||||
var isIOS = /iphone|ipad|ipod/i.test(ua);
|
||||
var isAndroid = /android/i.test(ua);
|
||||
|
||||
// Only show on mobile
|
||||
if (!isIOS && !isAndroid) return;
|
||||
|
||||
var banner = document.getElementById('pwa-install-banner');
|
||||
var msgEl = document.getElementById('pwa-banner-msg');
|
||||
var titleEl = document.getElementById('pwa-banner-title');
|
||||
|
||||
if (isIOS) {
|
||||
// Detect Safari: has WebKit in UA but NOT Chrome/CriOS/FxiOS
|
||||
var isSafari = /webkit/i.test(ua) && !/crios|chrome|fxios|opios/i.test(ua);
|
||||
if (isSafari) {
|
||||
titleEl.textContent = 'Add to Home Screen';
|
||||
msgEl.innerHTML = 'For the best experience — and so the camera only asks once — open the ' +
|
||||
'<strong>Share menu</strong> <span style="font-size:1.1em">▲</span> at the bottom of Safari ' +
|
||||
'and tap <strong>Add to Home Screen</strong>.';
|
||||
} else {
|
||||
titleEl.textContent = 'Open in Safari to Install';
|
||||
msgEl.innerHTML = 'To add Powder Coating Logix to your home screen, <strong>open this page in Safari</strong> ' +
|
||||
'(not Chrome or another browser), then tap the <strong>Share menu</strong> ' +
|
||||
'<span style="font-size:1.1em">▲</span> and choose <strong>Add to Home Screen</strong>. ' +
|
||||
'This also means the camera only asks for permission once.';
|
||||
}
|
||||
} else {
|
||||
// Android
|
||||
titleEl.textContent = 'Add to Home Screen';
|
||||
msgEl.innerHTML = 'Tap the browser <strong>menu (⋮)</strong> and choose ' +
|
||||
'<strong>Add to Home Screen</strong> or <strong>Install App</strong> for the best experience ' +
|
||||
'and persistent camera access.';
|
||||
}
|
||||
|
||||
banner.style.removeProperty('display');
|
||||
|
||||
document.getElementById('pwa-banner-dismiss').addEventListener('click', function () {
|
||||
localStorage.setItem(DISMISSED_KEY, '1');
|
||||
banner.style.display = 'none';
|
||||
});
|
||||
}());
|
||||
</script>
|
||||
|
||||
@functions {
|
||||
IHtmlContent PriorityBadge(string priorityCode, string displayName, string colorClass)
|
||||
{
|
||||
|
||||
@@ -379,6 +379,56 @@
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section id="mobile-install" class="mb-5">
|
||||
<h2 class="h4 fw-bold border-bottom pb-2 mb-3">
|
||||
<i class="bi bi-phone text-primary me-2"></i>Using on Mobile — Add to Home Screen
|
||||
</h2>
|
||||
<p>
|
||||
Powder Coating Logix works in any phone browser, but installing it as a <strong>home screen app</strong>
|
||||
gives your shop floor workers a much better experience:
|
||||
</p>
|
||||
<ul class="mb-3">
|
||||
<li class="mb-1">Opens full-screen with no browser chrome — feels like a native app.</li>
|
||||
<li class="mb-1">The camera (used by the inventory label scanner) only asks for permission <strong>once</strong> after installation, instead of every browser session.</li>
|
||||
<li class="mb-1">Faster to launch — one tap from the home screen.</li>
|
||||
</ul>
|
||||
|
||||
<div class="alert alert-permanent alert-warning d-flex gap-2 mb-4" role="alert">
|
||||
<i class="bi bi-apple flex-shrink-0 mt-1"></i>
|
||||
<div>
|
||||
<strong>iPhone / iPad users must use Safari.</strong> Adding to the home screen from Chrome,
|
||||
Firefox, or other iOS browsers creates a regular bookmark that opens in that browser — not
|
||||
a standalone app. Only Safari on iOS supports the full home screen install.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<h3 class="h6 fw-semibold mt-3 mb-2"><i class="bi bi-apple me-1"></i>iOS (iPhone / iPad) — Safari only</h3>
|
||||
<ol class="mb-4">
|
||||
<li class="mb-1">Open the app in <strong>Safari</strong>.</li>
|
||||
<li class="mb-1">Tap the <strong>Share</strong> button <i class="bi bi-box-arrow-up"></i> at the bottom of the screen.</li>
|
||||
<li class="mb-1">Scroll down and tap <strong>Add to Home Screen</strong>.</li>
|
||||
<li class="mb-1">Confirm the name and tap <strong>Add</strong>.</li>
|
||||
<li class="mb-1">The app icon appears on your home screen. Tap it to open in full-screen mode.</li>
|
||||
</ol>
|
||||
|
||||
<h3 class="h6 fw-semibold mt-3 mb-2"><i class="bi bi-android2 me-1"></i>Android — Chrome</h3>
|
||||
<ol class="mb-4">
|
||||
<li class="mb-1">Open the app in <strong>Chrome</strong>.</li>
|
||||
<li class="mb-1">Chrome may show an <strong>Install App</strong> banner at the bottom automatically — tap it to install.</li>
|
||||
<li class="mb-1">If no banner appears, tap the <strong>menu (⋮)</strong> in the top-right corner and choose <strong>Add to Home Screen</strong> or <strong>Install App</strong>.</li>
|
||||
<li class="mb-1">Confirm and the icon is added to your home screen.</li>
|
||||
</ol>
|
||||
|
||||
<div class="alert alert-permanent alert-info d-flex gap-2 mb-0" role="alert">
|
||||
<i class="bi bi-lightbulb-fill flex-shrink-0 mt-1"></i>
|
||||
<div>
|
||||
When you open the Dashboard on a mobile browser (before installing), a banner at the top
|
||||
of the page guides you through the install steps for your specific device. Once the app is
|
||||
installed as a home screen app, the banner disappears automatically.
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="col-lg-3 d-none d-lg-block">
|
||||
@@ -393,6 +443,7 @@
|
||||
<a class="nav-link py-1 px-3 small text-body" href="#roles-and-permissions">Roles and Permissions</a>
|
||||
<a class="nav-link py-1 px-3 small text-body" href="#your-first-steps">Your First Steps</a>
|
||||
<a class="nav-link py-1 px-3 small text-body" href="#after-the-wizard">After the Wizard</a>
|
||||
<a class="nav-link py-1 px-3 small text-body" href="#mobile-install">Using on Mobile</a>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -73,6 +73,54 @@
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section id="catalog-lookup" class="mb-5">
|
||||
<h2 class="h4 fw-bold border-bottom pb-2 mb-3">
|
||||
<i class="bi bi-search text-primary me-2"></i>Catalog Lookup & Label Scanner
|
||||
</h2>
|
||||
<p>
|
||||
When adding or editing an inventory item, you don't have to type every field manually.
|
||||
Two shortcuts let you auto-fill product details in seconds:
|
||||
</p>
|
||||
|
||||
<h3 class="h6 fw-semibold mt-3 mb-2"><i class="bi bi-upc-scan me-1"></i>Smart Catalog Lookup</h3>
|
||||
<p>
|
||||
Click the <strong>Lookup</strong> button next to the SKU/Part Number field. Type a color name,
|
||||
SKU, or part number and the system searches a built-in catalog of thousands of Prismatic Powders
|
||||
and other manufacturer SKUs. Select a match and the form fills in automatically — item name,
|
||||
manufacturer, color code, finish, coverage rate, SDS/TDS links, and cure specifications.
|
||||
</p>
|
||||
<ul class="mb-3">
|
||||
<li class="mb-1">The catalog only shows products <strong>not already in your inventory</strong>, preventing duplicates. When editing an existing item, its own catalog entry is always shown.</li>
|
||||
<li class="mb-1">If no catalog match is found, the lookup falls back to <strong>AI Lookup</strong> — Claude searches the web for product specs and fills in whatever it can find.</li>
|
||||
<li class="mb-1">If a vendor name is selected in the Vendor field before searching, results are scoped to that vendor first, then broadened automatically if nothing matches.</li>
|
||||
</ul>
|
||||
|
||||
<h3 class="h6 fw-semibold mt-4 mb-2"><i class="bi bi-camera me-1"></i>Label Scanner (Camera)</h3>
|
||||
<p>
|
||||
Click the <strong>camera icon</strong> next to the Lookup button to open the label scanner.
|
||||
Point your phone or webcam at the QR code printed on a powder bag or manufacturer label.
|
||||
The scanner reads the code and attempts to identify the product:
|
||||
</p>
|
||||
<ol class="mb-3">
|
||||
<li class="mb-1">If the QR code matches a product in the platform catalog, the form fills in automatically — same as a manual catalog lookup.</li>
|
||||
<li class="mb-1">If no catalog match is found, the AI analyzes the label image and fills in whatever details it can extract (color name, SKU, manufacturer, finish).</li>
|
||||
<li class="mb-1">
|
||||
If the scanned product is <strong>already in your inventory</strong>, a prompt appears to
|
||||
<strong>Add Stock</strong> to the existing item instead — enter the quantity received and an
|
||||
optional updated unit cost, then save. No duplicate item is created.
|
||||
</li>
|
||||
</ol>
|
||||
<div class="alert alert-permanent alert-info d-flex gap-2 mb-0" role="alert">
|
||||
<i class="bi bi-phone me-1 flex-shrink-0 mt-1"></i>
|
||||
<div>
|
||||
The label scanner works best on a phone. If you're on iOS, open the page in
|
||||
<strong>Safari</strong> for reliable camera access. For persistent camera permission
|
||||
(no prompt each session), <a asp-controller="Help" asp-action="GettingStarted" class="alert-link"
|
||||
fragment="mobile-install">add the app to your home screen</a>.
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section id="stock-levels" class="mb-5">
|
||||
<h2 class="h4 fw-bold border-bottom pb-2 mb-3">
|
||||
<i class="bi bi-boxes text-primary me-2"></i>Stock Levels and Reorder Points
|
||||
@@ -485,6 +533,7 @@
|
||||
<nav class="nav flex-column">
|
||||
<a class="nav-link py-1 px-3 small text-body" href="#overview">Overview</a>
|
||||
<a class="nav-link py-1 px-3 small text-body" href="#adding-items">Adding Inventory Items</a>
|
||||
<a class="nav-link py-1 px-3 small text-body" href="#catalog-lookup">Catalog Lookup & Label Scanner</a>
|
||||
<a class="nav-link py-1 px-3 small text-body" href="#stock-levels">Stock Levels and Reorder Points</a>
|
||||
<a class="nav-link py-1 px-3 small text-body" href="#stock-adjustment">Stock Adjustment</a>
|
||||
<a class="nav-link py-1 px-3 small text-body" href="#transactions">Transaction Types</a>
|
||||
|
||||
@@ -77,7 +77,7 @@
|
||||
</button>
|
||||
@if ((bool)(ViewBag.AiInventoryAssistEnabled ?? false))
|
||||
{
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary ms-1" id="scan-label-btn" title="Scan a powder bag label with your camera">
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary ms-1 d-lg-none" id="scan-label-btn" title="Scan a powder bag label with your camera">
|
||||
<i class="bi bi-qr-code-scan me-1"></i>Scan Label
|
||||
</button>
|
||||
}
|
||||
|
||||
@@ -74,12 +74,12 @@
|
||||
<div class="d-flex align-items-center gap-2 border-bottom pb-2 mb-3">
|
||||
<h5 class="mb-0 d-flex align-items-center">
|
||||
<i class="bi bi-palette me-2 text-primary"></i>Product Details
|
||||
<button type="button" class="btn btn-sm btn-primary ms-2" id="smart-lookup-btn">
|
||||
<button type="button" class="btn btn-sm btn-primary ms-2" id="smart-lookup-btn" data-current-id="@Model.Id">
|
||||
<i class="bi bi-search me-1"></i>Lookup
|
||||
</button>
|
||||
@if ((bool)(ViewBag.AiInventoryAssistEnabled ?? false))
|
||||
{
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary ms-1" id="scan-label-btn" title="Scan a powder bag label with your camera">
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary ms-1 d-lg-none" id="scan-label-btn" title="Scan a powder bag label with your camera">
|
||||
<i class="bi bi-qr-code-scan me-1"></i>Scan Label
|
||||
</button>
|
||||
}
|
||||
|
||||
@@ -37,6 +37,7 @@
|
||||
const categorySelect = document.getElementById('field-category');
|
||||
const coatingSection = document.getElementById('coating-specs-section');
|
||||
const smartLookupBtn = document.getElementById('smart-lookup-btn');
|
||||
const scanLabelBtn = document.getElementById('scan-label-btn');
|
||||
|
||||
let coatingMap = {};
|
||||
if (categorySelect && categorySelect.dataset.coatingMap) {
|
||||
@@ -54,6 +55,7 @@
|
||||
const show = isCoatingCategory(catId);
|
||||
if (coatingSection) coatingSection.style.display = show ? '' : 'none';
|
||||
if (smartLookupBtn) smartLookupBtn.style.display = show ? '' : 'none';
|
||||
if (scanLabelBtn) scanLabelBtn.style.display = show ? '' : 'none';
|
||||
const samplePanelSection = document.getElementById('sample-panel-section');
|
||||
if (samplePanelSection) samplePanelSection.style.display = show ? '' : 'none';
|
||||
coatingOnlyFields.forEach(id => {
|
||||
|
||||
@@ -101,6 +101,13 @@
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>@ViewData["Title"] - Powder Coating Logix</title>
|
||||
<link rel="manifest" href="/manifest.json" />
|
||||
<meta name="theme-color" content="#1A1A1C" />
|
||||
<meta name="mobile-web-app-capable" content="yes" />
|
||||
<meta name="apple-mobile-web-app-capable" content="yes" />
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
|
||||
<meta name="apple-mobile-web-app-title" content="PCLogix" />
|
||||
<link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon.png" />
|
||||
|
||||
<!-- First-paint theme: server already stamped data-surface from cookie.
|
||||
This script only corrects first-visit (no cookie) using OS preference. -->
|
||||
@@ -2192,5 +2199,10 @@
|
||||
}, true);
|
||||
})();
|
||||
</script>
|
||||
<script>
|
||||
if ('serviceWorker' in navigator) {
|
||||
navigator.serviceWorker.register('/sw.js').catch(() => {});
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 23 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.5 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.5 MiB After Width: | Height: | Size: 192 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 34 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 37 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 192 KiB |
@@ -54,6 +54,8 @@
|
||||
const params = new URLSearchParams();
|
||||
if (searchTerm) params.set('q', searchTerm);
|
||||
if (manufacturer) params.set('vendor', manufacturer);
|
||||
const currentId = smartBtn.dataset.currentId;
|
||||
if (currentId) params.set('currentId', currentId);
|
||||
|
||||
const resp = await fetch(`${LOOKUP_URL}?${params}`);
|
||||
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
|
||||
@@ -358,7 +360,7 @@
|
||||
<div class="modal-dialog modal-dialog-scrollable">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header py-2">
|
||||
<h6 class="modal-title"><i class="bi bi-list-ul me-2"></i>Multiple matches — pick one</h6>
|
||||
<h6 class="modal-title"><i class="bi bi-list-ul me-2"></i>${items.length} match${items.length !== 1 ? 'es' : ''} — pick one <span class="text-muted fw-normal small">(already in inventory excluded)</span></h6>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal"></button>
|
||||
</div>
|
||||
<div class="modal-body p-0">
|
||||
|
||||
@@ -935,7 +935,9 @@ async function _aiRecalcPriceAsync() {
|
||||
const sqft = parseFloat(document.getElementById('ai_sqftOverride')?.value) || b.surfaceAreaSqFt;
|
||||
const minutes = parseInt(document.getElementById('ai_minutesOverride')?.value) || b.estimatedMinutes;
|
||||
const complexity = document.getElementById('ai_complexityOverride')?.value || b.complexity;
|
||||
const coatCount = b.coatCount || 1;
|
||||
// Use the actual step-3 coat count so the price preview reflects whatever the user
|
||||
// added in the coating layers screen, not the coat count fixed at AI analysis time.
|
||||
const coatCount = (wz.data.coats || []).length || b.coatCount || 1;
|
||||
|
||||
const recalcUrl = pageMeta.aiRecalcUrl || '/Quotes/AiRecalcPrice';
|
||||
const csrf = document.querySelector('input[name="__RequestVerificationToken"]')?.value || '';
|
||||
@@ -1280,14 +1282,22 @@ async function aiAnalyze() {
|
||||
body: JSON.stringify(payload)
|
||||
});
|
||||
if (!resp.ok) {
|
||||
throw new Error(`Server returned ${resp.status} ${resp.statusText}`);
|
||||
if (resp.status === 401 || resp.status === 302 || resp.redirected) {
|
||||
throw new Error('Your session has expired. Please refresh the page and sign in again.');
|
||||
}
|
||||
throw new Error(`Server error (${resp.status}). Please try again.`);
|
||||
}
|
||||
const contentType = resp.headers.get('Content-Type') || '';
|
||||
if (!contentType.includes('application/json')) {
|
||||
// Server returned HTML (e.g. login redirect) instead of JSON
|
||||
throw new Error('Your session may have expired. Please refresh the page and sign in again.');
|
||||
}
|
||||
const result = await resp.json();
|
||||
aiHandleResult(result);
|
||||
} catch (err) {
|
||||
console.error('AI analyze error:', err);
|
||||
aiSetLoading(false);
|
||||
aiShowError('Error: ' + err.message);
|
||||
aiShowError(err.message);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1334,14 +1344,21 @@ async function aiSendFollowup() {
|
||||
body: JSON.stringify(payload)
|
||||
});
|
||||
if (!resp.ok) {
|
||||
throw new Error(`Server returned ${resp.status} ${resp.statusText}`);
|
||||
if (resp.status === 401 || resp.status === 302 || resp.redirected) {
|
||||
throw new Error('Your session has expired. Please refresh the page and sign in again.');
|
||||
}
|
||||
throw new Error(`Server error (${resp.status}). Please try again.`);
|
||||
}
|
||||
const contentType = resp.headers.get('Content-Type') || '';
|
||||
if (!contentType.includes('application/json')) {
|
||||
throw new Error('Your session may have expired. Please refresh the page and sign in again.');
|
||||
}
|
||||
const result = await resp.json();
|
||||
aiHandleResult(result);
|
||||
} catch (err) {
|
||||
console.error('AI follow-up error:', err);
|
||||
aiSetLoading(false);
|
||||
aiShowError('Error: ' + err.message);
|
||||
aiShowError(err.message);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1363,6 +1380,9 @@ function aiHandleResult(result) {
|
||||
wz.ai.followUpQuestion = result.followUpQuestion || 'Can you provide more details?';
|
||||
renderStep(wz.step);
|
||||
document.getElementById('ai_followupAnswer')?.focus();
|
||||
requestAnimationFrame(() => {
|
||||
document.getElementById('ai_followupSection')?.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
|
||||
});
|
||||
} else {
|
||||
// Store result state and re-render step so elements are guaranteed fresh
|
||||
wz.ai.result = result;
|
||||
@@ -1371,6 +1391,9 @@ function aiHandleResult(result) {
|
||||
wz.ai.phase = 'result';
|
||||
renderStep(wz.step);
|
||||
document.getElementById('ai_acceptError')?.classList.add('d-none');
|
||||
requestAnimationFrame(() => {
|
||||
document.getElementById('ai_resultsSection')?.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1488,6 +1511,8 @@ function addCoatRow() {
|
||||
if (!container) return;
|
||||
container.insertAdjacentHTML('beforeend', buildCoatRowHtml(i, coat));
|
||||
updatePowderNeeded(i);
|
||||
// AI items: recalculate price preview so multi-coat surcharge is visible before saving
|
||||
if (wz.itemType === 'ai') aiRecalcPrice();
|
||||
}
|
||||
|
||||
const COAT_NAME_PRESETS = ['Primer', 'Base Coat', 'Mid Coat', 'Top Coat', 'Clear Coat'];
|
||||
@@ -1699,6 +1724,8 @@ function removeCoatRow(i) {
|
||||
container.insertAdjacentHTML('beforeend', buildCoatRowHtml(idx, coat));
|
||||
restoreCoatRow(idx, coat);
|
||||
});
|
||||
updateAllPowderNeeded();
|
||||
if (wz.itemType === 'ai') aiRecalcPrice();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "Powder Coating Logix",
|
||||
"short_name": "PCLogix",
|
||||
"description": "Powder coating shop management — jobs, quotes, inventory, and scheduling.",
|
||||
"start_url": "/",
|
||||
"display": "standalone",
|
||||
"background_color": "#ffffff",
|
||||
"theme_color": "#1A1A1C",
|
||||
"orientation": "any",
|
||||
"icons": [
|
||||
{
|
||||
"src": "/images/pwa-icon-192.png",
|
||||
"sizes": "192x192",
|
||||
"type": "image/png",
|
||||
"purpose": "any maskable"
|
||||
},
|
||||
{
|
||||
"src": "/images/pwa-icon-512.png",
|
||||
"sizes": "512x512",
|
||||
"type": "image/png",
|
||||
"purpose": "any maskable"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
// Minimal service worker — required for PWA installability.
|
||||
// No caching: all requests pass through to the network normally.
|
||||
// This exists solely so browsers recognize the site as installable,
|
||||
// which causes iOS/Android to persist camera permissions after "Add to Home Screen."
|
||||
|
||||
self.addEventListener('install', () => self.skipWaiting());
|
||||
self.addEventListener('activate', e => e.waitUntil(self.clients.claim()));
|
||||
self.addEventListener('fetch', e => e.respondWith(fetch(e.request)));
|
||||
Reference in New Issue
Block a user