// Shared live crawl progress display (AutoSearch + Admin Global Crawler). const { useMemo } = React; window.crawlProgressStatusLabel = function crawlProgressStatusLabel(entry) { const raw = entry?.message || entry?.stop_reason || entry?.status || ''; const normalized = String(raw).replace(/_/g, ' ').trim(); if (!normalized || normalized === 'running') return ''; if (normalized === 'budget reached') return 'budget reached'; if (normalized === 'depth reached') return 'depth reached'; if (normalized === 'target met') return 'run target reached'; if (normalized === 'source exhausted') return 'source exhausted'; if (normalized === 'stopped by user') return 'stopped by user'; return normalized; }; window.crawlProgressMaxCount = function crawlProgressMaxCount(...values) { return Math.max( 0, ...values.map(value => Number(value || 0)).filter(value => Number.isFinite(value)), ); }; window.crawlProgressFirstCount = function crawlProgressFirstCount(...values) { for (const value of values) { const count = Number(value); if (Number.isFinite(count)) return Math.max(0, count); } return 0; }; const _CRAWL_TERMINAL_DIRECTION_STATUSES = new Set([ 'target_met', 'exhausted', 'budget_reached', 'stopped', 'failed', 'completed', ]); /** Share of per-direction fetch caps consumed (moves while ads are still scoring). */ window.crawlDirectionFetchPercent = function crawlDirectionFetchPercent(progress) { const rows = Object.values(progress?.directionProgress || {}).filter(row => row && typeof row === 'object'); const totalSlots = Number(progress?.directionsTotal || 0) || rows.length; if (!totalSlots) return 0; let checked = 0; let cap = 0; rows.forEach((row) => { const maxAds = Number(row.max_ads || 0); const adsChecked = Number(row.ads_checked || 0); if (maxAds > 0) { cap += maxAds; checked += Math.min(maxAds, adsChecked); return; } if (adsChecked > 0 || row.touched) { cap += 1; checked += 1; } }); if (!cap) return 0; return Math.min(100, Math.round((checked * 100) / cap)); }; /** Overall crawl bar: completed checks, touched searches, and in-flight fetch work. */ window.crawlProgressBarPercent = function crawlProgressBarPercent(progress) { if (!progress || typeof progress !== 'object') return 0; if (progress.crawlState === 'done') return 100; const directionsTotal = Number(progress.directionsTotal || 0); const directionsSatisfied = Number(progress.directionsSatisfied || 0); const coverageTotal = Number(progress.coverageTotal || 0) || Object.keys(progress.directionProgress || {}).length; const coverageTouched = Number(progress.coverageTouched || 0); const donePct = directionsTotal > 0 ? (directionsSatisfied * 100) / directionsTotal : 0; const coveragePct = coverageTotal > 0 ? (coverageTouched * 100) / coverageTotal : 0; const fetchPct = window.crawlDirectionFetchPercent(progress); // In-flight directions: partial credit from ads_checked vs per-direction cap. const rows = Object.values(progress.directionProgress || {}).filter(row => row && typeof row === 'object'); const slotTotal = directionsTotal || rows.length; let partialUnits = 0; if (slotTotal > 0) { rows.forEach((row) => { const status = String(row.status || ''); if (_CRAWL_TERMINAL_DIRECTION_STATUSES.has(status)) { partialUnits += 1; return; } const maxAds = Number(row.max_ads || 0); const adsChecked = Number(row.ads_checked || 0); if (maxAds > 0) { partialUnits += Math.min(1, adsChecked / maxAds); } else if (adsChecked > 0 || row.touched) { partialUnits += 0.15; } }); } const partialPct = slotTotal > 0 ? (partialUnits * 100) / slotTotal : 0; return Math.min(100, Math.round(Math.max(donePct, coveragePct, fetchPct, partialPct))); }; window.formatCrawlPlatformProgress = function formatCrawlPlatformProgress(source, progress) { const entry = progress?.platformProgress?.[source] || {}; const platform = progress?.platformCounts?.[source] || {}; const label = (PLATFORMS[source]?.label || source).toUpperCase(); const maxCount = window.crawlProgressMaxCount; const firstCount = window.crawlProgressFirstCount; const statusLabel = window.crawlProgressStatusLabel; const winners = firstCount( entry.visible_winners, platform.visible_winners, entry.winners, platform.winners, ); const scored = maxCount(entry.scored, platform.scored); const adsChecked = maxCount(entry.ads_checked, progress?.platformAdsFetched?.[source]); const maxAds = Number(entry.max_ads || 0); const stageParts = [ ['queued', Number(entry.queued || 0)], ['dedupe skipped', Number(entry.dedupe_skipped || 0)], ['transcribing', Number(entry.transcribing || 0)], ['scoring', Number(entry.scoring || 0)], ['rate limited', Number(entry.rate_limited || 0)], ['persisted', Number(entry.persisted || 0)], ] .filter(([, count]) => count > 0) .map(([stageLabel, count]) => `${count} ${stageLabel}`); const stageText = stageParts.length ? ` · ${stageParts.join(' · ')}` : ''; const adsText = `${adsChecked}${maxAds ? `/${maxAds}` : ''} ads checked`; const winnerText = `${winners} winners`; const suffix = statusLabel(entry); if (source === 'brandsearch') { const pages = Number(entry.internal_pages_fetched || 0); return `${label}: ${adsText} · ${pages} internal page${pages === 1 ? '' : 's'} · ${scored} scored · ${winnerText}${stageText}${suffix ? ` · ${suffix}` : ''}`; } const pages = Number(entry.pages_fetched || 0); return `${label}: ${pages} page${pages === 1 ? '' : 's'} · ${adsText} · ${scored} scored · ${winnerText}${stageText}${suffix ? ` · ${suffix}` : ''}`; }; window.formatCrawlDirectionProgress = function formatCrawlDirectionProgress(row) { const statusLabel = window.crawlProgressStatusLabel; const label = (PLATFORMS[row.source]?.label || row.source || 'source').toUpperCase(); const ads = Number(row.ads_checked || 0); const pages = Number(row.pages_fetched || row.internal_pages_fetched || 0); const scored = Number(row.scored || 0); const winners = Number(row.visible_winners ?? row.winners ?? 0); const picked = Number(row.picked || 0); const status = statusLabel(row) || (row.touched ? 'touched' : (row.status || 'pending')); return `${label}: ${ads} ads fetched · ${pages} page${pages === 1 ? '' : 's'} · ${scored} scored · ${winners} winners${picked ? ` · ${picked} picked` : ''} · ${status}`; }; window.mapCrawlerStateToProgress = function mapCrawlerStateToProgress(crawlerOrLive) { if (!crawlerOrLive || typeof crawlerOrLive !== 'object') return null; const c = crawlerOrLive; const platformProgress = c.platformProgress || c.platform_progress || {}; const directionProgress = c.directionProgress || c.direction_progress || {}; const platformAdsFetched = c.platformAdsFetched || c.platform_ads_fetched || {}; let totalScored = Number(c.totalScored ?? c.total_scored ?? 0); let totalWinners = Number(c.totalWinners ?? c.total_winners ?? c.winners ?? 0); const globalScored = Number(c.globalScored ?? c.global_scored ?? 0); const globalWinners = Number(c.globalWinners ?? c.global_winners ?? 0); const loadedWinners = Number(c.loadedWinners ?? c.loaded_winners ?? 0); if (loadedWinners > 0) { totalWinners = loadedWinners; } else if (!totalWinners) { Object.values(platformProgress).forEach((entry) => { if (!entry || typeof entry !== 'object') return; totalWinners += Number(entry.visible_winners ?? entry.winners ?? 0); }); } if (!totalScored) { Object.values(platformProgress).forEach((entry) => { if (!entry || typeof entry !== 'object') return; totalScored += Number(entry.scored || 0); }); } return { crawlState: c.crawlState || c.crawl_state || c.crawl_status || c.status || 'idle', coverageTouched: Number(c.coverageTouched ?? c.coverage_touched ?? 0), coverageTotal: Number(c.coverageTotal ?? c.coverage_total ?? 0), directionsSatisfied: Number( c.directionsSatisfied ?? c.directions_satisfied ?? c.run_directions_done ?? 0, ), directionsTotal: Number( c.directionsTotal ?? c.directions_total ?? c.run_directions_total ?? 0, ), platformAdsFetched, platformProgress, directionProgress, platformCounts: c.platformCounts || c.platform_counts || {}, totalScored, totalWinners, winners: Number(c.winners ?? totalWinners), globalScored, globalWinners, creditsRemaining: typeof c.creditsRemaining === 'number' ? c.creditsRemaining : (typeof c.credits_remaining === 'number' ? c.credits_remaining : undefined), }; }; window.buildCrawlPlatformProgressRows = function buildCrawlPlatformProgressRows(progress, platformKeys) { if (!progress) return []; const liveKeys = Array.isArray(platformKeys) && platformKeys.length ? platformKeys : Object.keys(PLATFORMS).filter(k => k === 'foreplay' || k === 'adspy' || k === 'brandsearch'); const keys = [ ...new Set([ ...liveKeys, ...Object.keys(progress.platformProgress || {}), ...Object.keys(progress.platformAdsFetched || {}), ...Object.keys(progress.platformCounts || {}), ]), ]; return keys.sort().map(source => ({ source, text: window.formatCrawlPlatformProgress(source, progress), title: window.crawlProgressStatusLabel(progress.platformProgress?.[source] || {}), })); }; window.CrawlProgressView = function CrawlProgressView({ progress, running = false, statusText = '', winnersText = '', showBar = true, platformKeys = null, runKind = 'manual', className = '', }) { const view = useMemo(() => { if (!progress) return null; const crawlState = progress.crawlState; const isComplete = crawlState === 'done'; const directionsTotal = Number(progress.directionsTotal || 0); const directionsSatisfied = Number(progress.directionsSatisfied || 0); const dirPct = window.crawlProgressBarPercent(progress); const directionRows = Object.values(progress.directionProgress || {}) .filter(Boolean) .sort((a, b) => { const left = `${a.direction_name || a.direction_id || ''}:${a.source || ''}`; const right = `${b.direction_name || b.direction_id || ''}:${b.source || ''}`; return left.localeCompare(right); }); const coverageTouched = Number(progress.coverageTouched || 0) || directionRows.filter(row => row.touched).length; const coverageTotal = Number(progress.coverageTotal || 0) || directionRows.length; const directionGroups = directionRows.reduce((groups, row) => { const key = row.direction_id || row.direction_name || 'unknown'; if (!groups[key]) { groups[key] = { key, name: row.direction_name || row.direction_id || 'Unknown direction', rows: [], }; } groups[key].rows.push(row); return groups; }, {}); const totalFetched = Object.values(progress.platformAdsFetched || {}) .reduce((sum, count) => sum + (Number(count) || 0), 0); const currentRunWinners = Math.max( Number(progress.currentRunWinners || 0), Number(progress.totalWinners || 0), Number(progress.winners || 0), ); const globalWinners = Number(progress.globalWinners || 0); const globalScored = Number(progress.globalScored || 0); const globalParts = []; if (globalWinners > currentRunWinners) { globalParts.push(`${globalWinners} global winners`); } if (globalScored > Number(progress.totalScored || 0)) { globalParts.push(`${globalScored} global evaluated`); } const defaultWinnersText = [ `${currentRunWinners} current-run winners`, ...globalParts, ].join(' · '); const platformProgressRows = window.buildCrawlPlatformProgressRows(progress, platformKeys); return { crawlState, isComplete, dirPct, directionRows, coverageTouched, coverageTotal, directionGroups, totalFetched, winnersLine: winnersText || defaultWinnersText, platformProgressRows, }; }, [progress, platformKeys, winnersText]); if (!view || !progress) return null; return (
{runKind === 'scheduler' ? ( Scheduler ) : null} {view.isComplete ? ( ) : ( )} {statusText ? {statusText} : null} {runKind === 'autoresearch' && view.isComplete ? ( <> {view.winnersLine} {view.coverageTotal ? ` · ${view.coverageTouched}/${view.coverageTotal} searches checked before target was met` : ''} {view.totalFetched ? ` · ${view.totalFetched} fetched` : ''} {progress.totalScored ? ` · ${progress.totalScored} current-run scored` : ''} ) : ( <> {view.coverageTotal ? `${view.coverageTouched}/${view.coverageTotal} searches touched · ` : ''} {view.totalFetched} fetched · {(progress.totalScored || 0)} current-run scored · {view.winnersLine} {' · '}{(progress.directionsSatisfied || 0)}/{(progress.directionsTotal || 0)} direction checks done )} {typeof progress.creditsRemaining === 'number' ? ` · ${progress.creditsRemaining} credits left` : ''}
{showBar ? (
) : null} {view.platformProgressRows.length > 0 && (
{view.platformProgressRows.map(row => ( {row.text} ))}
)} {view.directionRows.length > 0 && (
Direction details ({view.coverageTouched}/{view.coverageTotal || view.directionRows.length} searches touched)
{Object.values(view.directionGroups).map(group => (
{group.name}
{group.rows.map(row => ( {window.formatCrawlDirectionProgress(row)} ))}
))}
)}
); };