1697 lines
62 KiB
TypeScript
1697 lines
62 KiB
TypeScript
/**
|
||
* Pipeline admin dashboard — video list with status, retrigger/revoke,
|
||
* expandable event log with token usage and collapsible JSON viewer.
|
||
*/
|
||
|
||
import React, { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||
import { useSearchParams } from "react-router-dom";
|
||
import { useDocumentTitle } from "../hooks/useDocumentTitle";
|
||
import {
|
||
fetchPipelineVideos,
|
||
fetchPipelineEvents,
|
||
fetchPipelineRuns,
|
||
fetchWorkerStatus,
|
||
fetchDebugMode,
|
||
setDebugMode,
|
||
triggerPipeline,
|
||
revokePipeline,
|
||
cleanRetriggerPipeline,
|
||
rerunStage,
|
||
fetchChunkingData,
|
||
fetchStalePages,
|
||
bulkResynthesize,
|
||
wipeAllOutput,
|
||
fetchCreators,
|
||
fetchRecentActivity,
|
||
type PipelineVideoItem,
|
||
type PipelineEvent,
|
||
type PipelineRunItem,
|
||
type WorkerStatusResponse,
|
||
type RecentActivityItem,
|
||
} from "../api/public-client";
|
||
|
||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||
|
||
function formatDate(iso: string | null): string {
|
||
if (!iso) return "—";
|
||
return new Date(iso).toLocaleString(undefined, {
|
||
month: "short",
|
||
day: "numeric",
|
||
hour: "2-digit",
|
||
minute: "2-digit",
|
||
second: "2-digit",
|
||
});
|
||
}
|
||
|
||
function formatTokens(n: number): string {
|
||
if (n === 0) return "0";
|
||
if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
|
||
if (n >= 1_000) return `${(n / 1_000).toFixed(1)}k`;
|
||
return String(n);
|
||
}
|
||
|
||
|
||
function formatElapsed(isoStart: string | null): string {
|
||
if (!isoStart) return "";
|
||
const ms = Date.now() - new Date(isoStart).getTime();
|
||
if (ms < 0) return "";
|
||
const secs = Math.floor(ms / 1000);
|
||
if (secs < 60) return `${secs}s`;
|
||
const mins = Math.floor(secs / 60);
|
||
const remainSecs = secs % 60;
|
||
if (mins < 60) return `${mins}m ${remainSecs}s`;
|
||
const hrs = Math.floor(mins / 60);
|
||
return `${hrs}h ${mins % 60}m`;
|
||
}
|
||
const STATUS_LABELS: Record<string, string> = {
|
||
not_started: "Not Started",
|
||
queued: "Queued",
|
||
processing: "In Progress",
|
||
error: "Errored",
|
||
complete: "Complete",
|
||
};
|
||
|
||
function statusBadgeClass(status: string): string {
|
||
switch (status) {
|
||
case "complete":
|
||
return "pipeline-badge--success";
|
||
case "processing":
|
||
return "pipeline-badge--active";
|
||
case "error":
|
||
return "pipeline-badge--error";
|
||
case "not_started":
|
||
case "queued":
|
||
return "pipeline-badge--pending";
|
||
default:
|
||
return "";
|
||
}
|
||
}
|
||
|
||
function eventTypeIcon(eventType: string): string {
|
||
switch (eventType) {
|
||
case "start":
|
||
return "▶";
|
||
case "complete":
|
||
return "✓";
|
||
case "error":
|
||
return "✗";
|
||
case "llm_call":
|
||
return "⚙";
|
||
default:
|
||
return "·";
|
||
}
|
||
}
|
||
|
||
// ── Collapsible JSON ─────────────────────────────────────────────────────────
|
||
|
||
function JsonViewer({ data }: { data: Record<string, unknown> | null }) {
|
||
const [open, setOpen] = useState(false);
|
||
if (!data || Object.keys(data).length === 0) return null;
|
||
|
||
return (
|
||
<div className="json-viewer">
|
||
<button
|
||
className="json-viewer__toggle"
|
||
onClick={() => setOpen((v) => !v)}
|
||
aria-expanded={open}
|
||
title="Raw JSON event data logged by the pipeline stage"
|
||
>
|
||
{open ? "▾ Hide payload" : "▸ Show payload"}
|
||
</button>
|
||
{open && (
|
||
<pre className="json-viewer__content">
|
||
{JSON.stringify(data, null, 2)}
|
||
</pre>
|
||
)}
|
||
</div>
|
||
);
|
||
}
|
||
|
||
// ── Debug Payload Viewer ─────────────────────────────────────────────────────
|
||
|
||
interface DebugSection {
|
||
label: string;
|
||
content: string;
|
||
}
|
||
|
||
function DebugPayloadViewer({ event }: { event: PipelineEvent }) {
|
||
const sections: DebugSection[] = [];
|
||
if (event.payload?.request_params) sections.push({ label: "Request Params", content: JSON.stringify(event.payload.request_params, null, 2) });
|
||
if (event.system_prompt_text) sections.push({ label: "System Prompt", content: event.system_prompt_text });
|
||
if (event.user_prompt_text) sections.push({ label: "User Prompt", content: event.user_prompt_text });
|
||
if (event.response_text) sections.push({ label: "Response", content: event.response_text });
|
||
|
||
const [openSections, setOpenSections] = useState<Record<string, boolean>>({});
|
||
const [copiedKey, setCopiedKey] = useState<string | null>(null);
|
||
|
||
const SECTION_TOOLTIPS: Record<string, string> = {
|
||
"System Prompt": "Instructions sent to the LLM defining its role and output format for this pipeline stage",
|
||
"User Prompt": "The actual transcript content and extraction request sent to the LLM",
|
||
"Response": "Raw LLM output before parsing — the extracted data that feeds the next stage",
|
||
};
|
||
|
||
if (sections.length === 0) return null;
|
||
|
||
const toggleSection = (label: string) => {
|
||
setOpenSections((prev) => ({ ...prev, [label]: !prev[label] }));
|
||
};
|
||
|
||
const copyToClipboard = async (label: string, text: string) => {
|
||
try {
|
||
await navigator.clipboard.writeText(text);
|
||
setCopiedKey(label);
|
||
setTimeout(() => setCopiedKey(null), 1500);
|
||
} catch {
|
||
// Fallback for non-HTTPS contexts
|
||
const ta = document.createElement("textarea");
|
||
ta.value = text;
|
||
ta.style.position = "fixed";
|
||
ta.style.opacity = "0";
|
||
document.body.appendChild(ta);
|
||
ta.select();
|
||
document.execCommand("copy");
|
||
document.body.removeChild(ta);
|
||
setCopiedKey(label);
|
||
setTimeout(() => setCopiedKey(null), 1500);
|
||
}
|
||
};
|
||
|
||
const exportAsJson = () => {
|
||
// Dump everything — full rawdog debug payload with request params
|
||
const data: Record<string, unknown> = {
|
||
event_id: event.id,
|
||
stage: event.stage,
|
||
event_type: event.event_type,
|
||
model: event.model,
|
||
prompt_tokens: event.prompt_tokens,
|
||
completion_tokens: event.completion_tokens,
|
||
total_tokens: event.total_tokens,
|
||
duration_ms: event.duration_ms,
|
||
created_at: event.created_at,
|
||
payload: event.payload,
|
||
request_params: event.payload?.request_params ?? null,
|
||
system_prompt_text: event.system_prompt_text,
|
||
user_prompt_text: event.user_prompt_text,
|
||
response_text: event.response_text,
|
||
};
|
||
const blob = new Blob([JSON.stringify(data, null, 2)], { type: "application/json" });
|
||
const url = URL.createObjectURL(blob);
|
||
const a = document.createElement("a");
|
||
a.href = url;
|
||
a.download = `debug-${event.stage}-${event.id.slice(0, 8)}.json`;
|
||
a.click();
|
||
URL.revokeObjectURL(url);
|
||
};
|
||
|
||
return (
|
||
<div className="debug-viewer">
|
||
<div className="debug-viewer__header">
|
||
<span className="debug-viewer__label">LLM Debug</span>
|
||
<button className="debug-viewer__export" onClick={exportAsJson} title="Export debug payload as JSON">
|
||
↓ JSON
|
||
</button>
|
||
</div>
|
||
{sections.map((sec) => {
|
||
const isOpen = !!openSections[sec.label];
|
||
return (
|
||
<div key={sec.label} className="debug-viewer__section">
|
||
<div className="debug-viewer__section-header">
|
||
<button
|
||
className="debug-viewer__section-toggle"
|
||
onClick={() => toggleSection(sec.label)}
|
||
aria-expanded={isOpen}
|
||
title={SECTION_TOOLTIPS[sec.label] ?? ""}
|
||
>
|
||
{isOpen ? "▾" : "▸"} {sec.label}
|
||
</button>
|
||
<button
|
||
className="debug-viewer__copy"
|
||
onClick={() => void copyToClipboard(sec.label, sec.content)}
|
||
title={`Copy ${sec.label}`}
|
||
>
|
||
{copiedKey === sec.label ? "✓ Copied" : "Copy"}
|
||
</button>
|
||
</div>
|
||
{isOpen && (
|
||
<pre className="debug-viewer__content">{sec.content}</pre>
|
||
)}
|
||
</div>
|
||
);
|
||
})}
|
||
</div>
|
||
);
|
||
}
|
||
|
||
// ── Event Log ────────────────────────────────────────────────────────────────
|
||
|
||
function EventLog({ videoId, status, runId }: { videoId: string; status: string; runId?: string }) {
|
||
const [events, setEvents] = useState<PipelineEvent[]>([]);
|
||
const [total, setTotal] = useState(0);
|
||
const [loading, setLoading] = useState(true);
|
||
const [error, setError] = useState<string | null>(null);
|
||
const [offset, setOffset] = useState(0);
|
||
const [viewMode, setViewMode] = useState<"head" | "tail">("tail");
|
||
const limit = 50;
|
||
const initialLoaded = useRef(false);
|
||
|
||
const load = useCallback(async (silent = false) => {
|
||
if (!silent) setLoading(true);
|
||
setError(null);
|
||
try {
|
||
const res = await fetchPipelineEvents(videoId, {
|
||
offset,
|
||
limit,
|
||
order: viewMode === "head" ? "asc" : "desc",
|
||
run_id: runId,
|
||
});
|
||
setEvents(res.items);
|
||
setTotal(res.total);
|
||
initialLoaded.current = true;
|
||
} catch (err) {
|
||
if (!silent) setError(err instanceof Error ? err.message : "Failed to load events");
|
||
} finally {
|
||
if (!silent) setLoading(false);
|
||
}
|
||
}, [videoId, offset, viewMode]);
|
||
|
||
useEffect(() => {
|
||
void load();
|
||
}, [load]);
|
||
|
||
// Auto-refresh when video is actively processing
|
||
useEffect(() => {
|
||
if (status !== "processing" && status !== "queued") return;
|
||
const id = setInterval(() => void load(true), 10_000);
|
||
return () => clearInterval(id);
|
||
}, [status, load]);
|
||
|
||
if (loading) return <div className="loading">Loading events…</div>;
|
||
if (error) return <div className="loading error-text">Error: {error}</div>;
|
||
if (events.length === 0) return <div className="pipeline-events__empty">No events recorded.</div>;
|
||
|
||
const hasNext = offset + limit < total;
|
||
const hasPrev = offset > 0;
|
||
|
||
return (
|
||
<div className="pipeline-events">
|
||
<div className="pipeline-events__header">
|
||
<span className="pipeline-events__count">{total} event{total !== 1 ? "s" : ""}</span>
|
||
<div className="pipeline-events__view-toggle">
|
||
<button
|
||
className={`pipeline-events__view-btn${viewMode === "head" ? " pipeline-events__view-btn--active" : ""}`}
|
||
onClick={() => { setViewMode("head"); setOffset(0); }}
|
||
>
|
||
Oldest first
|
||
</button>
|
||
<button
|
||
className={`pipeline-events__view-btn${viewMode === "tail" ? " pipeline-events__view-btn--active" : ""}`}
|
||
onClick={() => { setViewMode("tail"); setOffset(0); }}
|
||
>
|
||
Newest first
|
||
</button>
|
||
</div>
|
||
</div>
|
||
|
||
<div className="pipeline-events__list">
|
||
{events.map((evt) => (
|
||
<div key={evt.id} className={`pipeline-event pipeline-event--${evt.event_type}`}>
|
||
<div className="pipeline-event__row">
|
||
<span className="pipeline-event__icon">{eventTypeIcon(evt.event_type)}</span>
|
||
<span className="pipeline-event__stage">{evt.stage}</span>
|
||
<span className={`pipeline-badge pipeline-badge--event-${evt.event_type}`}>
|
||
{evt.event_type}
|
||
</span>
|
||
{typeof evt.payload?.context === "string" && (
|
||
<span className="pipeline-event__context" title="Processing context">
|
||
{evt.payload.context}
|
||
</span>
|
||
)}
|
||
{evt.model && <span className="pipeline-event__model">{evt.model}</span>}
|
||
{evt.total_tokens != null && evt.total_tokens > 0 && (
|
||
<span className="pipeline-event__tokens" title={`prompt: ${evt.prompt_tokens ?? 0} / completion: ${evt.completion_tokens ?? 0}`}>
|
||
{formatTokens(evt.total_tokens)} tok
|
||
</span>
|
||
)}
|
||
{evt.duration_ms != null && (
|
||
<span className="pipeline-event__duration">{evt.duration_ms}ms</span>
|
||
)}
|
||
<span className="pipeline-event__time">{formatDate(evt.created_at)}</span>
|
||
</div>
|
||
<JsonViewer data={evt.payload} />
|
||
<DebugPayloadViewer event={evt} />
|
||
</div>
|
||
))}
|
||
</div>
|
||
|
||
{(hasPrev || hasNext) && (
|
||
<div className="pipeline-events__pager">
|
||
<button
|
||
className="btn btn--small btn--secondary"
|
||
disabled={!hasPrev}
|
||
onClick={() => setOffset((o) => Math.max(0, o - limit))}
|
||
>
|
||
← Prev
|
||
</button>
|
||
<span className="pipeline-events__pager-info">
|
||
{offset + 1}–{Math.min(offset + limit, total)} of {total}
|
||
</span>
|
||
<button
|
||
className="btn btn--small btn--secondary"
|
||
disabled={!hasNext}
|
||
onClick={() => setOffset((o) => o + limit)}
|
||
>
|
||
Next →
|
||
</button>
|
||
</div>
|
||
)}
|
||
</div>
|
||
);
|
||
}
|
||
|
||
// ── Worker Status ────────────────────────────────────────────────────────────
|
||
|
||
const WorkerStatus = React.memo(function WorkerStatus() {
|
||
const [status, setStatus] = useState<WorkerStatusResponse | null>(null);
|
||
const [error, setError] = useState<string | null>(null);
|
||
|
||
const load = useCallback(async () => {
|
||
try {
|
||
setError(null);
|
||
const res = await fetchWorkerStatus();
|
||
setStatus(res);
|
||
} catch (err) {
|
||
setError(err instanceof Error ? err.message : "Failed");
|
||
}
|
||
}, []);
|
||
|
||
useEffect(() => {
|
||
void load();
|
||
const id = setInterval(() => void load(), 15_000);
|
||
return () => clearInterval(id);
|
||
}, [load]);
|
||
|
||
if (error) {
|
||
return (
|
||
<div className="worker-status worker-status--error">
|
||
<span className="worker-status__dot worker-status__dot--offline" />
|
||
Worker: error ({error})
|
||
</div>
|
||
);
|
||
}
|
||
|
||
if (!status) {
|
||
return (
|
||
<div className="worker-status">
|
||
<span className="worker-status__dot worker-status__dot--unknown" />
|
||
Worker: checking…
|
||
</div>
|
||
);
|
||
}
|
||
|
||
return (
|
||
<div className={`worker-status ${status.online ? "worker-status--online" : "worker-status--offline"}`}>
|
||
<span className={`worker-status__dot ${status.online ? "worker-status__dot--online" : "worker-status__dot--offline"}`} />
|
||
<span className="worker-status__label">
|
||
{status.online ? `${status.workers.length} worker${status.workers.length !== 1 ? "s" : ""} online` : "Workers offline"}
|
||
</span>
|
||
{status.workers.map((w) => (
|
||
<span key={w.name} className="worker-status__detail" title={w.name}>
|
||
{w.active_tasks.length > 0
|
||
? `${w.active_tasks.length} active`
|
||
: "idle"}
|
||
{w.pool_size != null && ` · pool ${w.pool_size}`}
|
||
</span>
|
||
))}
|
||
</div>
|
||
);
|
||
});
|
||
|
||
// ── Debug Mode Toggle ────────────────────────────────────────────────────────
|
||
|
||
function DebugModeToggle({
|
||
debugMode,
|
||
onDebugModeChange,
|
||
}: {
|
||
debugMode: boolean | null;
|
||
onDebugModeChange: (mode: boolean) => void;
|
||
}) {
|
||
const [debugLoading, setDebugLoading] = useState(false);
|
||
|
||
async function handleToggle() {
|
||
if (debugMode === null || debugLoading) return;
|
||
setDebugLoading(true);
|
||
try {
|
||
const res = await setDebugMode(!debugMode);
|
||
onDebugModeChange(res.debug_mode);
|
||
} catch {
|
||
// swallow — leave previous state
|
||
} finally {
|
||
setDebugLoading(false);
|
||
}
|
||
}
|
||
|
||
if (debugMode === null) return null;
|
||
|
||
return (
|
||
<div className="debug-toggle" title="When enabled, pipeline runs capture full LLM prompts and responses for inspection in the event log">
|
||
<span className="debug-toggle__label">
|
||
Debug {debugMode ? "On" : "Off"}
|
||
</span>
|
||
<button
|
||
type="button"
|
||
className={`debug-toggle__switch ${debugMode ? "debug-toggle__switch--active" : ""}`}
|
||
onClick={handleToggle}
|
||
disabled={debugLoading}
|
||
aria-label={`Turn debug mode ${debugMode ? "off" : "on"}`}
|
||
/>
|
||
</div>
|
||
);
|
||
}
|
||
|
||
// ── Status Filter ────────────────────────────────────────────────────────────
|
||
|
||
const StatusFilter = React.memo(function StatusFilter({
|
||
videos,
|
||
activeFilter,
|
||
onFilterChange,
|
||
}: {
|
||
videos: PipelineVideoItem[];
|
||
activeFilter: string | null;
|
||
onFilterChange: (filter: string | null) => void;
|
||
}) {
|
||
// Fixed display order for pipeline lifecycle
|
||
const STATUS_ORDER = ["not_started", "queued", "processing", "error", "complete"];
|
||
const present = new Set(videos.map((v) => v.processing_status));
|
||
const ordered = STATUS_ORDER.filter((s) => present.has(s));
|
||
|
||
if (ordered.length <= 1) return null;
|
||
|
||
return (
|
||
<div className="filter-tabs">
|
||
<button
|
||
type="button"
|
||
className={`filter-tab ${activeFilter === null ? "filter-tab--active" : ""}`}
|
||
onClick={() => onFilterChange(null)}
|
||
>
|
||
All ({videos.length})
|
||
</button>
|
||
{ordered.map((status) => {
|
||
const count = videos.filter((v) => v.processing_status === status).length;
|
||
return (
|
||
<button
|
||
key={status}
|
||
type="button"
|
||
className={`filter-tab ${activeFilter === status ? "filter-tab--active" : ""}`}
|
||
onClick={() => onFilterChange(status)}
|
||
>
|
||
{STATUS_LABELS[status] ?? status} ({count})
|
||
</button>
|
||
);
|
||
})}
|
||
</div>
|
||
);
|
||
});
|
||
|
||
// ── Stage Tab View ───────────────────────────────────────────────────────────
|
||
|
||
const STAGE_TAB_ORDER = [
|
||
{ key: "stage2_segmentation", label: "Segment", short: "S2" },
|
||
{ key: "stage3_extraction", label: "Extract", short: "S3" },
|
||
{ key: "stage4_classification", label: "Classify", short: "S4" },
|
||
{ key: "stage5_synthesis", label: "Synthesize", short: "S5" },
|
||
{ key: "stage6_embed_and_index", label: "Embed", short: "S6" },
|
||
];
|
||
|
||
function stageTabStatus(events: PipelineEvent[]): "idle" | "running" | "done" | "error" {
|
||
if (events.length === 0) return "idle";
|
||
const hasError = events.some((e) => e.event_type === "error");
|
||
if (hasError) return "error";
|
||
const hasComplete = events.some((e) => e.event_type === "complete");
|
||
if (hasComplete) return "done";
|
||
const hasStart = events.some((e) => e.event_type === "start");
|
||
if (hasStart) return "running";
|
||
return "running"; // llm_call without start — treat as running
|
||
}
|
||
|
||
function StageTabView({ videoId, runId, status }: { videoId: string; runId: string; status: string }) {
|
||
const [events, setEvents] = useState<PipelineEvent[]>([]);
|
||
const [loading, setLoading] = useState(true);
|
||
const [activeTab, setActiveTab] = useState<string | null>(null);
|
||
const initialSelect = useRef(false);
|
||
|
||
const load = useCallback(async (silent = false) => {
|
||
if (!silent) setLoading(true);
|
||
try {
|
||
const res = await fetchPipelineEvents(videoId, {
|
||
offset: 0,
|
||
limit: 200,
|
||
order: "asc",
|
||
run_id: runId,
|
||
});
|
||
setEvents(res.items);
|
||
// Auto-select tab on first load only
|
||
if (!initialSelect.current && res.items.length > 0) {
|
||
initialSelect.current = true;
|
||
const stages = new Set(res.items.map((e) => e.stage));
|
||
const errorStage = res.items.find((e) => e.event_type === "error")?.stage;
|
||
if (errorStage) {
|
||
setActiveTab(errorStage);
|
||
} else {
|
||
const lastStage = [...STAGE_TAB_ORDER].reverse().find((s) => stages.has(s.key));
|
||
if (lastStage) setActiveTab(lastStage.key);
|
||
}
|
||
}
|
||
} catch {
|
||
// silent
|
||
} finally {
|
||
if (!silent) setLoading(false);
|
||
}
|
||
}, [videoId, runId]);
|
||
|
||
useEffect(() => { void load(); }, [load]);
|
||
|
||
// Auto-refresh while processing
|
||
useEffect(() => {
|
||
if (status !== "running") return;
|
||
const id = setInterval(() => void load(true), 8_000);
|
||
return () => clearInterval(id);
|
||
}, [status, load]);
|
||
|
||
if (loading) return <div className="loading">Loading stages…</div>;
|
||
|
||
// Group events by stage
|
||
const byStage = new Map<string, PipelineEvent[]>();
|
||
for (const evt of events) {
|
||
const list = byStage.get(evt.stage) ?? [];
|
||
list.push(evt);
|
||
byStage.set(evt.stage, list);
|
||
}
|
||
|
||
const tabEvents = activeTab ? (byStage.get(activeTab) ?? []) : [];
|
||
|
||
// Compute summary stats for active tab
|
||
const tabTokens = tabEvents.reduce((sum, e) => sum + (e.total_tokens ?? 0), 0);
|
||
const tabLlmCalls = tabEvents.filter((e) => e.event_type === "llm_call").length;
|
||
const tabDuration = (() => {
|
||
const starts = tabEvents.filter((e) => e.event_type === "start");
|
||
const ends = tabEvents.filter((e) => e.event_type === "complete" || e.event_type === "error");
|
||
if (starts.length > 0 && ends.length > 0) {
|
||
const endEvt = ends[ends.length - 1];
|
||
const startEvt = starts[0];
|
||
if (endEvt && startEvt && endEvt.created_at && startEvt.created_at) {
|
||
const ms = new Date(endEvt.created_at).getTime() - new Date(startEvt.created_at).getTime();
|
||
if (ms > 0) return formatElapsed(startEvt.created_at);
|
||
}
|
||
}
|
||
return null;
|
||
})();
|
||
|
||
return (
|
||
<div className="stage-tabs">
|
||
<div className="stage-tabs__bar">
|
||
{STAGE_TAB_ORDER.map((stage) => {
|
||
const stageEvents = byStage.get(stage.key) ?? [];
|
||
const tabStatus = stageTabStatus(stageEvents);
|
||
const isActive = activeTab === stage.key;
|
||
return (
|
||
<button
|
||
key={stage.key}
|
||
className={`stage-tabs__tab stage-tabs__tab--${tabStatus}${isActive ? " stage-tabs__tab--active" : ""}`}
|
||
onClick={() => setActiveTab(stage.key)}
|
||
disabled={tabStatus === "idle"}
|
||
title={stageEvents.length > 0 ? `${stageEvents.length} events` : "Not started"}
|
||
>
|
||
<span className="stage-tabs__indicator" />
|
||
<span className="stage-tabs__label">{stage.label}</span>
|
||
</button>
|
||
);
|
||
})}
|
||
</div>
|
||
|
||
{activeTab && tabEvents.length > 0 && (
|
||
<div className="stage-tabs__panel">
|
||
<div className="stage-tabs__summary">
|
||
{tabLlmCalls > 0 && (
|
||
<span className="stage-tabs__stat">{tabLlmCalls} LLM call{tabLlmCalls !== 1 ? "s" : ""}</span>
|
||
)}
|
||
{tabTokens > 0 && (
|
||
<span className="stage-tabs__stat">{formatTokens(tabTokens)} tokens</span>
|
||
)}
|
||
{tabDuration && (
|
||
<span className="stage-tabs__stat">{tabDuration}</span>
|
||
)}
|
||
</div>
|
||
<div className="stage-tabs__events">
|
||
{tabEvents.map((evt) => (
|
||
<div key={evt.id} className={`pipeline-event pipeline-event--${evt.event_type}`}>
|
||
<div className="pipeline-event__row">
|
||
<span className="pipeline-event__icon">{eventTypeIcon(evt.event_type)}</span>
|
||
<span className={`pipeline-badge pipeline-badge--event-${evt.event_type}`}>
|
||
{evt.event_type}
|
||
</span>
|
||
{typeof evt.payload?.context === "string" && (
|
||
<span className="pipeline-event__context" title="Processing context">
|
||
{evt.payload.context}
|
||
</span>
|
||
)}
|
||
{evt.model && <span className="pipeline-event__model">{evt.model}</span>}
|
||
{evt.total_tokens != null && evt.total_tokens > 0 && (
|
||
<span className="pipeline-event__tokens" title={`prompt: ${evt.prompt_tokens ?? 0} / completion: ${evt.completion_tokens ?? 0}`}>
|
||
{formatTokens(evt.total_tokens)} tok
|
||
</span>
|
||
)}
|
||
{evt.duration_ms != null && (
|
||
<span className="pipeline-event__duration">{evt.duration_ms}ms</span>
|
||
)}
|
||
<span className="pipeline-event__time">{formatDate(evt.created_at)}</span>
|
||
</div>
|
||
{evt.event_type === "error" && evt.payload?.error != null && (
|
||
<div className="stage-tabs__error-detail">
|
||
{`${evt.payload.error}`.slice(0, 500)}
|
||
</div>
|
||
)}
|
||
<JsonViewer data={evt.payload} />
|
||
<DebugPayloadViewer event={evt} />
|
||
</div>
|
||
))}
|
||
</div>
|
||
</div>
|
||
)}
|
||
|
||
{activeTab && tabEvents.length === 0 && (
|
||
<div className="stage-tabs__panel stage-tabs__panel--empty">
|
||
Stage not started yet.
|
||
</div>
|
||
)}
|
||
</div>
|
||
);
|
||
}
|
||
|
||
// ── Run List ─────────────────────────────────────────────────────────────────
|
||
|
||
const TRIGGER_LABELS: Record<string, string> = {
|
||
manual: "Manual",
|
||
clean_reprocess: "Clean Reprocess",
|
||
auto_ingest: "Auto Ingest",
|
||
bulk: "Bulk",
|
||
};
|
||
|
||
const RUN_STATUS_CLASS: Record<string, string> = {
|
||
running: "pipeline-badge--active",
|
||
complete: "pipeline-badge--success",
|
||
error: "pipeline-badge--error",
|
||
cancelled: "pipeline-badge--pending",
|
||
};
|
||
|
||
function RunList({ videoId, videoStatus }: { videoId: string; videoStatus: string }) {
|
||
const [runs, setRuns] = useState<PipelineRunItem[]>([]);
|
||
const [legacyCount, setLegacyCount] = useState(0);
|
||
const [loading, setLoading] = useState(true);
|
||
const [expandedRunId, setExpandedRunId] = useState<string | null>(null);
|
||
const [showLegacy, setShowLegacy] = useState(false);
|
||
|
||
const load = useCallback(async (silent = false) => {
|
||
if (!silent) setLoading(true);
|
||
try {
|
||
const res = await fetchPipelineRuns(videoId);
|
||
setRuns(res.items);
|
||
setLegacyCount(res.legacy_event_count);
|
||
// Auto-expand the latest run on first load
|
||
if (!silent && res.items.length > 0 && expandedRunId === null) {
|
||
const firstRun = res.items[0];
|
||
if (firstRun) setExpandedRunId(firstRun.id);
|
||
}
|
||
} catch {
|
||
// silently fail
|
||
} finally {
|
||
if (!silent) setLoading(false);
|
||
}
|
||
}, [videoId, expandedRunId]);
|
||
|
||
useEffect(() => {
|
||
void load();
|
||
}, [load]);
|
||
|
||
// Auto-refresh when video is processing
|
||
useEffect(() => {
|
||
if (videoStatus !== "processing" && videoStatus !== "queued") return;
|
||
const id = setInterval(() => void load(true), 10_000);
|
||
return () => clearInterval(id);
|
||
}, [videoStatus, load]);
|
||
|
||
if (loading) return <div className="loading">Loading runs…</div>;
|
||
|
||
if (runs.length === 0 && legacyCount === 0) {
|
||
return <div className="pipeline-events__empty">No pipeline runs recorded.</div>;
|
||
}
|
||
|
||
return (
|
||
<div className="run-list">
|
||
{runs.map((run) => {
|
||
const isExpanded = expandedRunId === run.id;
|
||
return (
|
||
<div key={run.id} className={`run-card run-card--${run.status}`}>
|
||
<button
|
||
className="run-card__header"
|
||
onClick={() => setExpandedRunId(isExpanded ? null : run.id)}
|
||
aria-expanded={isExpanded}
|
||
>
|
||
<span className="run-card__arrow">{isExpanded ? "▾" : "▸"}</span>
|
||
<span className="run-card__number">Run #{run.run_number}</span>
|
||
<span className="run-card__trigger">{TRIGGER_LABELS[run.trigger] ?? run.trigger}</span>
|
||
<span className={`pipeline-badge ${RUN_STATUS_CLASS[run.status] ?? ""}`}>
|
||
{run.status}
|
||
</span>
|
||
<span className="run-card__time">{formatDate(run.started_at)}</span>
|
||
{run.finished_at && (
|
||
<span className="run-card__duration" title={`Finished: ${formatDate(run.finished_at)}`}>
|
||
→ {formatDate(run.finished_at)}
|
||
</span>
|
||
)}
|
||
<span className="run-card__tokens">{formatTokens(run.total_tokens)} tokens</span>
|
||
<span className="run-card__events">{run.event_count} events</span>
|
||
{run.error_stage && (
|
||
<span className="run-card__error-stage">Failed at: {run.error_stage}</span>
|
||
)}
|
||
</button>
|
||
{isExpanded && (
|
||
<div className="run-card__body">
|
||
<StageTabView videoId={videoId} runId={run.id} status={run.status} />
|
||
</div>
|
||
)}
|
||
</div>
|
||
);
|
||
})}
|
||
|
||
{legacyCount > 0 && (
|
||
<div className="run-card run-card--legacy">
|
||
<button
|
||
className="run-card__header"
|
||
onClick={() => setShowLegacy((v) => !v)}
|
||
aria-expanded={showLegacy}
|
||
>
|
||
<span className="run-card__arrow">{showLegacy ? "▾" : "▸"}</span>
|
||
<span className="run-card__number">Legacy</span>
|
||
<span className="run-card__trigger">Pre-run tracking</span>
|
||
<span className="run-card__events">{legacyCount} events</span>
|
||
</button>
|
||
{showLegacy && (
|
||
<div className="run-card__body">
|
||
<EventLog videoId={videoId} status="complete" />
|
||
</div>
|
||
)}
|
||
</div>
|
||
)}
|
||
</div>
|
||
);
|
||
}
|
||
|
||
// ── Stage Timeline ───────────────────────────────────────────────────────────
|
||
|
||
const PIPELINE_STAGES = [
|
||
{ key: "stage2_segmentation", label: "Segment" },
|
||
{ key: "stage3_extraction", label: "Extract" },
|
||
{ key: "stage4_classification", label: "Classify" },
|
||
{ key: "stage5_synthesis", label: "Synthesize" },
|
||
{ key: "stage6_embed", label: "Embed" },
|
||
];
|
||
|
||
const StageTimeline = React.memo(function StageTimeline({ video }: { video: PipelineVideoItem }) {
|
||
const [, setTick] = useState(0);
|
||
const isProcessing = video.processing_status === "processing";
|
||
|
||
// Tick every second while processing to update elapsed time
|
||
useEffect(() => {
|
||
if (!isProcessing) return;
|
||
const id = setInterval(() => setTick((t) => t + 1), 1000);
|
||
return () => clearInterval(id);
|
||
}, [isProcessing]);
|
||
|
||
if (!isProcessing && video.processing_status !== "complete" && video.processing_status !== "error") {
|
||
return null;
|
||
}
|
||
|
||
const activeStage = video.active_stage;
|
||
const activeStatus = video.active_stage_status;
|
||
const activeIdx = PIPELINE_STAGES.findIndex((s) => s.key === activeStage);
|
||
|
||
return (
|
||
<div className="stage-timeline">
|
||
{PIPELINE_STAGES.map((stage, i) => {
|
||
let stateClass = "stage-timeline__step--future";
|
||
if (activeIdx >= 0) {
|
||
if (i < activeIdx) {
|
||
stateClass = "stage-timeline__step--done";
|
||
} else if (i === activeIdx) {
|
||
if (activeStatus === "start") stateClass = "stage-timeline__step--active";
|
||
else if (activeStatus === "complete") stateClass = "stage-timeline__step--done";
|
||
else if (activeStatus === "error") stateClass = "stage-timeline__step--error";
|
||
}
|
||
}
|
||
if (video.processing_status === "complete") {
|
||
stateClass = "stage-timeline__step--done";
|
||
}
|
||
|
||
return (
|
||
<div key={stage.key} className={`stage-timeline__step ${stateClass}`}>
|
||
<div className="stage-timeline__dot" title={stage.label} />
|
||
<span className="stage-timeline__label">{stage.label}</span>
|
||
</div>
|
||
);
|
||
})}
|
||
{isProcessing && activeStatus === "start" && (
|
||
<span className="stage-timeline__elapsed">{formatElapsed(video.stage_started_at)}</span>
|
||
)}
|
||
{isProcessing && video.latest_run && video.latest_run.total_tokens > 0 && (
|
||
<span className="stage-timeline__tokens">{formatTokens(video.latest_run.total_tokens)} tok</span>
|
||
)}
|
||
</div>
|
||
);
|
||
})
|
||
|
||
// ── Recent Activity Feed ─────────────────────────────────────────────────────
|
||
|
||
const RecentActivityFeed = React.memo(function RecentActivityFeed() {
|
||
const [items, setItems] = useState<RecentActivityItem[]>([]);
|
||
const [collapsed, setCollapsed] = useState(false);
|
||
const [loading, setLoading] = useState(true);
|
||
|
||
const load = useCallback(async () => {
|
||
try {
|
||
const res = await fetchRecentActivity(8);
|
||
setItems(res.items);
|
||
} catch {
|
||
// silently fail
|
||
} finally {
|
||
setLoading(false);
|
||
}
|
||
}, []);
|
||
|
||
useEffect(() => {
|
||
void load();
|
||
const id = setInterval(() => void load(), 15_000);
|
||
return () => clearInterval(id);
|
||
}, [load]);
|
||
|
||
if (loading || items.length === 0) return null;
|
||
|
||
return (
|
||
<div className="recent-activity">
|
||
<button
|
||
className="recent-activity__toggle"
|
||
onClick={() => setCollapsed((v) => !v)}
|
||
aria-expanded={!collapsed}
|
||
>
|
||
<span className="recent-activity__title">Recent Activity</span>
|
||
<span className="recent-activity__arrow">{collapsed ? "▸" : "▾"}</span>
|
||
</button>
|
||
{!collapsed && (
|
||
<div className="recent-activity__list">
|
||
{items.map((item) => (
|
||
<div key={item.id} className={`recent-activity__item recent-activity__item--${item.event_type}`}>
|
||
<span className="recent-activity__icon">
|
||
{item.event_type === "complete" ? "✓" : "✗"}
|
||
</span>
|
||
<span className="recent-activity__stage">{item.stage.replace("stage", "S").replace("_", " ")}</span>
|
||
<span className="recent-activity__file" title={item.filename}>
|
||
{item.filename}
|
||
</span>
|
||
<span className="recent-activity__creator">{item.creator_name}</span>
|
||
{item.duration_ms != null && (
|
||
<span className="recent-activity__duration">
|
||
{item.duration_ms > 60000
|
||
? `${(item.duration_ms / 60000).toFixed(1)}m`
|
||
: item.duration_ms > 1000
|
||
? `${(item.duration_ms / 1000).toFixed(1)}s`
|
||
: `${item.duration_ms}ms`}
|
||
</span>
|
||
)}
|
||
<span className="recent-activity__time">{formatDate(item.created_at)}</span>
|
||
</div>
|
||
))}
|
||
</div>
|
||
)}
|
||
</div>
|
||
);
|
||
});
|
||
|
||
// ── Main Page ────────────────────────────────────────────────────────────────
|
||
|
||
interface BulkProgress {
|
||
total: number;
|
||
completed: number;
|
||
failed: number;
|
||
current: string | null;
|
||
active: boolean;
|
||
}
|
||
|
||
interface BulkLogEntry {
|
||
filename: string;
|
||
ok: boolean;
|
||
message: string;
|
||
}
|
||
|
||
// ── Chunking Inspector ─────────────────────────────────────────────────────
|
||
|
||
function ChunkingInspector({ videoId }: { videoId: string }) {
|
||
const [data, setData] = useState<Awaited<ReturnType<typeof fetchChunkingData>> | null>(null);
|
||
const [loading, setLoading] = useState(true);
|
||
const [error, setError] = useState<string | null>(null);
|
||
const [expanded, setExpanded] = useState(false);
|
||
|
||
useEffect(() => {
|
||
if (!expanded) return;
|
||
let cancelled = false;
|
||
setLoading(true);
|
||
fetchChunkingData(videoId)
|
||
.then((res) => { if (!cancelled) { setData(res); setError(null); } })
|
||
.catch((err) => { if (!cancelled) setError(err instanceof Error ? err.message : "Failed"); })
|
||
.finally(() => { if (!cancelled) setLoading(false); });
|
||
return () => { cancelled = true; };
|
||
}, [videoId, expanded]);
|
||
|
||
return (
|
||
<div className="chunking-inspector">
|
||
<button
|
||
className="chunking-inspector__toggle"
|
||
onClick={() => setExpanded(!expanded)}
|
||
>
|
||
{expanded ? "▾" : "▸"} Chunking Inspector
|
||
</button>
|
||
|
||
{expanded && loading && <div className="chunking-inspector__loading">Loading chunking data...</div>}
|
||
{expanded && error && <div className="chunking-inspector__error">{error}</div>}
|
||
|
||
{expanded && data && (
|
||
<div className="chunking-inspector__body">
|
||
<div className="chunking-inspector__summary">
|
||
<span>{data.total_segments} segments</span>
|
||
<span>{data.total_moments} moments</span>
|
||
<span>Classification: {data.classification_source}</span>
|
||
<span>Chunk size: {data.synthesis_chunk_size}</span>
|
||
</div>
|
||
|
||
{/* Topic Boundaries */}
|
||
<div className="chunking-inspector__section">
|
||
<h4>Topic Boundaries (Stage 2)</h4>
|
||
<div className="chunking-inspector__topics">
|
||
{data.topic_boundaries.map((tb, i) => (
|
||
<div key={i} className="chunking-topic" title={tb.topic_label}>
|
||
<span className="chunking-topic__label">{tb.topic_label}</span>
|
||
<span className="chunking-topic__meta">
|
||
{tb.segment_count} segs · {tb.start_time.toFixed(0)}s-{tb.end_time.toFixed(0)}s
|
||
</span>
|
||
</div>
|
||
))}
|
||
</div>
|
||
</div>
|
||
|
||
{/* Synthesis Groups */}
|
||
<div className="chunking-inspector__section">
|
||
<h4>Synthesis Groups (Stage 5 Input)</h4>
|
||
<div className="chunking-inspector__groups">
|
||
{data.synthesis_groups.map((g) => (
|
||
<div
|
||
key={g.category}
|
||
className={`chunking-group ${g.exceeds_chunk_threshold ? "chunking-group--split" : ""}`}
|
||
>
|
||
<span className="chunking-group__category">{g.category}</span>
|
||
<span className="chunking-group__count">{g.moment_count} moments</span>
|
||
{g.exceeds_chunk_threshold && (
|
||
<span className="chunking-group__warn">
|
||
Will split into {g.chunks_needed} chunks
|
||
</span>
|
||
)}
|
||
</div>
|
||
))}
|
||
</div>
|
||
</div>
|
||
</div>
|
||
)}
|
||
</div>
|
||
);
|
||
}
|
||
|
||
|
||
export default function AdminPipeline() {
|
||
useDocumentTitle("Pipeline Management — Chrysopedia");
|
||
const [searchParams] = useSearchParams();
|
||
const [videos, setVideos] = useState<PipelineVideoItem[]>([]);
|
||
const [loading, setLoading] = useState(true);
|
||
const [error, setError] = useState<string | null>(null);
|
||
const [expandedId, setExpandedId] = useState<string | null>(null);
|
||
const [actionLoading, setActionLoading] = useState<string | null>(null);
|
||
const [actionMessage, setActionMessage] = useState<{ id: string; text: string; ok: boolean } | null>(null);
|
||
const [activeFilter, setActiveFilter] = useState<string | null>(null);
|
||
const [debugMode, setDebugModeState] = useState<boolean | null>(null);
|
||
const [creators, setCreators] = useState<{ name: string; slug: string }[]>([]);
|
||
const [creatorFilter, setCreatorFilter] = useState<string | null>(null);
|
||
const [searchQuery, setSearchQuery] = useState("");
|
||
const [selectedIds, setSelectedIds] = useState<Set<string>>(new Set());
|
||
// Re-run stage modal state
|
||
const [rerunModalVideo, setRerunModalVideo] = useState<string | null>(null);
|
||
const [rerunStageSelect, setRerunStageSelect] = useState("stage5_synthesis");
|
||
const [rerunPromptOverride, setRerunPromptOverride] = useState("");
|
||
// Stale pages state
|
||
const [stalePagesCount, setStalePagesCount] = useState<number | null>(null);
|
||
const [bulkProgress, setBulkProgress] = useState<BulkProgress | null>(null);
|
||
const [bulkLog, setBulkLog] = useState<BulkLogEntry[]>([]);
|
||
const [changedIds, setChangedIds] = useState<Set<string>>(new Set());
|
||
const [autoRefresh, setAutoRefresh] = useState(true);
|
||
const bulkCancelRef = useRef(false);
|
||
const prevStatusRef = useRef<Map<string, string>>(new Map());
|
||
const videoRefs = useRef<Map<string, HTMLDivElement>>(new Map());
|
||
const deepLinked = useRef(false);
|
||
|
||
// Compute filtered list (status + creator) — memoized to avoid recomputing on unrelated state changes
|
||
const filteredVideos = useMemo(() => videos.filter((v) => {
|
||
if (activeFilter !== null && v.processing_status !== activeFilter) return false;
|
||
if (creatorFilter !== null && v.creator_name !== creatorFilter) return false;
|
||
if (searchQuery) {
|
||
const q = searchQuery.toLowerCase();
|
||
if (!v.filename.toLowerCase().includes(q) && !v.creator_name.toLowerCase().includes(q)) return false;
|
||
}
|
||
return true;
|
||
}), [videos, activeFilter, creatorFilter, searchQuery]);
|
||
|
||
const load = useCallback(async (isAutoRefresh = false) => {
|
||
if (!isAutoRefresh) setLoading(true);
|
||
setError(null);
|
||
try {
|
||
const res = await fetchPipelineVideos();
|
||
// Detect status changes for highlight animation
|
||
if (prevStatusRef.current.size > 0) {
|
||
const changed = new Set<string>();
|
||
for (const v of res.items) {
|
||
const prev = prevStatusRef.current.get(v.id);
|
||
if (prev && prev !== v.processing_status) {
|
||
changed.add(v.id);
|
||
}
|
||
}
|
||
if (changed.size > 0) {
|
||
setChangedIds(changed);
|
||
// Clear highlights after animation
|
||
setTimeout(() => setChangedIds(new Set()), 2000);
|
||
}
|
||
}
|
||
// Store current statuses for next comparison
|
||
const statusMap = new Map<string, string>();
|
||
for (const v of res.items) {
|
||
statusMap.set(v.id, v.processing_status);
|
||
}
|
||
prevStatusRef.current = statusMap;
|
||
setVideos(res.items);
|
||
} catch (err) {
|
||
if (!isAutoRefresh) {
|
||
setError(err instanceof Error ? err.message : "Failed to load videos");
|
||
}
|
||
} finally {
|
||
if (!isAutoRefresh) setLoading(false);
|
||
}
|
||
}, []);
|
||
|
||
useEffect(() => {
|
||
void load();
|
||
}, [load]);
|
||
|
||
// Auto-refresh every 15 seconds
|
||
useEffect(() => {
|
||
if (!autoRefresh) return;
|
||
const id = setInterval(() => void load(true), 15_000);
|
||
return () => clearInterval(id);
|
||
}, [autoRefresh, load]);
|
||
|
||
// Load creators for filter dropdown
|
||
useEffect(() => {
|
||
fetchCreators({ limit: 200 })
|
||
.then((res) => {
|
||
const list = res.items.map((c: { name: string; slug: string }) => ({
|
||
name: c.name,
|
||
slug: c.slug,
|
||
}));
|
||
list.sort((a: { name: string }, b: { name: string }) => a.name.localeCompare(b.name));
|
||
setCreators(list);
|
||
})
|
||
.catch(() => {
|
||
// silently fail — filter stays hidden
|
||
});
|
||
}, []);
|
||
|
||
// Load stale page count
|
||
useEffect(() => {
|
||
fetchStalePages()
|
||
.then((res) => setStalePagesCount(res.stale_pages))
|
||
.catch(() => { /* silently fail */ });
|
||
}, []);
|
||
|
||
const handleBulkResynth = async () => {
|
||
if (!confirm("Re-run stage 5 synthesis on all completed videos with the current prompt?")) return;
|
||
try {
|
||
const res = await bulkResynthesize();
|
||
setActionMessage({
|
||
id: "__bulk__",
|
||
text: `Bulk re-synthesize dispatched: ${res.dispatched}/${res.total} videos`,
|
||
ok: true,
|
||
});
|
||
setStalePagesCount(null); // will refresh
|
||
} catch (err) {
|
||
setActionMessage({
|
||
id: "__bulk__",
|
||
text: err instanceof Error ? err.message : "Bulk re-synth failed",
|
||
ok: false,
|
||
});
|
||
}
|
||
};
|
||
|
||
const handleWipeAll = async () => {
|
||
if (!confirm("WIPE ALL pipeline output? This deletes all technique pages, key moments, pipeline events, runs, and Qdrant vectors. Creators, videos, and transcripts are preserved. This cannot be undone.")) return;
|
||
if (!confirm("Are you sure? This is irreversible.")) return;
|
||
try {
|
||
const res = await wipeAllOutput();
|
||
setActionMessage({
|
||
id: "__wipe__",
|
||
text: `Wiped: ${JSON.stringify(res.deleted)}`,
|
||
ok: true,
|
||
});
|
||
setStalePagesCount(null);
|
||
void load();
|
||
} catch (err) {
|
||
setActionMessage({
|
||
id: "__wipe__",
|
||
text: err instanceof Error ? err.message : "Wipe failed",
|
||
ok: false,
|
||
});
|
||
}
|
||
};
|
||
|
||
// Deep-link: auto-expand and scroll to ?video=<id> on first load
|
||
useEffect(() => {
|
||
if (deepLinked.current || loading || videos.length === 0) return;
|
||
const targetVideoId = searchParams.get("video");
|
||
if (!targetVideoId) return;
|
||
const match = videos.find((v) => v.id === targetVideoId);
|
||
if (!match) return;
|
||
deepLinked.current = true;
|
||
setExpandedId(targetVideoId);
|
||
requestAnimationFrame(() => {
|
||
const el = videoRefs.current.get(targetVideoId);
|
||
if (el) el.scrollIntoView({ behavior: "smooth", block: "start" });
|
||
});
|
||
}, [loading, videos, searchParams]);
|
||
|
||
useEffect(() => {
|
||
let cancelled = false;
|
||
fetchDebugMode()
|
||
.then((res) => {
|
||
if (!cancelled) setDebugModeState(res.debug_mode);
|
||
})
|
||
.catch(() => {});
|
||
return () => { cancelled = true; };
|
||
}, []);
|
||
|
||
// Clear selection when filters change
|
||
useEffect(() => {
|
||
setSelectedIds(new Set());
|
||
}, [activeFilter, creatorFilter, searchQuery]);
|
||
|
||
const handleTrigger = async (videoId: string) => {
|
||
setActionLoading(videoId);
|
||
setActionMessage(null);
|
||
try {
|
||
const res = await cleanRetriggerPipeline(videoId);
|
||
setActionMessage({ id: videoId, text: `Retriggered (${res.status})`, ok: true });
|
||
setTimeout(() => void load(), 2000);
|
||
} catch (err) {
|
||
setActionMessage({
|
||
id: videoId,
|
||
text: err instanceof Error ? err.message : "Retrigger failed",
|
||
ok: false,
|
||
});
|
||
} finally {
|
||
setActionLoading(null);
|
||
}
|
||
};
|
||
|
||
const handleRevoke = async (videoId: string) => {
|
||
setActionLoading(videoId);
|
||
setActionMessage(null);
|
||
try {
|
||
const res = await revokePipeline(videoId);
|
||
setActionMessage({
|
||
id: videoId,
|
||
text: res.tasks_revoked > 0
|
||
? `Revoked ${res.tasks_revoked} task${res.tasks_revoked !== 1 ? "s" : ""}`
|
||
: "No active tasks",
|
||
ok: true,
|
||
});
|
||
setTimeout(() => void load(), 2000);
|
||
} catch (err) {
|
||
setActionMessage({
|
||
id: videoId,
|
||
text: err instanceof Error ? err.message : "Revoke failed",
|
||
ok: false,
|
||
});
|
||
} finally {
|
||
setActionLoading(null);
|
||
}
|
||
};
|
||
|
||
const handleRerunStage = async (videoId: string) => {
|
||
setActionLoading(videoId);
|
||
setActionMessage(null);
|
||
try {
|
||
const res = await rerunStage(
|
||
videoId,
|
||
rerunStageSelect,
|
||
rerunPromptOverride.trim() || undefined,
|
||
);
|
||
setActionMessage({
|
||
id: videoId,
|
||
text: `Stage re-run dispatched: ${res.stage}${res.prompt_override ? " (with prompt override)" : ""}`,
|
||
ok: true,
|
||
});
|
||
setRerunModalVideo(null);
|
||
setRerunPromptOverride("");
|
||
setTimeout(() => void load(), 2000);
|
||
} catch (err) {
|
||
setActionMessage({
|
||
id: videoId,
|
||
text: err instanceof Error ? err.message : "Stage re-run failed",
|
||
ok: false,
|
||
});
|
||
} finally {
|
||
setActionLoading(null);
|
||
}
|
||
};
|
||
|
||
const toggleExpand = (id: string) => {
|
||
setExpandedId((prev) => (prev === id ? null : id));
|
||
};
|
||
|
||
// ── Selection ───────────────────────────────────────────────────────────
|
||
|
||
const toggleSelect = (id: string) => {
|
||
setSelectedIds((prev) => {
|
||
const next = new Set(prev);
|
||
if (next.has(id)) next.delete(id);
|
||
else next.add(id);
|
||
return next;
|
||
});
|
||
};
|
||
|
||
const toggleSelectAll = () => {
|
||
const visibleIds = filteredVideos.map((v) => v.id);
|
||
const allSelected = visibleIds.every((id) => selectedIds.has(id));
|
||
if (allSelected) {
|
||
setSelectedIds(new Set());
|
||
} else {
|
||
setSelectedIds(new Set(visibleIds));
|
||
}
|
||
};
|
||
|
||
const allVisibleSelected = filteredVideos.length > 0 && filteredVideos.every((v) => selectedIds.has(v.id));
|
||
|
||
// ── Bulk Actions ────────────────────────────────────────────────────────
|
||
|
||
const runBulk = async (clean: boolean) => {
|
||
const ids = Array.from(selectedIds);
|
||
if (ids.length === 0) return;
|
||
|
||
bulkCancelRef.current = false;
|
||
setBulkLog([]);
|
||
setBulkProgress({ total: ids.length, completed: 0, failed: 0, current: null, active: true });
|
||
|
||
let completed = 0;
|
||
let failed = 0;
|
||
|
||
for (const id of ids) {
|
||
if (bulkCancelRef.current) break;
|
||
|
||
const video = videos.find((v) => v.id === id);
|
||
const filename = video?.filename ?? id;
|
||
setBulkProgress((p) => p ? { ...p, current: filename } : null);
|
||
|
||
try {
|
||
if (clean) {
|
||
await cleanRetriggerPipeline(id);
|
||
} else {
|
||
await triggerPipeline(id);
|
||
}
|
||
completed++;
|
||
setBulkLog((prev) => [...prev, { filename, ok: true, message: clean ? "clean retriggered" : "triggered" }]);
|
||
} catch (err) {
|
||
failed++;
|
||
setBulkLog((prev) => [...prev, {
|
||
filename,
|
||
ok: false,
|
||
message: err instanceof Error ? err.message : "failed",
|
||
}]);
|
||
}
|
||
|
||
setBulkProgress((p) => p ? { ...p, completed: completed, failed } : null);
|
||
|
||
// Brief pause between dispatches to avoid slamming the API
|
||
if (!bulkCancelRef.current && ids.indexOf(id) < ids.length - 1) {
|
||
await new Promise((r) => setTimeout(r, 500));
|
||
}
|
||
}
|
||
|
||
setBulkProgress((p) => p ? { ...p, active: false, current: null } : null);
|
||
setSelectedIds(new Set());
|
||
// Refresh video list after bulk completes
|
||
setTimeout(() => void load(), 2000);
|
||
};
|
||
|
||
const cancelBulk = () => {
|
||
bulkCancelRef.current = true;
|
||
};
|
||
|
||
return (
|
||
<div className="admin-pipeline">
|
||
<div className="admin-pipeline__header">
|
||
<div>
|
||
<h1 className="admin-pipeline__title">Pipeline Management</h1>
|
||
<p className="admin-pipeline__subtitle">
|
||
{videos.length} video{videos.length !== 1 ? "s" : ""}
|
||
</p>
|
||
</div>
|
||
<div className="admin-pipeline__header-right">
|
||
{stalePagesCount !== null && stalePagesCount > 0 && (
|
||
<button
|
||
className="btn btn--small btn--warning"
|
||
onClick={() => void handleBulkResynth()}
|
||
title={`${stalePagesCount} pages synthesized with an older prompt version`}
|
||
>
|
||
{stalePagesCount} orphaned pages
|
||
</button>
|
||
)}
|
||
<button
|
||
className="btn btn--small btn--danger"
|
||
onClick={() => void handleWipeAll()}
|
||
title="Delete all pipeline output (technique pages, moments, events). Preserves videos and transcripts."
|
||
>
|
||
Wipe All Output
|
||
</button>
|
||
<DebugModeToggle debugMode={debugMode} onDebugModeChange={setDebugModeState} />
|
||
<WorkerStatus />
|
||
<label className="auto-refresh-toggle" title="Auto-refresh video list every 15 seconds">
|
||
<input
|
||
type="checkbox"
|
||
checked={autoRefresh}
|
||
onChange={(e) => setAutoRefresh(e.target.checked)}
|
||
/>
|
||
Auto-refresh
|
||
</label>
|
||
<button className="btn btn--secondary" onClick={() => void load()} disabled={loading}>
|
||
↻ Refresh
|
||
</button>
|
||
</div>
|
||
</div>
|
||
|
||
{loading ? (
|
||
<div className="loading">Loading videos…</div>
|
||
) : error ? (
|
||
<div className="loading error-text">Error: {error}</div>
|
||
) : videos.length === 0 ? (
|
||
<div className="empty-state">No videos in pipeline.</div>
|
||
) : (
|
||
<>
|
||
<RecentActivityFeed />
|
||
<div className="admin-pipeline__filters">
|
||
<StatusFilter
|
||
videos={videos}
|
||
activeFilter={activeFilter}
|
||
onFilterChange={setActiveFilter}
|
||
/>
|
||
{creators.length > 1 && (
|
||
<div className="creator-filter">
|
||
<select
|
||
className="creator-filter__select"
|
||
value={creatorFilter ?? ""}
|
||
onChange={(e) => setCreatorFilter(e.target.value || null)}
|
||
>
|
||
<option value="">All Creators</option>
|
||
{creators.map((c) => (
|
||
<option key={c.slug} value={c.name}>{c.name}</option>
|
||
))}
|
||
</select>
|
||
</div>
|
||
)}
|
||
<div className="pipeline-search">
|
||
<input
|
||
type="text"
|
||
className="pipeline-search__input"
|
||
placeholder="Filter by filename or creator…"
|
||
value={searchQuery}
|
||
onChange={(e) => setSearchQuery(e.target.value)}
|
||
/>
|
||
{searchQuery && (
|
||
<button
|
||
className="pipeline-search__clear"
|
||
onClick={() => setSearchQuery("")}
|
||
aria-label="Clear search"
|
||
>
|
||
✕
|
||
</button>
|
||
)}
|
||
</div>
|
||
</div>
|
||
|
||
{/* Bulk toolbar */}
|
||
{selectedIds.size > 0 && (
|
||
<div className="bulk-toolbar">
|
||
<span className="bulk-toolbar__count">
|
||
{selectedIds.size} selected
|
||
</span>
|
||
{bulkProgress?.active ? (
|
||
<>
|
||
<div className="bulk-toolbar__progress">
|
||
<div className="bulk-toolbar__progress-bar">
|
||
<div
|
||
className="bulk-toolbar__progress-fill"
|
||
style={{ width: `${((bulkProgress.completed + bulkProgress.failed) / bulkProgress.total) * 100}%` }}
|
||
/>
|
||
</div>
|
||
<span className="bulk-toolbar__progress-text">
|
||
{bulkProgress.completed + bulkProgress.failed}/{bulkProgress.total}
|
||
{bulkProgress.failed > 0 && <span className="error-text"> ({bulkProgress.failed} failed)</span>}
|
||
</span>
|
||
{bulkProgress.current && (
|
||
<span className="bulk-toolbar__current" title={bulkProgress.current}>
|
||
{bulkProgress.current}
|
||
</span>
|
||
)}
|
||
</div>
|
||
<button className="btn btn--small btn--danger" onClick={cancelBulk}>
|
||
Cancel
|
||
</button>
|
||
</>
|
||
) : (
|
||
<>
|
||
<button
|
||
className="btn btn--small btn--primary"
|
||
onClick={() => void runBulk(false)}
|
||
title="Retrigger pipeline for selected videos"
|
||
>
|
||
▶ Retrigger Selected
|
||
</button>
|
||
<button
|
||
className="btn btn--small btn--warning"
|
||
onClick={() => void runBulk(true)}
|
||
title="Wipe pipeline output (events, moments, segments) then retrigger"
|
||
>
|
||
🧹 Clean Reprocess
|
||
</button>
|
||
<button
|
||
className="btn btn--small btn--secondary"
|
||
onClick={() => setSelectedIds(new Set())}
|
||
>
|
||
Clear
|
||
</button>
|
||
</>
|
||
)}
|
||
</div>
|
||
)}
|
||
|
||
{/* Bulk completion message */}
|
||
{bulkProgress && !bulkProgress.active && (
|
||
<div className={`bulk-toolbar__done ${bulkProgress.failed > 0 ? "bulk-toolbar__done--warn" : "bulk-toolbar__done--ok"}`}>
|
||
Bulk operation complete: {bulkProgress.completed} succeeded
|
||
{bulkProgress.failed > 0 && `, ${bulkProgress.failed} failed`}
|
||
{bulkCancelRef.current && " (cancelled)"}
|
||
<button className="bulk-toolbar__dismiss" onClick={() => setBulkProgress(null)}>✕</button>
|
||
</div>
|
||
)}
|
||
|
||
{/* Bulk operation log */}
|
||
{bulkLog.length > 0 && (bulkProgress?.active || (bulkProgress && !bulkProgress.active)) && (
|
||
<div className="bulk-log">
|
||
<div className="bulk-log__list">
|
||
{bulkLog.map((entry, i) => (
|
||
<div key={i} className={`bulk-log__entry ${entry.ok ? "bulk-log__entry--ok" : "bulk-log__entry--err"}`}>
|
||
<span className="bulk-log__icon">{entry.ok ? "✓" : "✗"}</span>
|
||
<span className="bulk-log__file">{entry.filename}</span>
|
||
<span className="bulk-log__msg">{entry.message}</span>
|
||
</div>
|
||
))}
|
||
</div>
|
||
</div>
|
||
)}
|
||
|
||
<div className="admin-pipeline__list">
|
||
{filteredVideos.map((video) => (
|
||
<div key={video.id} className={`pipeline-video${changedIds.has(video.id) ? " pipeline-video--changed" : ""}`} ref={(el) => { if (el) videoRefs.current.set(video.id, el); }}>
|
||
<div
|
||
className="pipeline-video__header"
|
||
onClick={() => toggleExpand(video.id)}
|
||
>
|
||
<div className="pipeline-video__checkbox" onClick={(e) => e.stopPropagation()}>
|
||
<input
|
||
type="checkbox"
|
||
checked={selectedIds.has(video.id)}
|
||
onChange={() => toggleSelect(video.id)}
|
||
disabled={bulkProgress?.active ?? false}
|
||
aria-label={`Select ${video.filename}`}
|
||
/>
|
||
</div>
|
||
<div className="pipeline-video__info">
|
||
<span className="pipeline-video__filename" title={video.filename}>
|
||
{video.filename}
|
||
</span>
|
||
<span className="pipeline-video__creator">{video.creator_name}</span>
|
||
</div>
|
||
|
||
<div className="pipeline-video__meta">
|
||
<span className={`pipeline-badge ${statusBadgeClass(video.processing_status)}`}>
|
||
{STATUS_LABELS[video.processing_status] ?? video.processing_status}
|
||
</span>
|
||
<StageTimeline video={video} />
|
||
<span className="pipeline-video__stat" title="Events">
|
||
{video.event_count} events
|
||
</span>
|
||
<span className="pipeline-video__stat" title="Total tokens used">
|
||
{formatTokens(video.total_tokens_used)} tokens
|
||
</span>
|
||
<span className="pipeline-video__time">
|
||
{formatDate(video.last_event_at)}
|
||
</span>
|
||
</div>
|
||
|
||
<div className="pipeline-video__actions" onClick={(e) => e.stopPropagation()}>
|
||
<button
|
||
className="btn btn--small btn--primary"
|
||
onClick={() => void handleTrigger(video.id)}
|
||
disabled={actionLoading === video.id}
|
||
title="Retrigger pipeline"
|
||
>
|
||
{actionLoading === video.id ? "…" : "▶ Retrigger"}
|
||
</button>
|
||
<button
|
||
className="btn btn--small btn--danger"
|
||
onClick={() => void handleRevoke(video.id)}
|
||
disabled={actionLoading === video.id}
|
||
title="Revoke active tasks"
|
||
>
|
||
{actionLoading === video.id ? "…" : "■ Revoke"}
|
||
</button>
|
||
<button
|
||
className="btn btn--small"
|
||
onClick={() => setRerunModalVideo(video.id)}
|
||
disabled={actionLoading === video.id}
|
||
title="Re-run a single pipeline stage"
|
||
>
|
||
Re-run Stage
|
||
</button>
|
||
</div>
|
||
</div>
|
||
|
||
{actionMessage?.id === video.id && (
|
||
<div className={`pipeline-video__message ${actionMessage.ok ? "pipeline-video__message--ok" : "pipeline-video__message--err"}`}>
|
||
{actionMessage.text}
|
||
</div>
|
||
)}
|
||
|
||
{expandedId === video.id && (
|
||
<div className="pipeline-video__detail">
|
||
<div className="pipeline-video__detail-meta">
|
||
<span>Created: {formatDate(video.created_at)}</span>
|
||
<span>Updated: {formatDate(video.updated_at)}</span>
|
||
</div>
|
||
<RunList videoId={video.id} videoStatus={video.processing_status} />
|
||
<ChunkingInspector videoId={video.id} />
|
||
</div>
|
||
)}
|
||
</div>
|
||
))}
|
||
{filteredVideos.length > 0 && (
|
||
<div className="admin-pipeline__select-all">
|
||
<label>
|
||
<input
|
||
type="checkbox"
|
||
checked={allVisibleSelected}
|
||
onChange={toggleSelectAll}
|
||
disabled={bulkProgress?.active ?? false}
|
||
/>
|
||
{allVisibleSelected ? "Deselect" : "Select"} all {filteredVideos.length} visible
|
||
</label>
|
||
</div>
|
||
)}
|
||
</div>
|
||
</>
|
||
)}
|
||
|
||
{/* Re-run Stage Modal */}
|
||
{rerunModalVideo && (
|
||
<div className="modal-overlay" onClick={() => setRerunModalVideo(null)}>
|
||
<div className="modal-content" onClick={(e) => e.stopPropagation()}>
|
||
<h3>Re-run Single Stage</h3>
|
||
<p className="modal-subtitle">
|
||
Re-run a specific pipeline stage without re-running predecessors.
|
||
</p>
|
||
<div className="modal-field">
|
||
<label htmlFor="rerun-stage-select">Stage</label>
|
||
<select
|
||
id="rerun-stage-select"
|
||
value={rerunStageSelect}
|
||
onChange={(e) => setRerunStageSelect(e.target.value)}
|
||
>
|
||
<option value="stage2_segmentation">Stage 2: Segmentation</option>
|
||
<option value="stage3_extraction">Stage 3: Extraction</option>
|
||
<option value="stage4_classification">Stage 4: Classification</option>
|
||
<option value="stage5_synthesis">Stage 5: Synthesis</option>
|
||
<option value="stage6_embed_and_index">Stage 6: Embed & Index</option>
|
||
</select>
|
||
</div>
|
||
<div className="modal-field">
|
||
<label htmlFor="rerun-prompt-override">
|
||
Prompt Override <span className="modal-optional">(optional)</span>
|
||
</label>
|
||
<textarea
|
||
id="rerun-prompt-override"
|
||
value={rerunPromptOverride}
|
||
onChange={(e) => setRerunPromptOverride(e.target.value)}
|
||
placeholder="Paste a modified prompt here to test it for this run only. Leave empty to use the current on-disk prompt."
|
||
rows={6}
|
||
/>
|
||
</div>
|
||
<div className="modal-actions">
|
||
<button
|
||
className="btn btn--primary"
|
||
onClick={() => void handleRerunStage(rerunModalVideo)}
|
||
disabled={actionLoading === rerunModalVideo}
|
||
>
|
||
{actionLoading === rerunModalVideo ? "Dispatching..." : "Run Stage"}
|
||
</button>
|
||
<button
|
||
className="btn"
|
||
onClick={() => { setRerunModalVideo(null); setRerunPromptOverride(""); }}
|
||
>
|
||
Cancel
|
||
</button>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
)}
|
||
</div>
|
||
);
|
||
}
|