| 1 | export type Role = "user" | "assistant" | "tool" | "system"; |
| 2 | |
| 3 | /** "trace" rows are intermediate agent breadcrumbs (tool-call hints, |
| 4 | * progress pings) that should not be rendered as conversational replies. */ |
| 5 | export type MessageKind = "message" | "trace"; |
| 6 | |
| 7 | /** One image attached to a UIMessage. |
| 8 | * |
| 9 | * ``url`` can arrive in three different shapes, which the bubble renders |
| 10 | * identically: |
| 11 | * - A ``data:image/...;base64,...`` URL generated by the Composer for the |
| 12 | * optimistic preview of an in-flight user turn. Self-contained, no |
| 13 | * lifecycle. |
| 14 | * - A signed ``/api/media/...`` URL attached to a historical user turn by |
| 15 | * the backend on session replay. Safe to drop into an ``<img src>``. |
| 16 | * - Absent. The backend couldn't resolve a stored path (file moved, |
| 17 | * deleted, or pre-media-persistence session). The bubble shows a |
| 18 | * placeholder tile with ``name`` as the label. |
| 19 | */ |
| 20 | export interface UIImage { |
| 21 | url?: string; |
| 22 | name?: string; |
| 23 | } |
| 24 | |
| 25 | export interface UIVideo { |
| 26 | url: string; |
| 27 | name?: string; |
| 28 | } |
| 29 | |
| 30 | export interface UIQuestionCard { |
| 31 | id: string; |
| 32 | question: string; |
| 33 | options: { label: string }[]; |
| 34 | allowCustom?: boolean; |
| 35 | answered?: string | null; |
| 36 | } |
| 37 | |
| 38 | export interface UIMessage { |
| 39 | id: string; |
| 40 | role: Role; |
| 41 | content: string; |
| 42 | kind?: MessageKind; |
| 43 | isStreaming?: boolean; |
| 44 | createdAt: number; |
| 45 | /** For trace rows: each individual hint line, so consecutive hints can |
| 46 | * render as a single collapsible group. */ |
| 47 | traces?: string[]; |
| 48 | /** User turn: optimistic blob URLs for preview. Replay: placeholder chips. */ |
| 49 | images?: UIImage[]; |
| 50 | /** Assistant turn: resolved video shot URLs rendered inline under the text. */ |
| 51 | videos?: UIVideo[]; |
| 52 | /** Structured question cards embedded in an assistant message. */ |
| 53 | questions?: UIQuestionCard[]; |
| 54 | /** Human approval gate for VLM-selected image/audio memories. */ |
| 55 | memoryReview?: MemoryReview; |
| 56 | /** Batch id for persisting question-card answers over WebSocket. */ |
| 57 | questionBatchId?: string; |
| 58 | /** Top-of-turn wait indicator (TypingDots); cleared on ``stream_end resuming:false``. */ |
| 59 | turnWaiting?: boolean; |
| 60 | } |
| 61 | |
| 62 | /** Wire shape for question cards on inbound WS / session replay. */ |
| 63 | export interface WireQuestionCard { |
| 64 | id: string; |
| 65 | question: string; |
| 66 | options: Array<string | { label: string }>; |
| 67 | allow_custom?: boolean; |
| 68 | allowCustom?: boolean; |
| 69 | answered?: string | null; |
| 70 | status?: string; |
| 71 | } |
| 72 | |
| 73 | export interface WireMediaRef { |
| 74 | url: string; |
| 75 | name?: string; |
| 76 | } |
| 77 | |
| 78 | export type MemoryReviewStatus = |
| 79 | | "awaiting_method" |
| 80 | | "selecting" |
| 81 | | "awaiting_review" |
| 82 | | "reselecting" |
| 83 | | "approved" |
| 84 | | "error"; |
| 85 | |
| 86 | export interface MemorySelection { |
| 87 | memory_id: string; |
| 88 | /** Human-readable label for UI; falls back to memory_id when absent. */ |
| 89 | display_name?: string; |
| 90 | kind: "character" | "previous_shot"; |
| 91 | candidate_index: number; |
| 92 | frame_index: number; |
| 93 | timestamp_sec: number; |
| 94 | confidence: number; |
| 95 | visual_status: string; |
| 96 | reasoning: string; |
| 97 | source_shot_id: number; |
| 98 | audio_source_shot_id?: number | null; |
| 99 | image: WireMediaRef; |
| 100 | audio?: WireMediaRef | null; |
| 101 | } |
| 102 | |
| 103 | /** Prefer display_name for UI labels; keep memory_id as stable fallback. */ |
| 104 | export function memoryDisplayName( |
| 105 | memory: { display_name?: string | null; memory_id?: string; id?: string }, |
| 106 | ): string { |
| 107 | const label = memory.display_name?.trim(); |
| 108 | if (label) return label; |
| 109 | return (memory.memory_id || memory.id || "").trim(); |
| 110 | } |
| 111 | |
| 112 | export interface MemoryReviewAttempt { |
| 113 | attempt: number; |
| 114 | rejected_candidate_indices: number[]; |
| 115 | selections: MemorySelection[]; |
| 116 | updated_at: string; |
| 117 | } |
| 118 | |
| 119 | export interface MemoryReview { |
| 120 | review_id: string; |
| 121 | shot_id: number; |
| 122 | status: MemoryReviewStatus; |
| 123 | attempt: number; |
| 124 | candidate_count: number; |
| 125 | rejected_candidate_indices: number[]; |
| 126 | selections: MemorySelection[]; |
| 127 | selection_mode?: "manual" | "vlm" | null; |
| 128 | required_memory_ids?: string[]; |
| 129 | manual_selected_ids?: string[]; |
| 130 | /** References the user chose to keep; an empty list is valid. */ |
| 131 | retained_memory_ids?: string[]; |
| 132 | /** Generated shot video used for VLM and manual frame selection. */ |
| 133 | source_video?: WireMediaRef | null; |
| 134 | history: MemoryReviewAttempt[]; |
| 135 | error?: string | null; |
| 136 | updated_at: string; |
| 137 | } |
| 138 | |
| 139 | export interface GenerationMemory { |
| 140 | id: string; |
| 141 | /** Stable Memory Workspace asset id used to restore a shot's slot draft. */ |
| 142 | workspace_asset_id?: string; |
| 143 | /** Human-readable label for UI; falls back to id when absent. */ |
| 144 | display_name?: string; |
| 145 | image: WireMediaRef; |
| 146 | audio?: WireMediaRef | null; |
| 147 | metadata: { |
| 148 | source?: string; |
| 149 | visual_status?: string; |
| 150 | source_shot_id?: number | null; |
| 151 | frame_index?: number | null; |
| 152 | timestamp_sec?: number | null; |
| 153 | confidence?: number | null; |
| 154 | audio_source_shot_id?: number | null; |
| 155 | audio_workspace_asset_id?: string; |
| 156 | reference_type?: MemoryReferenceType; |
| 157 | reference_label?: string; |
| 158 | identity_ids?: string[]; |
| 159 | profile_text?: string; |
| 160 | }; |
| 161 | } |
| 162 | |
| 163 | export type MemoryWorkspaceSource = "automatic" | "local"; |
| 164 | export type MemoryReferenceType = |
| 165 | | "character" |
| 166 | | "scene" |
| 167 | | "style" |
| 168 | | "object" |
| 169 | | "other"; |
| 170 | |
| 171 | /** One reusable item in the per-project local Memory Workspace. */ |
| 172 | export interface MemoryWorkspaceAsset { |
| 173 | asset_id: string; |
| 174 | display_name: string; |
| 175 | source: MemoryWorkspaceSource; |
| 176 | kind: "character" | "previous_shot" | "manual"; |
| 177 | memory_id?: string; |
| 178 | source_shot_id?: number | null; |
| 179 | frame_index?: number | null; |
| 180 | image?: WireMediaRef; |
| 181 | audio?: WireMediaRef | null; |
| 182 | media_type?: "image" | "audio" | "image_audio"; |
| 183 | profile_text?: string; |
| 184 | profile_status?: "ready" | "missing" | "profiling" | "error" | string; |
| 185 | profile_source?: "vlm" | "human" | "none" | string; |
| 186 | identity_ids?: string[]; |
| 187 | reference_type?: MemoryReferenceType; |
| 188 | reference_label?: string; |
| 189 | provenance?: { |
| 190 | source: "generated_shot" | "local_upload" | string; |
| 191 | shot_id?: number | null; |
| 192 | timestamp_sec?: number | null; |
| 193 | }; |
| 194 | } |
| 195 | |
| 196 | /** Compact reference persisted when applying an ordered slot draft to a shot. */ |
| 197 | export interface MemorySlotReference { |
| 198 | source?: MemoryWorkspaceSource; |
| 199 | asset_id?: string; |
| 200 | image_asset_id?: string; |
| 201 | audio_asset_id?: string; |
| 202 | reason?: string; |
| 203 | } |
| 204 | |
| 205 | export interface MemoryAssetUpload { |
| 206 | asset_id?: string; |
| 207 | display_name?: string; |
| 208 | image?: OutboundMedia; |
| 209 | audio?: OutboundMedia; |
| 210 | remove_audio?: boolean; |
| 211 | profile_text?: string; |
| 212 | identity_ids?: string[]; |
| 213 | reference_type?: MemoryReferenceType | null; |
| 214 | reference_label?: string; |
| 215 | } |
| 216 | |
| 217 | export interface ShotMemoryAssetCreate { |
| 218 | timestamp_sec: number; |
| 219 | reference_type: MemoryReferenceType; |
| 220 | reference_label?: string; |
| 221 | profile_text?: string; |
| 222 | include_audio?: boolean; |
| 223 | audio_start_sec?: number; |
| 224 | audio_end_sec?: number; |
| 225 | } |
| 226 | |
| 227 | export interface StoryProfileBeat { |
| 228 | shot_id: number; |
| 229 | summary: string; |
| 230 | } |
| 231 | |
| 232 | export interface StoryProfile { |
| 233 | summary?: string; |
| 234 | beats?: StoryProfileBeat[]; |
| 235 | characters?: unknown; |
| 236 | genre?: string; |
| 237 | setting?: string; |
| 238 | title?: string; |
| 239 | tone?: string; |
| 240 | language?: string; |
| 241 | dialogue_language?: string; |
| 242 | anchors?: unknown; |
| 243 | shot_to_content?: Record<string, string>; |
| 244 | content_to_shots?: Record<string, string[]>; |
| 245 | [key: string]: unknown; |
| 246 | } |
| 247 | |
| 248 | export interface WorkplaceShot { |
| 249 | shot_id: number; |
| 250 | shot_key: string; |
| 251 | status: string; |
| 252 | summary: string; |
| 253 | caption?: string; |
| 254 | num_frames?: number | null; |
| 255 | cut: boolean; |
| 256 | video?: WireMediaRef | null; |
| 257 | has_video: boolean; |
| 258 | has_actions: boolean; |
| 259 | accepted: boolean; |
| 260 | last_review?: string | null; |
| 261 | review_notes?: string; |
| 262 | generation_error?: string; |
| 263 | /** R2V remote version id when generation has been submitted. */ |
| 264 | version_id?: string; |
| 265 | /** Raw Echo/R2V job status mirrored from shot.echo.status. */ |
| 266 | echo_status?: string; |
| 267 | updated_at?: string | null; |
| 268 | planned_reference_shot_ids?: number[]; |
| 269 | reference_shot_ids?: number[]; |
| 270 | reference_selection_note?: string; |
| 271 | references_planned?: boolean; |
| 272 | memory_review?: Omit<MemoryReview, "shot_id"> | null; |
| 273 | /** Exact Memory slots submitted when this shot generation was queued. */ |
| 274 | generation_memories?: GenerationMemory[]; |
| 275 | /** Ordered Memory Workspace slots currently applied to this shot. */ |
| 276 | memory_slots?: GenerationMemory[]; |
| 277 | memory_slots_configured?: boolean; |
| 278 | /** Stable references already approved in Build Memory. */ |
| 279 | approved_memory_slot_refs?: MemorySlotReference[]; |
| 280 | /** Agent/profile recommendation; never sent to R2V until explicitly applied. */ |
| 281 | recommended_memory_slots?: GenerationMemory[]; |
| 282 | recommended_memory_slot_refs?: MemorySlotReference[]; |
| 283 | memory_recommendation_source?: "agent" | "profile_fallback" | string | null; |
| 284 | /** 是否开启首尾衔接（使用前一 Shot 尾帧作为 I2V 条件图） */ |
| 285 | continuous_enabled?: boolean; |
| 286 | /** 本镜头视频尾帧的持久化 URL（由下一次 continuous-generate 提取并回写） */ |
| 287 | tail_frame_url?: string; |
| 288 | timeline: { |
| 289 | start_seconds: number; |
| 290 | end_seconds: number; |
| 291 | duration_seconds: number; |
| 292 | label: string; |
| 293 | }; |
| 294 | } |
| 295 | |
| 296 | /** User-uploaded first-frame reference image for shot 1. */ |
| 297 | export interface WorkplaceReferenceImage { |
| 298 | url: string; |
| 299 | name?: string; |
| 300 | width?: number; |
| 301 | height?: number; |
| 302 | } |
| 303 | |
| 304 | export interface WorkplaceData { |
| 305 | session_key: string; |
| 306 | work_id: string | null; |
| 307 | story_md: string; |
| 308 | story_empty: boolean; |
| 309 | story_profile?: StoryProfile; |
| 310 | story_editable?: boolean; |
| 311 | beats_editable?: boolean; |
| 312 | story_confirmed?: boolean; |
| 313 | shot_prompts_ready?: boolean; |
| 314 | shot_prompts_progress?: { ready: number; total: number } | null; |
| 315 | references_ready?: boolean; |
| 316 | /** ISO timestamp when stage entered shot_generating via start-generation. */ |
| 317 | shot_generating_started_at?: string | null; |
| 318 | stage: string | null; |
| 319 | goal: Record<string, unknown>; |
| 320 | final_output_path?: string | null; |
| 321 | final_output_url?: string | null; |
| 322 | final_video?: WireMediaRef | null; |
| 323 | /** Durable character identities plus the latest previous-shot continuity slot. */ |
| 324 | memory_bank?: MemorySelection[]; |
| 325 | /** Automatic and locally uploaded assets available for slot assembly. */ |
| 326 | memory_workspace_assets?: MemoryWorkspaceAsset[]; |
| 327 | /** Human-readable generation failure when the workflow stage is ``failed``. */ |
| 328 | generation_error?: string | null; |
| 329 | /** |
| 330 | * Shot1 first-frame reference image persisted by the backend. |
| 331 | * Composer PUT/DELETE; shot 1 I2V reads this from state. |
| 332 | */ |
| 333 | reference_image?: WorkplaceReferenceImage | null; |
| 334 | /** True after shot count is confirmed; PUT/DELETE then 409. */ |
| 335 | reference_image_locked?: boolean; |
| 336 | /** Generate remaining shots automatically after story confirmation. */ |
| 337 | auto_generate?: boolean; |
| 338 | /** Workflow bar: ``01``–``04`` or ``done``. */ |
| 339 | progress?: "01" | "02" | "03" | "04" | "done" | string | null; |
| 340 | /** Echo tracking: remote request id; LikeButton hidden when absent. */ |
| 341 | echo_request_id?: string | null; |
| 342 | /** Echo tracking: 0 = none, 1 = like, 2 = dislike. */ |
| 343 | like_status?: number; |
| 344 | prompt_downloaded?: boolean; |
| 345 | video_downloaded?: boolean; |
| 346 | shots: WorkplaceShot[]; |
| 347 | updated_at: string | null; |
| 348 | } |
| 349 | |
| 350 | export type EchoTrackingResponse = { |
| 351 | ok: boolean; |
| 352 | session_key: string; |
| 353 | echo_request_id: string | null; |
| 354 | like_status: number; |
| 355 | prompt_downloaded: boolean; |
| 356 | video_downloaded: boolean; |
| 357 | workplace: WorkplaceData; |
| 358 | }; |
| 359 | |
| 360 | export interface ChatSummary { |
| 361 | /** Server-side session key, e.g. ``websocket:abcd-...``. */ |
| 362 | key: string; |
| 363 | /** Local channel + chat_id parts derived from ``key`` for convenience. */ |
| 364 | channel: string; |
| 365 | chatId: string; |
| 366 | createdAt: string | null; |
| 367 | updatedAt: string | null; |
| 368 | preview: string; |
| 369 | /** Director workflow source. */ |
| 370 | source?: "stepwise" | null; |
| 371 | /** Stepwise auto-generate flag persisted on the session. */ |
| 372 | autoGenerate?: boolean | null; |
| 373 | } |
| 374 | |
| 375 | export interface BootstrapResponse { |
| 376 | token: string; |
| 377 | ws_path: string; |
| 378 | expires_in: number; |
| 379 | model_name?: string | null; |
| 380 | /** Local browser identity used to namespace on-disk sessions. */ |
| 381 | user_id?: string | null; |
| 382 | /** Currently active Prompt Engineering set name (server-global). */ |
| 383 | active_pe?: string | null; |
| 384 | } |
| 385 | |
| 386 | export type ConnectionStatus = |
| 387 | | "idle" |
| 388 | | "connecting" |
| 389 | | "open" |
| 390 | | "reconnecting" |
| 391 | | "closed" |
| 392 | | "error"; |
| 393 | |
| 394 | export type InboundEvent = |
| 395 | | { event: "ready"; chat_id: string; client_id: string } |
| 396 | | { event: "attached"; chat_id: string; active_pe?: string } |
| 397 | | { |
| 398 | event: "workplace_updated"; |
| 399 | chat_id: string; |
| 400 | work_id?: string; |
| 401 | workplace?: WorkplaceData; |
| 402 | } |
| 403 | | { |
| 404 | event: "workplace_action_ok"; |
| 405 | chat_id: string; |
| 406 | request_id: string; |
| 407 | work_id: string; |
| 408 | workplace: WorkplaceData; |
| 409 | } |
| 410 | | { |
| 411 | event: "workplace_action_error"; |
| 412 | chat_id: string; |
| 413 | request_id: string; |
| 414 | detail: string; |
| 415 | } |
| 416 | | { |
| 417 | event: "message"; |
| 418 | chat_id: string; |
| 419 | text: string; |
| 420 | reply_to?: string; |
| 421 | media?: string[]; |
| 422 | media_urls?: WireMediaRef[]; |
| 423 | questions?: WireQuestionCard[]; |
| 424 | question_batch_id?: string; |
| 425 | /** Present when the frame is an agent breadcrumb (e.g. tool hint, |
| 426 | * generic progress line) rather than a conversational reply. */ |
| 427 | kind?: "tool_hint" | "progress"; |
| 428 | } |
| 429 | | { |
| 430 | event: "question_answer_ok"; |
| 431 | chat_id: string; |
| 432 | question_batch_id: string; |
| 433 | card_id: string; |
| 434 | value: string; |
| 435 | } |
| 436 | | { |
| 437 | event: "delta"; |
| 438 | chat_id: string; |
| 439 | text: string; |
| 440 | stream_id?: string; |
| 441 | } |
| 442 | | { |
| 443 | event: "stream_end"; |
| 444 | chat_id: string; |
| 445 | stream_id?: string; |
| 446 | /** ``false`` means the agent turn is finished; question cards may be answered. */ |
| 447 | resuming?: boolean; |
| 448 | } |
| 449 | | { event: "pe_updated"; chat_id: string; active: string } |
| 450 | | { event: "error"; chat_id?: string; detail?: string }; |
| 451 | |
| 452 | /** Base64-encoded image attached to an outbound ``message`` envelope. |
| 453 | * |
| 454 | * ``data_url`` must be a ``data:image/<png|jpeg|webp|gif>;base64,...`` string |
| 455 | * — the server whitelists those MIME types and rejects everything else |
| 456 | * (including SVG, to avoid an XSS surface). ``name`` is advisory: it's |
| 457 | * preserved for the file on disk and surfaced as the placeholder label when |
| 458 | * the session is replayed. |
| 459 | */ |
| 460 | export interface OutboundMedia { |
| 461 | data_url: string; |
| 462 | name?: string; |
| 463 | } |
| 464 | |
| 465 | export type Outbound = |
| 466 | | { type: "new_chat"; autoGenerate?: boolean } |
| 467 | | { type: "attach"; chat_id: string } |
| 468 | | { |
| 469 | type: "message"; |
| 470 | chat_id: string; |
| 471 | content: string; |
| 472 | media?: OutboundMedia[]; |
| 473 | temperature?: number; |
| 474 | top_p?: number; |
| 475 | top_k?: number; |
| 476 | autoGenerate?: boolean; |
| 477 | duration_sec?: number; |
| 478 | reference_image_url?: string; |
| 479 | reference_image_name?: string; |
| 480 | reference_image_width?: number; |
| 481 | reference_image_height?: number; |
| 482 | } |
| 483 | | { |
| 484 | type: "workplace_save_story"; |
| 485 | chat_id: string; |
| 486 | request_id: string; |
| 487 | story_md: string; |
| 488 | } |
| 489 | | { |
| 490 | type: "workplace_save_story_profile"; |
| 491 | chat_id: string; |
| 492 | request_id: string; |
| 493 | story_profile: StoryProfile; |
| 494 | } |
| 495 | | { |
| 496 | type: "workplace_save_reference_image"; |
| 497 | chat_id: string; |
| 498 | request_id: string; |
| 499 | image: WorkplaceReferenceImage; |
| 500 | } |
| 501 | | { |
| 502 | type: "workplace_save_memory_asset"; |
| 503 | chat_id: string; |
| 504 | request_id: string; |
| 505 | asset: MemoryAssetUpload; |
| 506 | } |
| 507 | | { |
| 508 | type: "workplace_create_shot_memory_asset"; |
| 509 | chat_id: string; |
| 510 | request_id: string; |
| 511 | shot_id: number; |
| 512 | asset: ShotMemoryAssetCreate; |
| 513 | } |
| 514 | | { |
| 515 | type: "workplace_delete_memory_asset"; |
| 516 | chat_id: string; |
| 517 | request_id: string; |
| 518 | asset_id: string; |
| 519 | } |
| 520 | | { |
| 521 | type: "workplace_save_shot_memory_slots"; |
| 522 | chat_id: string; |
| 523 | request_id: string; |
| 524 | shot_id: number; |
| 525 | slots: MemorySlotReference[]; |
| 526 | } |
| 527 | | { |
| 528 | type: "answer_question"; |
| 529 | chat_id: string; |
| 530 | question_batch_id: string; |
| 531 | card_id: string; |
| 532 | value: string; |
| 533 | } |
| 534 | | { type: "set_pe"; name: string; chat_id: string }; |
| 535 |