| 1 | import os |
| 2 | import time |
| 3 | from pathlib import Path |
| 4 | from typing import Any |
| 5 | from moviepy.editor import VideoFileClip |
| 6 | |
| 7 | import streamlit as st |
| 8 | from loguru import logger |
| 9 | import httpx |
| 10 | from web.i18n import tr, get_language |
| 11 | from web.pipelines.base import PipelineUI, register_pipeline_ui |
| 12 | from web.pipelines.api_workflows import ( |
| 13 | is_api_workflow, |
| 14 | list_api_media_workflows, |
| 15 | list_local_media_workflows, |
| 16 | render_api_video_controls, |
| 17 | workflow_select_help, |
| 18 | workflow_source_help, |
| 19 | workflow_source_label, |
| 20 | ) |
| 21 | from web.components.content_input import render_version_info |
| 22 | from web.utils.async_helpers import run_async |
| 23 | from web.utils.history_persistence import save_web_generation_history |
| 24 | from web.utils.streamlit_helpers import check_and_warn_selfhost_workflow |
| 25 | from pixelle_video.config import config_manager |
| 26 | from pixelle_video.utils.os_util import create_task_output_dir |
| 27 | |
| 28 | class ActionTransferPipelineUI(PipelineUI): |
| 29 | """ |
| 30 | UI for the Action transfer Video Generation Pipeline. |
| 31 | Generates videos from user-provided assets (images&text&video). |
| 32 | """ |
| 33 | name = "action_transfer" |
| 34 | icon = "💃" |
| 35 | |
| 36 | @property |
| 37 | def display_name(self): |
| 38 | return tr("pipeline.action_transfer.name") |
| 39 | |
| 40 | @property |
| 41 | def description(self): |
| 42 | return tr("pipeline.action_transfer.description") |
| 43 | |
| 44 | def render(self, pixelle_video: Any): |
| 45 | # Three-column layout |
| 46 | left_col,middle_col,right_col = st.columns([1, 1, 1]) |
| 47 | |
| 48 | # ==================================================================== |
| 49 | # Left Column: Video Upload |
| 50 | # ==================================================================== |
| 51 | with left_col: |
| 52 | video_params = self.render_action_transfer_video_input(pixelle_video) |
| 53 | render_version_info() |
| 54 | |
| 55 | # ==================================================================== |
| 56 | # Middle Column: Image Upload & Prompt |
| 57 | # ==================================================================== |
| 58 | with middle_col: |
| 59 | assets_params = self.render_action_transfer_assets_input(pixelle_video) |
| 60 | |
| 61 | |
| 62 | # ==================================================================== |
| 63 | # Right Column: Output Preview |
| 64 | # ==================================================================== |
| 65 | with right_col: |
| 66 | video_params = { |
| 67 | **video_params, |
| 68 | **assets_params |
| 69 | } |
| 70 | |
| 71 | self._render_output_preview(pixelle_video, video_params) |
| 72 | |
| 73 | def render_action_transfer_video_input(self, pixelle_video) -> dict: |
| 74 | with st.container(border=True): |
| 75 | st.markdown(f"**{tr('action_transfer.video_upload')}**") |
| 76 | |
| 77 | with st.expander(tr("help.feature_description"), expanded=False): |
| 78 | st.markdown(f"**{tr('help.what')}**") |
| 79 | st.markdown(tr("action_transfer.assets.video_what")) |
| 80 | st.markdown(f"**{tr('help.how')}**") |
| 81 | st.markdown(tr("action_transfer.assets.video_how")) |
| 82 | |
| 83 | # File uploader for multiple files |
| 84 | uploaded_files = st.file_uploader( |
| 85 | tr("action_transfer.assets.video_upload"), |
| 86 | type=["mp4","mkv","mov"], |
| 87 | accept_multiple_files=True, |
| 88 | help=tr("action_transfer.assets.video_upload_help"), |
| 89 | key="action_reference_files" |
| 90 | ) |
| 91 | |
| 92 | # Save uploaded files to temp directory with unique session ID |
| 93 | video_asset_paths = [] |
| 94 | if uploaded_files: |
| 95 | import uuid |
| 96 | session_id = str(uuid.uuid4()).replace('-', '')[:12] |
| 97 | temp_dir = Path(f"temp/assets_{session_id}") |
| 98 | temp_dir.mkdir(parents=True, exist_ok=True) |
| 99 | |
| 100 | for uploaded_file in uploaded_files: |
| 101 | file_path = temp_dir / uploaded_file.name |
| 102 | with open(file_path, "wb") as f: |
| 103 | f.write(uploaded_file.getbuffer()) |
| 104 | video_asset_paths.append(str(file_path.absolute())) |
| 105 | |
| 106 | st.success(tr("action_transfer.assets.video_sucess")) |
| 107 | |
| 108 | # Preview uploaded assets |
| 109 | with st.expander(tr("action_transfer.assets.preview"), expanded=True): |
| 110 | # Show in a grid (3 columns) |
| 111 | cols = st.columns(3) |
| 112 | for i, (file, path) in enumerate(zip(uploaded_files, video_asset_paths)): |
| 113 | with cols[i % 3]: |
| 114 | # Check if image |
| 115 | ext = Path(path).suffix.lower() |
| 116 | if ext in [".mp4", ".mkv", ".mov"]: |
| 117 | st.video(file) |
| 118 | else: |
| 119 | st.info(tr("action_transfer.assets.video_empty_hint")) |
| 120 | |
| 121 | # Get the video length (rounded down). |
| 122 | if video_asset_paths: |
| 123 | clip = VideoFileClip(video_asset_paths[0]) |
| 124 | int_duration = int(clip.duration) |
| 125 | duration = min(int_duration, 30) |
| 126 | else: |
| 127 | duration = 0 |
| 128 | |
| 129 | return { |
| 130 | "video_assets": video_asset_paths, |
| 131 | "duration": duration |
| 132 | } |
| 133 | |
| 134 | def render_action_transfer_assets_input(self, pixelle_video) -> dict: |
| 135 | with st.container(border=True): |
| 136 | st.markdown(f"**{tr('action_transfer.image_upload')}**") |
| 137 | |
| 138 | with st.expander(tr("help.feature_description"), expanded=False): |
| 139 | st.markdown(f"**{tr('help.what')}**") |
| 140 | st.markdown(tr("action_transfer.assets.image_what")) |
| 141 | st.markdown(f"**{tr('help.how')}**") |
| 142 | st.markdown(tr("action_transfer.assets.image_how")) |
| 143 | |
| 144 | # File uploader for multiple files |
| 145 | uploaded_files = st.file_uploader( |
| 146 | tr("action_transfer.assets.image_upload"), |
| 147 | type=["jpg", "jpeg", "png", "webp"], |
| 148 | accept_multiple_files=True, |
| 149 | help=tr("action_transfer.assets.image_upload_help"), |
| 150 | key="image_files" |
| 151 | ) |
| 152 | |
| 153 | # Save uploaded files to temp directory with unique session ID |
| 154 | image_asset_paths = [] |
| 155 | if uploaded_files: |
| 156 | import uuid |
| 157 | session_id = str(uuid.uuid4()).replace('-', '')[:12] |
| 158 | temp_dir = Path(f"temp/assets_{session_id}") |
| 159 | temp_dir.mkdir(parents=True, exist_ok=True) |
| 160 | |
| 161 | for uploaded_file in uploaded_files: |
| 162 | file_path = temp_dir / uploaded_file.name |
| 163 | with open(file_path, "wb") as f: |
| 164 | f.write(uploaded_file.getbuffer()) |
| 165 | image_asset_paths.append(str(file_path.absolute())) |
| 166 | |
| 167 | st.success(tr("action_transfer.assets.image_sucess")) |
| 168 | |
| 169 | # Preview uploaded assets |
| 170 | with st.expander(tr("action_transfer.assets.preview"), expanded=True): |
| 171 | # Show in a grid (3 columns) |
| 172 | cols = st.columns(3) |
| 173 | for i, (file, path) in enumerate(zip(uploaded_files, image_asset_paths)): |
| 174 | with cols[i % 3]: |
| 175 | # Check if image |
| 176 | ext = Path(path).suffix.lower() |
| 177 | if ext in [".jpg", ".jpeg", ".png", ".webp"]: |
| 178 | st.image(file, caption=file.name, use_container_width=True) |
| 179 | else: |
| 180 | st.info(tr("action_transfer.assets.image_empty_hint")) |
| 181 | |
| 182 | def list_action_transfer_workflows(): |
| 183 | if workflow_source == "api": |
| 184 | return list_api_media_workflows( |
| 185 | pixelle_video, |
| 186 | "video", |
| 187 | required_adapter_abilities=["action_transfer"], |
| 188 | verified_only=True, |
| 189 | ) |
| 190 | return list_local_media_workflows( |
| 191 | pixelle_video, |
| 192 | "video", |
| 193 | workflow_source, |
| 194 | key_prefix="af_", |
| 195 | ) |
| 196 | |
| 197 | prompt_text = st.text_area( |
| 198 | tr("action_transfer.input_text"), |
| 199 | placeholder=tr("action_transfer.input.topic_placeholder"), |
| 200 | height=200, |
| 201 | help=tr("input.text_help_audio"), |
| 202 | key="prompt_box" |
| 203 | ) |
| 204 | |
| 205 | source_options = [] |
| 206 | if list_local_media_workflows(pixelle_video, "video", "runninghub", key_prefix="af_"): |
| 207 | source_options.append("runninghub") |
| 208 | if list_local_media_workflows(pixelle_video, "video", "selfhost", key_prefix="af_"): |
| 209 | source_options.append("selfhost") |
| 210 | if list_api_media_workflows( |
| 211 | pixelle_video, |
| 212 | "video", |
| 213 | required_adapter_abilities=["action_transfer"], |
| 214 | verified_only=True, |
| 215 | ): |
| 216 | source_options.append("api") |
| 217 | |
| 218 | if not source_options: |
| 219 | source_options = ["runninghub"] |
| 220 | st.warning( |
| 221 | "没有找到可用的动作迁移工作流或 API 模型。" |
| 222 | if get_language() == "zh_CN" |
| 223 | else "No available action-transfer workflow or API model was found." |
| 224 | ) |
| 225 | |
| 226 | source_key = "action_transfer_workflow_source" |
| 227 | if st.session_state.get(source_key) not in source_options: |
| 228 | st.session_state.pop(source_key, None) |
| 229 | |
| 230 | workflow_source = st.radio( |
| 231 | "生成来源" if get_language() == "zh_CN" else "Generation source", |
| 232 | source_options, |
| 233 | format_func=workflow_source_label, |
| 234 | horizontal=True, |
| 235 | key=source_key, |
| 236 | help=workflow_source_help("动作迁移" if get_language() == "zh_CN" else "action transfer"), |
| 237 | ) |
| 238 | |
| 239 | transfer_workflows = list_action_transfer_workflows() |
| 240 | if workflow_source != "api" and not transfer_workflows: |
| 241 | st.warning( |
| 242 | "当前来源下没有动作迁移工作流（需要 af_*.json）。" |
| 243 | if get_language() == "zh_CN" |
| 244 | else "No action-transfer workflow is available for this source (requires af_*.json)." |
| 245 | ) |
| 246 | if workflow_source == "api" and not transfer_workflows: |
| 247 | st.caption( |
| 248 | "当前已接入的 API 视频模型没有已验证的动作迁移数据契约，暂不展示 API 模型。" |
| 249 | if get_language() == "zh_CN" |
| 250 | else "No verified API action-transfer contract is available yet, so API video models are hidden here." |
| 251 | ) |
| 252 | workflow_options = [wf["display_name"] for wf in transfer_workflows] |
| 253 | workflow_keys = [wf["key"] for wf in transfer_workflows] |
| 254 | default_workflow_index = 0 |
| 255 | |
| 256 | workflow_display = st.selectbox( |
| 257 | tr("action_transfer.workflow_select"), |
| 258 | workflow_options if workflow_options else ["No workflow found"], |
| 259 | index=default_workflow_index, |
| 260 | label_visibility="visible", |
| 261 | key="action_transfer_workflow_select", |
| 262 | help=workflow_select_help(), |
| 263 | ) |
| 264 | |
| 265 | if workflow_options: |
| 266 | workflow_selected_index = workflow_options.index(workflow_display) |
| 267 | workflow_key = workflow_keys[workflow_selected_index] |
| 268 | workflow_info = transfer_workflows[workflow_selected_index] |
| 269 | else: |
| 270 | workflow_key = None |
| 271 | workflow_info = None |
| 272 | |
| 273 | # Check and warn for selfhost workflow (auto popup if not confirmed) |
| 274 | if workflow_key and not is_api_workflow(workflow_key): |
| 275 | check_and_warn_selfhost_workflow(workflow_key) |
| 276 | |
| 277 | api_video_params = render_api_video_controls( |
| 278 | workflow_info, |
| 279 | key_prefix="action_transfer", |
| 280 | default_duration=5, |
| 281 | ) if is_api_workflow(workflow_key) else {} |
| 282 | |
| 283 | return { |
| 284 | "image_assets": image_asset_paths, |
| 285 | "prompt_text": prompt_text, |
| 286 | "workflow_key": workflow_key, |
| 287 | "api_video_params": api_video_params, |
| 288 | } |
| 289 | |
| 290 | def _render_output_preview(self, pixelle_video: Any, video_params: dict): |
| 291 | """Render output preview section""" |
| 292 | with st.container(border=True): |
| 293 | st.markdown(f"**{tr('section.video_generation')}**") |
| 294 | |
| 295 | # Check configuration |
| 296 | if not config_manager.validate(): |
| 297 | st.warning(tr("settings.not_configured")) |
| 298 | |
| 299 | image_assets = video_params.get("image_assets", []) |
| 300 | video_assets = video_params.get("video_assets", []) |
| 301 | prompt_text = video_params.get("prompt_text", "") |
| 302 | duration = video_params.get("duration") |
| 303 | workflow_key = video_params.get("workflow_key") |
| 304 | api_video_params = video_params.get("api_video_params") or {} |
| 305 | |
| 306 | logger.info(f" - video_params: {video_params}") |
| 307 | |
| 308 | if not video_assets: |
| 309 | st.info(tr("action_transfer.assets.video_warning")) |
| 310 | st.button( |
| 311 | tr("btn.generate"), |
| 312 | type="primary", |
| 313 | use_container_width=True, |
| 314 | disabled=True, |
| 315 | key="action_transfer_generate_video_disabled" |
| 316 | ) |
| 317 | return |
| 318 | |
| 319 | if not image_assets: |
| 320 | st.info(tr("action_transfer.assets.image_warning")) |
| 321 | st.button( |
| 322 | tr("btn.generate"), |
| 323 | type="primary", |
| 324 | use_container_width=True, |
| 325 | disabled=True, |
| 326 | key="action_transfer_generate_image_disabled" |
| 327 | ) |
| 328 | return |
| 329 | |
| 330 | if not prompt_text: |
| 331 | st.info(tr("action_transfer.assets.prompt_warning")) |
| 332 | st.button( |
| 333 | tr("btn.generate"), |
| 334 | type="primary", |
| 335 | use_container_width=True, |
| 336 | disabled=True, |
| 337 | key="action_transfer_generate" |
| 338 | ) |
| 339 | return |
| 340 | |
| 341 | # Generate button |
| 342 | if st.button(tr("btn.generate"), type="primary", use_container_width=True, key="transfer_generate"): |
| 343 | if not config_manager.validate(): |
| 344 | st.error(tr("settings.not_configured")) |
| 345 | st.stop() |
| 346 | |
| 347 | progress_bar = st.progress(0) |
| 348 | status_text = st.empty() |
| 349 | |
| 350 | start_time = time.time() |
| 351 | |
| 352 | try: |
| 353 | async def generate_audio_visual_video(): |
| 354 | task_dir, task_id = create_task_output_dir() |
| 355 | logger.info(f"[Initialization] Task Directory: {task_dir}") |
| 356 | |
| 357 | import json |
| 358 | from pathlib import Path |
| 359 | |
| 360 | status_text.text(tr("progress.generation")) |
| 361 | progress_bar.progress(10) |
| 362 | image_path = image_assets[0] |
| 363 | video_path = video_assets[0] |
| 364 | second = duration |
| 365 | prompt = prompt_text |
| 366 | final_video_path = os.path.join(task_dir, "final.mp4") |
| 367 | |
| 368 | if is_api_workflow(workflow_key): |
| 369 | media_params = { |
| 370 | **api_video_params, |
| 371 | "prompt": prompt, |
| 372 | "workflow": workflow_key, |
| 373 | "media_type": "video", |
| 374 | "output_path": final_video_path, |
| 375 | "duration": second, |
| 376 | "first_clip_path": video_path, |
| 377 | "reference_image_path": image_path, |
| 378 | } |
| 379 | media_result = await pixelle_video.media( |
| 380 | **media_params, |
| 381 | ) |
| 382 | progress_bar.progress(100) |
| 383 | status_text.text(tr("status.success")) |
| 384 | await save_web_generation_history( |
| 385 | pixelle_video, |
| 386 | task_id=task_id, |
| 387 | video_path=media_result.url, |
| 388 | pipeline="action_transfer", |
| 389 | title="动作迁移" if get_language() == "zh_CN" else "Action Transfer", |
| 390 | input_params={ |
| 391 | "text": prompt, |
| 392 | "prompt_text": prompt, |
| 393 | "image_assets": image_assets, |
| 394 | "video_assets": video_assets, |
| 395 | "duration": second, |
| 396 | "workflow_key": workflow_key, |
| 397 | "api_video_params": api_video_params, |
| 398 | }, |
| 399 | ) |
| 400 | return media_result.url |
| 401 | |
| 402 | kit = await pixelle_video._get_or_create_comfykit() |
| 403 | |
| 404 | workflow_path = Path("workflows") / workflow_key |
| 405 | |
| 406 | if not workflow_path.exists(): |
| 407 | raise Exception(f"The workflow file does not exist: {workflow_path}") |
| 408 | |
| 409 | with open(workflow_path, 'r', encoding='utf-8') as f: |
| 410 | workflow_config = json.load(f) |
| 411 | |
| 412 | workflow_params = { |
| 413 | "video": video_path, |
| 414 | "image": image_path, |
| 415 | "prompt": prompt, |
| 416 | "second": second |
| 417 | } |
| 418 | |
| 419 | if workflow_config.get("source") == "runninghub" and "workflow_id" in workflow_config: |
| 420 | workflow_input = workflow_config["workflow_id"] |
| 421 | else: |
| 422 | workflow_input = str(workflow_path) |
| 423 | |
| 424 | video_result = await kit.execute(workflow_input, workflow_params) |
| 425 | |
| 426 | generated_video_url = None |
| 427 | if hasattr(video_result, 'videos') and video_result.videos: |
| 428 | generated_video_url = video_result.videos[0] |
| 429 | elif hasattr(video_result, 'outputs') and video_result.outputs: |
| 430 | for node_id, node_output in video_result.outputs.items(): |
| 431 | if isinstance(node_output, dict) and 'videos' in node_output: |
| 432 | videos = node_output['videos'] |
| 433 | if videos and len(videos) > 0: |
| 434 | generated_video_url = videos[0] |
| 435 | break |
| 436 | |
| 437 | if not generated_video_url: |
| 438 | raise Exception("The workflow did not return a video. Please check the workflow configuration.") |
| 439 | |
| 440 | timeout = httpx.Timeout(300.0) |
| 441 | async with httpx.AsyncClient(timeout=timeout) as client: |
| 442 | response = await client.get(generated_video_url) |
| 443 | response.raise_for_status() |
| 444 | with open(final_video_path, 'wb') as f: |
| 445 | f.write(response.content) |
| 446 | progress_bar.progress(100) |
| 447 | status_text.text(tr("status.success")) |
| 448 | await save_web_generation_history( |
| 449 | pixelle_video, |
| 450 | task_id=task_id, |
| 451 | video_path=final_video_path, |
| 452 | pipeline="action_transfer", |
| 453 | title="动作迁移" if get_language() == "zh_CN" else "Action Transfer", |
| 454 | input_params={ |
| 455 | "text": prompt, |
| 456 | "prompt_text": prompt, |
| 457 | "image_assets": image_assets, |
| 458 | "video_assets": video_assets, |
| 459 | "duration": second, |
| 460 | "workflow_key": workflow_key, |
| 461 | }, |
| 462 | ) |
| 463 | return final_video_path |
| 464 | |
| 465 | # Execute async generation |
| 466 | final_video_path = run_async(generate_audio_visual_video()) |
| 467 | |
| 468 | total_time = time.time() - start_time |
| 469 | progress_bar.progress(100) |
| 470 | status_text.text(tr("status.success")) |
| 471 | |
| 472 | # Display result |
| 473 | st.success(tr("status.video_generated", path=final_video_path)) |
| 474 | |
| 475 | st.markdown("---") |
| 476 | |
| 477 | # Video info |
| 478 | if os.path.exists(final_video_path): |
| 479 | file_size_mb = os.path.getsize(final_video_path) / (1024 * 1024) |
| 480 | info_text = ( |
| 481 | f"⏱️ {tr('info.generation_time')} {total_time:.1f}s " |
| 482 | f"📦 {file_size_mb:.2f}MB" |
| 483 | ) |
| 484 | st.caption(info_text) |
| 485 | |
| 486 | st.markdown("---") |
| 487 | |
| 488 | # Video preview |
| 489 | st.video(final_video_path) |
| 490 | |
| 491 | # Download button |
| 492 | with open(final_video_path, "rb") as video_file: |
| 493 | video_bytes = video_file.read() |
| 494 | video_filename = os.path.basename(final_video_path) |
| 495 | st.download_button( |
| 496 | label="⬇️ 下载视频" if get_language() == "zh_CN" else "⬇️ Download Video", |
| 497 | data=video_bytes, |
| 498 | file_name=video_filename, |
| 499 | mime="video/mp4", |
| 500 | use_container_width=True |
| 501 | ) |
| 502 | else: |
| 503 | st.error(tr("status.video_not_found", path=final_video_path)) |
| 504 | |
| 505 | except Exception as e: |
| 506 | logger.exception(e) |
| 507 | status_text.text("") |
| 508 | progress_bar.empty() |
| 509 | st.error(tr("status.error", error=str(e))) |
| 510 | st.stop() |
| 511 | |
| 512 | register_pipeline_ui(ActionTransferPipelineUI) |
| 513 |