| 1 | import os |
| 2 | import time |
| 3 | from pathlib import Path |
| 4 | from typing import Any |
| 5 | |
| 6 | import streamlit as st |
| 7 | from loguru import logger |
| 8 | import httpx |
| 9 | from web.i18n import tr, get_language |
| 10 | from web.pipelines.base import PipelineUI, register_pipeline_ui |
| 11 | from web.pipelines.api_workflows import ( |
| 12 | is_api_workflow, |
| 13 | list_api_media_workflows, |
| 14 | list_local_media_workflows, |
| 15 | render_api_video_controls, |
| 16 | workflow_select_help, |
| 17 | workflow_source_help, |
| 18 | workflow_source_label, |
| 19 | ) |
| 20 | from web.components.content_input import render_version_info |
| 21 | from web.utils.async_helpers import run_async |
| 22 | from web.utils.history_persistence import save_web_generation_history |
| 23 | from web.utils.streamlit_helpers import check_and_warn_selfhost_workflow |
| 24 | from pixelle_video.config import config_manager |
| 25 | from pixelle_video.utils.os_util import create_task_output_dir |
| 26 | |
| 27 | class ImageToVideoPipelineUI(PipelineUI): |
| 28 | """ |
| 29 | UI for the Image To Video Video Generation Pipeline. |
| 30 | Generates videos from user-provided assets (images&text). |
| 31 | """ |
| 32 | name = "image_to_video" |
| 33 | icon = "🎥" |
| 34 | |
| 35 | @property |
| 36 | def display_name(self): |
| 37 | return tr("pipeline.i2v.name") |
| 38 | |
| 39 | @property |
| 40 | def description(self): |
| 41 | return tr("pipeline.i2v.description") |
| 42 | |
| 43 | def render(self, pixelle_video: Any): |
| 44 | # Two-column layout |
| 45 | left_col,right_col = st.columns([1, 1]) |
| 46 | |
| 47 | # ==================================================================== |
| 48 | # Left Column: Asset Upload |
| 49 | # ==================================================================== |
| 50 | with left_col: |
| 51 | asset_params = self.render_audio_visual_input(pixelle_video) |
| 52 | render_version_info() |
| 53 | |
| 54 | # ==================================================================== |
| 55 | # Right Column: Output Preview |
| 56 | # ==================================================================== |
| 57 | with right_col: |
| 58 | video_params = { |
| 59 | **asset_params |
| 60 | } |
| 61 | |
| 62 | self._render_output_preview(pixelle_video, video_params) |
| 63 | |
| 64 | def render_audio_visual_input(self, pixelle_video) -> dict: |
| 65 | with st.container(border=True): |
| 66 | st.markdown(f"**{tr('i2v.video_generation')}**") |
| 67 | |
| 68 | with st.expander(tr("help.feature_description"), expanded=False): |
| 69 | st.markdown(f"**{tr('help.what')}**") |
| 70 | st.markdown(tr("i2v.assets.image_what")) |
| 71 | st.markdown(f"**{tr('help.how')}**") |
| 72 | st.markdown(tr("i2v.assets.how")) |
| 73 | |
| 74 | def list_i2v_workflows(): |
| 75 | if workflow_source == "api": |
| 76 | return list_api_media_workflows( |
| 77 | pixelle_video, |
| 78 | "video", |
| 79 | required_adapter_abilities=["first_frame_i2v"], |
| 80 | verified_only=True, |
| 81 | ) |
| 82 | return list_local_media_workflows( |
| 83 | pixelle_video, |
| 84 | "video", |
| 85 | workflow_source, |
| 86 | key_prefix="i2v_", |
| 87 | ) |
| 88 | |
| 89 | # File uploader for multiple files |
| 90 | uploaded_files = st.file_uploader( |
| 91 | tr("i2v.assets.upload"), |
| 92 | type=["jpg", "jpeg", "png", "webp"], |
| 93 | accept_multiple_files=True, |
| 94 | help=tr("i2v.assets.upload_help"), |
| 95 | key="material_files" |
| 96 | ) |
| 97 | |
| 98 | # Save uploaded files to temp directory with unique session ID |
| 99 | audio_asset_paths = [] |
| 100 | if uploaded_files: |
| 101 | import uuid |
| 102 | session_id = str(uuid.uuid4()).replace('-', '')[:12] |
| 103 | temp_dir = Path(f"temp/assets_{session_id}") |
| 104 | temp_dir.mkdir(parents=True, exist_ok=True) |
| 105 | |
| 106 | for uploaded_file in uploaded_files: |
| 107 | file_path = temp_dir / uploaded_file.name |
| 108 | with open(file_path, "wb") as f: |
| 109 | f.write(uploaded_file.getbuffer()) |
| 110 | audio_asset_paths.append(str(file_path.absolute())) |
| 111 | |
| 112 | st.success(tr("i2v.assets.character_sucess")) |
| 113 | |
| 114 | # Preview uploaded assets |
| 115 | with st.expander(tr("i2v.assets.preview"), expanded=True): |
| 116 | # Show in a grid (3 columns) |
| 117 | cols = st.columns(3) |
| 118 | for i, (file, path) in enumerate(zip(uploaded_files, audio_asset_paths)): |
| 119 | with cols[i % 3]: |
| 120 | # Check if image |
| 121 | ext = Path(path).suffix.lower() |
| 122 | if ext in [".jpg", ".jpeg", ".png", ".webp"]: |
| 123 | st.image(file, caption=file.name, use_container_width=True) |
| 124 | else: |
| 125 | st.info(tr("i2v.assets.character_empty_hint")) |
| 126 | |
| 127 | prompt_text = st.text_area( |
| 128 | tr("i2v.input_text"), |
| 129 | placeholder=tr("i2v.input.topic_placeholder"), |
| 130 | height=200, |
| 131 | help=tr("input.text_help_audio"), |
| 132 | key="audio_box" |
| 133 | ) |
| 134 | |
| 135 | source_options = [] |
| 136 | if list_local_media_workflows(pixelle_video, "video", "runninghub", key_prefix="i2v_"): |
| 137 | source_options.append("runninghub") |
| 138 | if list_local_media_workflows(pixelle_video, "video", "selfhost", key_prefix="i2v_"): |
| 139 | source_options.append("selfhost") |
| 140 | if list_api_media_workflows( |
| 141 | pixelle_video, |
| 142 | "video", |
| 143 | required_adapter_abilities=["first_frame_i2v"], |
| 144 | verified_only=True, |
| 145 | ): |
| 146 | source_options.append("api") |
| 147 | |
| 148 | if not source_options: |
| 149 | source_options = ["runninghub"] |
| 150 | st.warning( |
| 151 | "没有找到可用的图生视频工作流或 API 模型。" |
| 152 | if get_language() == "zh_CN" |
| 153 | else "No available image-to-video workflow or API model was found." |
| 154 | ) |
| 155 | |
| 156 | source_key = "i2v_workflow_source" |
| 157 | if st.session_state.get(source_key) not in source_options: |
| 158 | st.session_state.pop(source_key, None) |
| 159 | |
| 160 | workflow_source = st.radio( |
| 161 | "生成来源" if get_language() == "zh_CN" else "Generation source", |
| 162 | source_options, |
| 163 | format_func=workflow_source_label, |
| 164 | horizontal=True, |
| 165 | key=source_key, |
| 166 | help=workflow_source_help("图生视频" if get_language() == "zh_CN" else "image-to-video"), |
| 167 | ) |
| 168 | |
| 169 | i2v_workflows = list_i2v_workflows() |
| 170 | if workflow_source != "api" and not i2v_workflows: |
| 171 | st.warning( |
| 172 | "当前来源下没有图生视频工作流（需要 i2v_*.json）。" |
| 173 | if get_language() == "zh_CN" |
| 174 | else "No image-to-video workflow is available for this source (requires i2v_*.json)." |
| 175 | ) |
| 176 | workflow_options = [wf["display_name"] for wf in i2v_workflows] |
| 177 | workflow_keys = [wf["key"] for wf in i2v_workflows] |
| 178 | default_workflow_index = 0 |
| 179 | |
| 180 | workflow_display = st.selectbox( |
| 181 | tr("i2v.workflow_select"), |
| 182 | workflow_options if workflow_options else ["No workflow found"], |
| 183 | index=default_workflow_index, |
| 184 | label_visibility="visible", |
| 185 | key="i2v_workflow_select", |
| 186 | help=workflow_select_help(), |
| 187 | ) |
| 188 | |
| 189 | if workflow_options: |
| 190 | workflow_selected_index = workflow_options.index(workflow_display) |
| 191 | workflow_key = workflow_keys[workflow_selected_index] |
| 192 | workflow_info = i2v_workflows[workflow_selected_index] |
| 193 | else: |
| 194 | workflow_key = None |
| 195 | workflow_info = None |
| 196 | |
| 197 | # Check and warn for selfhost workflow (auto popup if not confirmed) |
| 198 | if workflow_key and not is_api_workflow(workflow_key): |
| 199 | check_and_warn_selfhost_workflow(workflow_key) |
| 200 | |
| 201 | api_video_params = render_api_video_controls( |
| 202 | workflow_info, |
| 203 | key_prefix="i2v", |
| 204 | default_duration=5, |
| 205 | ) if is_api_workflow(workflow_key) else {} |
| 206 | |
| 207 | return { |
| 208 | "audio_assets": audio_asset_paths, |
| 209 | "prompt_text": prompt_text, |
| 210 | "workflow_key": workflow_key, |
| 211 | "api_video_params": api_video_params, |
| 212 | } |
| 213 | |
| 214 | def _render_output_preview(self, pixelle_video: Any, video_params: dict): |
| 215 | """Render output preview section""" |
| 216 | with st.container(border=True): |
| 217 | st.markdown(f"**{tr('section.video_generation')}**") |
| 218 | |
| 219 | # Check configuration |
| 220 | if not config_manager.validate(): |
| 221 | st.warning(tr("settings.not_configured")) |
| 222 | |
| 223 | audio_assets = video_params.get("audio_assets", []) |
| 224 | prompt_text = video_params.get("prompt_text", "") |
| 225 | workflow_key = video_params.get("workflow_key") |
| 226 | api_video_params = video_params.get("api_video_params") or {} |
| 227 | |
| 228 | logger.info(f" - video_params: {video_params}") |
| 229 | |
| 230 | if not audio_assets: |
| 231 | st.info(tr("i2v.assets.image_warning")) |
| 232 | st.button( |
| 233 | tr("btn.generate"), |
| 234 | type="primary", |
| 235 | use_container_width=True, |
| 236 | disabled=True, |
| 237 | key="audio_visual_generate_disabled" |
| 238 | ) |
| 239 | return |
| 240 | |
| 241 | if not prompt_text: |
| 242 | st.info(tr("i2v.assets.prompt_warning")) |
| 243 | st.button( |
| 244 | tr("btn.generate"), |
| 245 | type="primary", |
| 246 | use_container_width=True, |
| 247 | disabled=True, |
| 248 | key="audio_visual_generate" |
| 249 | ) |
| 250 | return |
| 251 | |
| 252 | # Generate button |
| 253 | if st.button(tr("btn.generate"), type="primary", use_container_width=True, key="i2v_generate"): |
| 254 | if not config_manager.validate(): |
| 255 | st.error(tr("settings.not_configured")) |
| 256 | st.stop() |
| 257 | |
| 258 | progress_bar = st.progress(0) |
| 259 | status_text = st.empty() |
| 260 | |
| 261 | start_time = time.time() |
| 262 | |
| 263 | try: |
| 264 | async def generate_audio_visual_video(): |
| 265 | task_dir, task_id = create_task_output_dir() |
| 266 | logger.info(f"[Initialization] Task Directory: {task_dir}") |
| 267 | |
| 268 | import json |
| 269 | from pathlib import Path |
| 270 | |
| 271 | status_text.text(tr("progress.generation")) |
| 272 | progress_bar.progress(10) |
| 273 | image_path = audio_assets[0] |
| 274 | prompt = prompt_text |
| 275 | final_video_path = os.path.join(task_dir, "final.mp4") |
| 276 | |
| 277 | if is_api_workflow(workflow_key): |
| 278 | media_params = { |
| 279 | **api_video_params, |
| 280 | "prompt": prompt, |
| 281 | "workflow": workflow_key, |
| 282 | "media_type": "video", |
| 283 | "image_path": image_path, |
| 284 | "output_path": final_video_path, |
| 285 | } |
| 286 | media_result = await pixelle_video.media( |
| 287 | **media_params, |
| 288 | ) |
| 289 | progress_bar.progress(100) |
| 290 | status_text.text(tr("status.success")) |
| 291 | await save_web_generation_history( |
| 292 | pixelle_video, |
| 293 | task_id=task_id, |
| 294 | video_path=media_result.url, |
| 295 | pipeline="image_to_video", |
| 296 | title="图生视频" if get_language() == "zh_CN" else "Image to Video", |
| 297 | input_params={ |
| 298 | "text": prompt, |
| 299 | "prompt_text": prompt, |
| 300 | "image_assets": audio_assets, |
| 301 | "workflow_key": workflow_key, |
| 302 | "api_video_params": api_video_params, |
| 303 | }, |
| 304 | ) |
| 305 | return media_result.url |
| 306 | |
| 307 | kit = await pixelle_video._get_or_create_comfykit() |
| 308 | |
| 309 | workflow_path = Path("workflows") / workflow_key |
| 310 | |
| 311 | if not workflow_path.exists(): |
| 312 | raise Exception(f"The workflow file does not exist: {workflow_path}") |
| 313 | |
| 314 | with open(workflow_path, 'r', encoding='utf-8') as f: |
| 315 | workflow_config = json.load(f) |
| 316 | |
| 317 | workflow_params = { |
| 318 | "image": image_path, |
| 319 | "prompt": prompt |
| 320 | } |
| 321 | |
| 322 | if workflow_config.get("source") == "runninghub" and "workflow_id" in workflow_config: |
| 323 | workflow_input = workflow_config["workflow_id"] |
| 324 | else: |
| 325 | workflow_input = str(workflow_path) |
| 326 | |
| 327 | video_result = await kit.execute(workflow_input, workflow_params) |
| 328 | |
| 329 | generated_video_url = None |
| 330 | if hasattr(video_result, 'videos') and video_result.videos: |
| 331 | generated_video_url = video_result.videos[0] |
| 332 | elif hasattr(video_result, 'outputs') and video_result.outputs: |
| 333 | for node_id, node_output in video_result.outputs.items(): |
| 334 | if isinstance(node_output, dict) and 'videos' in node_output: |
| 335 | videos = node_output['videos'] |
| 336 | if videos and len(videos) > 0: |
| 337 | generated_video_url = videos[0] |
| 338 | break |
| 339 | |
| 340 | if not generated_video_url: |
| 341 | raise Exception("The workflow did not return a video. Please check the workflow configuration.") |
| 342 | |
| 343 | timeout = httpx.Timeout(300.0) |
| 344 | async with httpx.AsyncClient(timeout=timeout) as client: |
| 345 | response = await client.get(generated_video_url) |
| 346 | response.raise_for_status() |
| 347 | with open(final_video_path, 'wb') as f: |
| 348 | f.write(response.content) |
| 349 | progress_bar.progress(100) |
| 350 | status_text.text(tr("status.success")) |
| 351 | await save_web_generation_history( |
| 352 | pixelle_video, |
| 353 | task_id=task_id, |
| 354 | video_path=final_video_path, |
| 355 | pipeline="image_to_video", |
| 356 | title="图生视频" if get_language() == "zh_CN" else "Image to Video", |
| 357 | input_params={ |
| 358 | "text": prompt, |
| 359 | "prompt_text": prompt, |
| 360 | "image_assets": audio_assets, |
| 361 | "workflow_key": workflow_key, |
| 362 | }, |
| 363 | ) |
| 364 | return final_video_path |
| 365 | |
| 366 | # Execute async generation |
| 367 | final_video_path = run_async(generate_audio_visual_video()) |
| 368 | |
| 369 | total_time = time.time() - start_time |
| 370 | progress_bar.progress(100) |
| 371 | status_text.text(tr("status.success")) |
| 372 | |
| 373 | # Display result |
| 374 | st.success(tr("status.video_generated", path=final_video_path)) |
| 375 | |
| 376 | st.markdown("---") |
| 377 | |
| 378 | # Video info |
| 379 | if os.path.exists(final_video_path): |
| 380 | file_size_mb = os.path.getsize(final_video_path) / (1024 * 1024) |
| 381 | info_text = ( |
| 382 | f"⏱️ {tr('info.generation_time')} {total_time:.1f}s " |
| 383 | f"📦 {file_size_mb:.2f}MB" |
| 384 | ) |
| 385 | st.caption(info_text) |
| 386 | |
| 387 | st.markdown("---") |
| 388 | |
| 389 | # Video preview |
| 390 | st.video(final_video_path) |
| 391 | |
| 392 | # Download button |
| 393 | with open(final_video_path, "rb") as video_file: |
| 394 | video_bytes = video_file.read() |
| 395 | video_filename = os.path.basename(final_video_path) |
| 396 | st.download_button( |
| 397 | label="⬇️ 下载视频" if get_language() == "zh_CN" else "⬇️ Download Video", |
| 398 | data=video_bytes, |
| 399 | file_name=video_filename, |
| 400 | mime="video/mp4", |
| 401 | use_container_width=True |
| 402 | ) |
| 403 | else: |
| 404 | st.error(tr("status.video_not_found", path=final_video_path)) |
| 405 | |
| 406 | except Exception as e: |
| 407 | logger.exception(e) |
| 408 | status_text.text("") |
| 409 | progress_bar.empty() |
| 410 | st.error(tr("status.error", error=str(e))) |
| 411 | st.stop() |
| 412 | |
| 413 | register_pipeline_ui(ImageToVideoPipelineUI) |
| 414 |