返回 Pixelle-Video
custom.py
1 # Copyright (C) 2025 AIDC-AI
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 # http://www.apache.org/licenses/LICENSE-2.0
7 # Unless required by applicable law or agreed to in writing, software
8 # distributed under the License is distributed on an "AS IS" BASIS,
9 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 # See the License for the specific language governing permissions and
11 # limitations under the License.
12
13 """
14 Custom Video Generation Pipeline
15
16 Template pipeline for creating your own custom video generation workflows.
17 This serves as a reference implementation showing how to extend BasePipeline.
18
19 For real projects, copy this file and modify it according to your needs.
20 """
21
22 from datetime import datetime
23 from pathlib import Path
24 from typing import Optional, Callable
25
26 from loguru import logger
27
28 from pixelle_video.pipelines.base import BasePipeline
29 from pixelle_video.models.progress import ProgressEvent
30 from pixelle_video.models.storyboard import (
31 Storyboard,
32 StoryboardFrame,
33 StoryboardConfig,
34 ContentMetadata,
35 VideoGenerationResult
36 )
37
38
39 class CustomPipeline(BasePipeline):
40 """
41 Custom video generation pipeline template
42
43 This is a template showing how to create your own pipeline with custom logic.
44 You can customize:
45 - Content processing logic
46 - Narration generation strategy
47 - Image prompt generation (conditional based on template)
48 - Frame composition
49 - Video assembly
50
51 KEY OPTIMIZATION: Conditional Image Generation
52 -----------------------------------------------
53 This pipeline supports automatic detection of template image requirements.
54 If your template doesn't use {{image}}, the entire image generation pipeline
55 can be skipped, providing:
56 ⚡ Faster generation (no image API calls)
57 💰 Lower cost (no LLM calls for image prompts)
58 🚀 Reduced dependencies (no ComfyUI needed for text-only videos)
59
60 Usage patterns:
61 1. Text-only videos: Use templates/1080x1920/simple.html
62 2. AI-generated images: Use templates with {{image}} placeholder
63 3. Custom logic: Modify template or override the detection logic in your subclass
64
65 Example usage:
66 # 1. Create your own pipeline by copying this file
67 # 2. Modify the __call__ method with your custom logic
68 # 3. Register it in service.py or dynamically
69
70 from pixelle_video.pipelines.custom import CustomPipeline
71 pixelle_video.pipelines["my_custom"] = CustomPipeline(pixelle_video)
72
73 # 4. Use it
74 result = await pixelle_video.generate_video(
75 text=your_content,
76 pipeline="my_custom",
77 # Your custom parameters here
78 )
79 """
80
81 async def __call__(
82 self,
83 text: str,
84 # === Custom Parameters ===
85 # Add your own parameters here
86 custom_param_example: str = "default_value",
87
88 # === Standard Parameters (keep these for compatibility) ===
89 tts_inference_mode: Optional[str] = None, # "local" or "comfyui"
90 voice_id: Optional[str] = None, # Deprecated, use tts_voice
91 tts_voice: Optional[str] = None, # Voice ID for local mode
92 tts_workflow: Optional[str] = None,
93 tts_speed: float = 1.2,
94 ref_audio: Optional[str] = None,
95
96 media_workflow: Optional[str] = None,
97 # Note: media_width and media_height are auto-determined from template
98
99 frame_template: Optional[str] = None,
100 video_fps: int = 30,
101 output_path: Optional[str] = None,
102
103 bgm_path: Optional[str] = None,
104 bgm_volume: float = 0.2,
105
106 progress_callback: Optional[Callable[[ProgressEvent], None]] = None,
107 ) -> VideoGenerationResult:
108 """
109 Custom video generation workflow
110
111 Customize this method to implement your own logic.
112
113 Args:
114 text: Input text (customize meaning as needed)
115 custom_param_example: Your custom parameter
116 (other standard parameters...)
117
118 Returns:
119 VideoGenerationResult
120
121 Image Generation Logic:
122 - image_*.html templates → automatically generates images
123 - video_*.html templates → automatically generates videos
124 - static_*.html templates → skips media generation (faster, cheaper)
125 - To customize: Override the template type detection logic in your subclass
126 """
127 logger.info("Starting CustomPipeline")
128 logger.info(f"Input text length: {len(text)} chars")
129 logger.info(f"Custom parameter: {custom_param_example}")
130
131 # === Handle TTS parameter compatibility ===
132 # Support both old API (voice_id) and new API (tts_inference_mode + tts_voice)
133 final_voice_id = None
134 final_tts_workflow = tts_workflow
135
136 if tts_inference_mode:
137 # New API from web UI
138 if tts_inference_mode == "local":
139 # Local Edge TTS mode - use tts_voice
140 final_voice_id = tts_voice or "zh-CN-YunjianNeural"
141 final_tts_workflow = None # Don't use workflow in local mode
142 logger.debug(f"TTS Mode: local (voice={final_voice_id})")
143 elif tts_inference_mode == "comfyui":
144 # ComfyUI workflow mode
145 final_voice_id = None # Don't use voice_id in ComfyUI mode
146 # tts_workflow already set from parameter
147 logger.debug(f"TTS Mode: comfyui (workflow={final_tts_workflow})")
148 else:
149 # Old API (backward compatibility)
150 final_voice_id = voice_id or tts_voice or "zh-CN-YunjianNeural"
151 # tts_workflow already set from parameter
152 logger.debug(f"TTS Mode: legacy (voice_id={final_voice_id}, workflow={final_tts_workflow})")
153
154 # ========== Step 0: Setup ==========
155 self._report_progress(progress_callback, "initializing", 0.05)
156
157 # Create task directory
158 from pixelle_video.utils.os_util import (
159 create_task_output_dir,
160 get_task_final_video_path
161 )
162
163 task_dir, task_id = create_task_output_dir()
164 logger.info(f"Task directory: {task_dir}")
165
166 user_specified_output = None
167 if output_path is None:
168 output_path = get_task_final_video_path(task_id)
169 else:
170 user_specified_output = output_path
171 output_path = get_task_final_video_path(task_id)
172
173 # Determine frame template
174 # Priority: explicit param > config default > hardcoded default
175 if frame_template is None:
176 template_config = self.core.config.get("template", {})
177 frame_template = template_config.get("default_template", "1080x1920/default.html")
178
179 # ========== Step 0.5: Check template requirements ==========
180 # Detect template type by filename prefix
181 from pathlib import Path
182 from pixelle_video.services.frame_html import HTMLFrameGenerator
183 from pixelle_video.utils.template_util import resolve_template_path, get_template_type
184
185 template_name = Path(frame_template).name
186 template_type = get_template_type(template_name)
187 template_requires_image = (template_type == "image")
188
189 # Read media size from template meta tags
190 template_path = resolve_template_path(frame_template)
191 generator = HTMLFrameGenerator(template_path)
192 media_width, media_height = generator.get_media_size()
193 logger.info(f"📐 Media size from template: {media_width}x{media_height}")
194
195 if template_type == "image":
196 logger.info(f"📸 Template requires image generation")
197 elif template_type == "video":
198 logger.info(f"🎬 Template requires video generation")
199 else: # static
200 logger.info(f"⚡ Static template - skipping media generation pipeline")
201 logger.info(f" 💡 Benefits: Faster generation + Lower cost + No ComfyUI dependency")
202
203 # ========== Step 1: Process content (CUSTOMIZE THIS) ==========
204 self._report_progress(progress_callback, "processing_content", 0.10)
205
206 # Example: Generate title using LLM
207 from pixelle_video.utils.content_generators import generate_title
208 title = await generate_title(self.llm, text, strategy="llm")
209 logger.info(f"Generated title: '{title}'")
210
211 # Example: Split or generate narrations
212 # Option A: Split by lines (for fixed script)
213 narrations = [line.strip() for line in text.split('\n') if line.strip()]
214
215 # Option B: Use LLM to generate narrations (uncomment to use)
216 # from pixelle_video.utils.content_generators import generate_narrations_from_topic
217 # narrations = await generate_narrations_from_topic(
218 # self.llm,
219 # topic=text,
220 # n_scenes=5,
221 # min_words=20,
222 # max_words=80
223 # )
224
225 logger.info(f"Generated {len(narrations)} narrations")
226
227 # ========== Step 2: Generate image prompts (CONDITIONAL - CUSTOMIZE THIS) ==========
228 self._report_progress(progress_callback, "generating_image_prompts", 0.25)
229
230 # IMPORTANT: Check if template is image type
231 # If your template is static_*.html, you can skip this entire step!
232 if template_requires_image:
233 # Template requires images - generate image prompts using LLM
234 from pixelle_video.utils.content_generators import generate_image_prompts
235
236 image_prompts = await generate_image_prompts(
237 self.llm,
238 narrations=narrations,
239 min_words=30,
240 max_words=60
241 )
242
243 # Example: Apply custom prompt prefix
244 from pixelle_video.utils.prompt_helper import build_image_prompt
245 custom_prefix = "cinematic style, professional lighting" # Customize this
246
247 final_image_prompts = []
248 for base_prompt in image_prompts:
249 final_prompt = build_image_prompt(base_prompt, custom_prefix)
250 final_image_prompts.append(final_prompt)
251
252 logger.info(f"✅ Generated {len(final_image_prompts)} image prompts")
253 else:
254 # Template doesn't need images - skip image generation entirely
255 final_image_prompts = [None] * len(narrations)
256 logger.info(f"⚡ Skipped image prompt generation (template doesn't need images)")
257 logger.info(f" 💡 Savings: {len(narrations)} LLM calls + {len(narrations)} image generations")
258
259 # ========== Step 3: Create storyboard ==========
260 config = StoryboardConfig(
261 task_id=task_id,
262 n_storyboard=len(narrations),
263 min_narration_words=20,
264 max_narration_words=80,
265 min_image_prompt_words=30,
266 max_image_prompt_words=60,
267 video_fps=video_fps,
268 tts_inference_mode=tts_inference_mode or "local", # TTS inference mode (CRITICAL FIX)
269 voice_id=final_voice_id, # Use processed voice_id
270 tts_workflow=final_tts_workflow, # Use processed workflow
271 tts_speed=tts_speed,
272 ref_audio=ref_audio,
273 media_width=media_width,
274 media_height=media_height,
275 media_workflow=media_workflow,
276 frame_template=frame_template
277 )
278
279 # Optional: Add custom metadata
280 content_metadata = ContentMetadata(
281 title=title,
282 subtitle="Custom Pipeline Output"
283 )
284
285 storyboard = Storyboard(
286 title=title,
287 config=config,
288 content_metadata=content_metadata,
289 created_at=datetime.now()
290 )
291
292 # Create frames
293 for i, (narration, image_prompt) in enumerate(zip(narrations, final_image_prompts)):
294 frame = StoryboardFrame(
295 index=i,
296 narration=narration,
297 image_prompt=image_prompt,
298 created_at=datetime.now()
299 )
300 storyboard.frames.append(frame)
301
302 try:
303 # ========== Step 4: Process each frame ==========
304 # This is the standard frame processing logic
305 # You can customize frame processing if needed
306
307 for i, frame in enumerate(storyboard.frames):
308 base_progress = 0.3
309 frame_range = 0.5
310 per_frame_progress = frame_range / len(storyboard.frames)
311
312 self._report_progress(
313 progress_callback,
314 "processing_frame",
315 base_progress + (per_frame_progress * i),
316 frame_current=i+1,
317 frame_total=len(storyboard.frames)
318 )
319
320 # Use core frame processor (standard logic)
321 processed_frame = await self.core.frame_processor(
322 frame=frame,
323 storyboard=storyboard,
324 config=config,
325 total_frames=len(storyboard.frames),
326 progress_callback=None
327 )
328 storyboard.total_duration += processed_frame.duration
329 logger.info(f"Frame {i+1} completed ({processed_frame.duration:.2f}s)")
330
331 # ========== Step 5: Concatenate videos ==========
332 self._report_progress(progress_callback, "concatenating", 0.85)
333 segment_paths = [frame.video_segment_path for frame in storyboard.frames]
334
335 from pixelle_video.services.video import VideoService
336 video_service = VideoService()
337
338 final_video_path = video_service.concat_videos(
339 videos=segment_paths,
340 output=output_path,
341 bgm_path=bgm_path,
342 bgm_volume=bgm_volume,
343 bgm_mode="loop"
344 )
345
346 storyboard.final_video_path = final_video_path
347 storyboard.completed_at = datetime.now()
348
349 # Copy to user-specified path if provided
350 if user_specified_output:
351 import shutil
352 Path(user_specified_output).parent.mkdir(parents=True, exist_ok=True)
353 shutil.copy2(final_video_path, user_specified_output)
354 logger.info(f"Final video copied to: {user_specified_output}")
355 final_video_path = user_specified_output
356 storyboard.final_video_path = user_specified_output
357
358 logger.success(f"Custom pipeline video completed: {final_video_path}")
359
360 # ========== Step 6: Create result ==========
361 self._report_progress(progress_callback, "completed", 1.0)
362
363 video_path_obj = Path(final_video_path)
364 file_size = video_path_obj.stat().st_size
365
366 result = VideoGenerationResult(
367 video_path=final_video_path,
368 storyboard=storyboard,
369 duration=storyboard.total_duration,
370 file_size=file_size
371 )
372
373 logger.info(f"Custom pipeline completed")
374 logger.info(f"Title: {title}")
375 logger.info(f"Duration: {storyboard.total_duration:.2f}s")
376 logger.info(f"Size: {file_size / (1024*1024):.2f} MB")
377 logger.info(f"Frames: {len(storyboard.frames)}")
378
379 # ========== Step 7: Persist metadata and storyboard ==========
380 await self._persist_task_data(
381 storyboard=storyboard,
382 result=result,
383 input_params={
384 "text": text,
385 "custom_param_example": custom_param_example,
386 "voice_id": voice_id,
387 "tts_workflow": tts_workflow,
388 "tts_speed": tts_speed,
389 "ref_audio": ref_audio,
390 "media_workflow": media_workflow,
391 "frame_template": frame_template,
392 "bgm_path": bgm_path,
393 "bgm_volume": bgm_volume,
394 }
395 )
396
397 return result
398
399 except Exception as e:
400 logger.error(f"Custom pipeline failed: {e}")
401 raise
402
403 # ==================== Persistence ====================
404
405 async def _persist_task_data(
406 self,
407 storyboard: Storyboard,
408 result: VideoGenerationResult,
409 input_params: dict
410 ):
411 """
412 Persist task metadata and storyboard to filesystem
413
414 Args:
415 storyboard: Complete storyboard
416 result: Video generation result
417 input_params: Input parameters used for generation
418 """
419 try:
420 task_id = storyboard.config.task_id
421 if not task_id:
422 logger.warning("No task_id in storyboard, skipping persistence")
423 return
424
425 # Build metadata
426 # If user didn't provide a title, use the generated one from storyboard
427 input_with_title = input_params.copy()
428 if not input_with_title.get("title"):
429 input_with_title["title"] = storyboard.title
430
431 metadata = {
432 "task_id": task_id,
433 "created_at": storyboard.created_at.isoformat() if storyboard.created_at else None,
434 "completed_at": storyboard.completed_at.isoformat() if storyboard.completed_at else None,
435 "status": "completed",
436
437 "input": input_with_title,
438
439 "result": {
440 "video_path": result.video_path,
441 "duration": result.duration,
442 "file_size": result.file_size,
443 "n_frames": len(storyboard.frames)
444 },
445
446 "config": {
447 "llm_model": self.core.config.get("llm", {}).get("model", "unknown"),
448 "llm_base_url": self.core.config.get("llm", {}).get("base_url", "unknown"),
449 "comfyui_url": self.core.config.get("comfyui", {}).get("comfyui_url", "unknown"),
450 "runninghub_enabled": bool(self.core.config.get("comfyui", {}).get("runninghub_api_key")),
451 }
452 }
453
454 # Save metadata
455 await self.core.persistence.save_task_metadata(task_id, metadata)
456 logger.info(f"💾 Saved task metadata: {task_id}")
457
458 # Save storyboard
459 await self.core.persistence.save_storyboard(task_id, storyboard)
460 logger.info(f"💾 Saved storyboard: {task_id}")
461
462 except Exception as e:
463 logger.error(f"Failed to persist task data: {e}")
464 # Don't raise - persistence failure shouldn't break video generation
465
466 # ==================== Custom Helper Methods ====================
467 # Add your own helper methods here
468
469 async def _custom_content_analysis(self, text: str) -> dict:
470 """
471 Example: Custom content analysis logic
472
473 You can add your own helper methods to process content,
474 extract metadata, or perform custom transformations.
475 """
476 # Your custom logic here
477 return {
478 "processed": text,
479 "metadata": {}
480 }
481
482 async def _custom_prompt_generation(self, context: str) -> str:
483 """
484 Example: Custom prompt generation logic
485
486 Create specialized prompts based on your use case.
487 """
488 prompt = f"Generate content based on: {context}"
489 response = await self.llm(prompt, temperature=0.7, max_tokens=500)
490 return response.strip()
491
492
493 # ==================== Usage Examples ====================
494
495 """
496 Example 1: Text-only video (no AI image generation)
497 ---------------------------------------------------
498 from pixelle_video import pixelle_video
499 from pixelle_video.pipelines.custom import CustomPipeline
500
501 # Initialize
502 await pixelle_video.initialize()
503
504 # Register custom pipeline
505 pixelle_video.pipelines["my_custom"] = CustomPipeline(pixelle_video)
506
507 # Use text-only template - no image generation!
508 result = await pixelle_video.generate_video(
509 text="Your content here",
510 pipeline="my_custom",
511 frame_template="1080x1920/simple.html" # Template without {{image}}
512 )
513 # Benefits: ⚡ Fast, 💰 Cheap, 🚀 No ComfyUI needed
514
515
516 Example 2: AI-generated image video
517 ---------------------------------------------------
518 # Use template with {{image}} - automatic image generation
519 result = await pixelle_video.generate_video(
520 text="Your content here",
521 pipeline="my_custom",
522 frame_template="1080x1920/default.html" # Template with {{image}}
523 )
524 # Will automatically generate images via LLM + ComfyUI
525
526
527 Example 3: Create your own pipeline class
528 ----------------------------------------
529 from pixelle_video.pipelines.custom import CustomPipeline
530
531 class MySpecialPipeline(CustomPipeline):
532 async def __call__(self, text: str, **kwargs):
533 # Your completely custom logic
534 logger.info("Running my special pipeline")
535
536 # You can reuse parts from CustomPipeline or start from scratch
537 # ...
538
539 return result
540
541
542 Example 4: Inline custom pipeline
543 ----------------------------------------
544 from pixelle_video.pipelines.base import BasePipeline
545
546 class QuickPipeline(BasePipeline):
547 async def __call__(self, text: str, **kwargs):
548 # Quick custom logic
549 narrations = text.split('\\n')
550
551 for narration in narrations:
552 audio = await self.tts(narration)
553 image = await self.image(prompt=f"illustration of {narration}")
554 # ... process frame
555
556 # ... concatenate and return
557 return result
558
559 # Use immediately
560 pixelle_video.pipelines["quick"] = QuickPipeline(pixelle_video)
561 result = await pixelle_video.generate_video(text=content, pipeline="quick")
562 """
563
564
564 lines PYTHON