| 1 | import json |
| 2 | import logging |
| 3 | import os |
| 4 | import re |
| 5 | from typing import Optional |
| 6 | |
| 7 | from models.llm_client import LLM |
| 8 | from prompts.loader import format_prompt, load_prompt |
| 9 | |
| 10 | from .api_media import generate_image_api, generate_video_api |
| 11 | from .storage import append_artifact, task_output_dir, update_task |
| 12 | from .tts import generate_edge_tts |
| 13 | from .utils import ( |
| 14 | artifact, |
| 15 | concat_videos, |
| 16 | create_static_image_clip, |
| 17 | media_duration_seconds, |
| 18 | render_static_text_image, |
| 19 | render_template_media_video, |
| 20 | render_template_text_image, |
| 21 | replace_video_audio, |
| 22 | run_blocking, |
| 23 | template_media_spec, |
| 24 | write_json, |
| 25 | write_text, |
| 26 | ) |
| 27 | |
| 28 | DEFAULT_STYLE_CONTROL = ( |
| 29 | "Minimalist black-and-white matchstick figure style illustration, clean lines, simple sketch style" |
| 30 | ) |
| 31 | |
| 32 | logger = logging.getLogger(__name__) |
| 33 | |
| 34 | |
| 35 | def required_param(params: dict, key: str) -> str: |
| 36 | value = params.get(key) |
| 37 | if not value: |
| 38 | raise ValueError(f"standard pipeline requires {key}") |
| 39 | return str(value) |
| 40 | |
| 41 | |
| 42 | def split_by_periods(text: str) -> list[str]: |
| 43 | parts = re.findall(r"[^。．.]+[。．.]?|[^。．.]+$", text.strip()) |
| 44 | return [part.strip() for part in parts if part.strip()] |
| 45 | |
| 46 | |
| 47 | def clamp_segment_count(value, default: int = 6) -> int: |
| 48 | try: |
| 49 | count = int(value) |
| 50 | except (TypeError, ValueError): |
| 51 | count = default |
| 52 | return max(1, min(20, count)) |
| 53 | |
| 54 | |
| 55 | def parse_json_object_response(text: str) -> dict: |
| 56 | response = text.strip() |
| 57 | try: |
| 58 | return json.loads(response) |
| 59 | except json.JSONDecodeError: |
| 60 | match = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", response, re.S) |
| 61 | if not match: |
| 62 | match = re.search(r"(\{.*\})", response, re.S) |
| 63 | if not match: |
| 64 | raise ValueError("Model response did not contain a JSON object.") |
| 65 | return json.loads(match.group(1)) |
| 66 | |
| 67 | |
| 68 | def parse_narrations_response(text: str, expected_count: int) -> list[str]: |
| 69 | data = parse_json_object_response(text) |
| 70 | narrations = data.get("narrations") if isinstance(data, dict) else None |
| 71 | if not isinstance(narrations, list): |
| 72 | raise ValueError("Model response missing narrations array.") |
| 73 | |
| 74 | segments = [str(item).strip() for item in narrations if str(item).strip()] |
| 75 | if len(segments) != expected_count: |
| 76 | raise ValueError(f"Expected {expected_count} narrations, got {len(segments)}.") |
| 77 | return segments |
| 78 | |
| 79 | |
| 80 | def parse_image_prompts_response(text: str, expected_count: int) -> list[str]: |
| 81 | data = parse_json_object_response(text) |
| 82 | |
| 83 | prompts = data.get("image_prompts") if isinstance(data, dict) else None |
| 84 | if not isinstance(prompts, list): |
| 85 | raise ValueError("Model response missing image_prompts array.") |
| 86 | |
| 87 | image_prompts = [str(prompt).strip() for prompt in prompts if str(prompt).strip()] |
| 88 | if len(image_prompts) != expected_count: |
| 89 | raise ValueError( |
| 90 | f"Expected {expected_count} image prompts, got {len(image_prompts)}." |
| 91 | ) |
| 92 | return image_prompts |
| 93 | |
| 94 | |
| 95 | def build_image_prompt( |
| 96 | visual_prompt: str, |
| 97 | style_control: str, |
| 98 | *, |
| 99 | subtitle: Optional[str] = None, |
| 100 | render_subtitle_in_image: bool = False, |
| 101 | ) -> str: |
| 102 | style = style_control.strip() |
| 103 | no_text_instruction = ( |
| 104 | "Do not include any text, captions, logos, watermarks, labels, typography, " |
| 105 | "or written characters in the image." |
| 106 | ) |
| 107 | text_in_pic_instruction = ( |
| 108 | "Render exactly one subtitle directly inside the image: " |
| 109 | f"\"{subtitle or ''}\". Place this subtitle in a visually appropriate area " |
| 110 | "that does not cover the main subject, using readable typography that matches " |
| 111 | "the image style, strong contrast, balanced spacing, and clean composition. " |
| 112 | "Copy the subtitle text exactly, preserving language, characters, and punctuation. " |
| 113 | "Do not add a title. Do not add logos, watermarks, labels, or any other text." |
| 114 | ) |
| 115 | text_instruction = text_in_pic_instruction if render_subtitle_in_image else no_text_instruction |
| 116 | if not style: |
| 117 | return f"{text_instruction}\n{visual_prompt}" |
| 118 | return f"{style}\n{text_instruction}\n{visual_prompt}" |
| 119 | |
| 120 | |
| 121 | async def generate_image_prompts( |
| 122 | narrations: list[str], |
| 123 | style_control: str, |
| 124 | llm: LLM, |
| 125 | llm_model: str, |
| 126 | *, |
| 127 | render_subtitle_in_image: bool = False, |
| 128 | ) -> list[str]: |
| 129 | template = load_prompt("pipelines", "standard_image_prompt_generation", "en") |
| 130 | prompt = format_prompt( |
| 131 | template, |
| 132 | narrations_count=len(narrations), |
| 133 | narrations_json=json.dumps({"narrations": narrations}, ensure_ascii=False, indent=2), |
| 134 | ) |
| 135 | response = await run_blocking(llm.query, prompt, model=llm_model) |
| 136 | visual_prompts = parse_image_prompts_response(response, len(narrations)) |
| 137 | return [ |
| 138 | build_image_prompt( |
| 139 | visual_prompt, |
| 140 | style_control, |
| 141 | subtitle=narrations[idx], |
| 142 | render_subtitle_in_image=render_subtitle_in_image, |
| 143 | ) |
| 144 | for idx, visual_prompt in enumerate(visual_prompts) |
| 145 | ] |
| 146 | |
| 147 | |
| 148 | async def generate_narrations_from_inspiration( |
| 149 | inspiration: str, |
| 150 | segment_count: int, |
| 151 | llm: LLM, |
| 152 | llm_model: str, |
| 153 | ) -> list[str]: |
| 154 | template = load_prompt("pipelines", "standard_narration_generation", "zh") |
| 155 | prompt = format_prompt( |
| 156 | template, |
| 157 | inspiration=inspiration, |
| 158 | segment_count=segment_count, |
| 159 | ) |
| 160 | response = await run_blocking(llm.query, prompt, model=llm_model) |
| 161 | return parse_narrations_response(response, segment_count) |
| 162 | |
| 163 | |
| 164 | async def run(task_id: str, params: dict) -> tuple[dict, list[dict]]: |
| 165 | output_dir = task_output_dir(task_id) |
| 166 | os.makedirs(output_dir, exist_ok=True) |
| 167 | |
| 168 | text = params.get("text") or params.get("topic") or "" |
| 169 | if not text.strip(): |
| 170 | raise ValueError("standard pipeline requires narration text") |
| 171 | |
| 172 | mode = params.get("mode") or "copy" |
| 173 | source_text = text.strip() |
| 174 | llm = None |
| 175 | llm_model = required_param(params, "llm_model") |
| 176 | if mode == "inspiration": |
| 177 | segment_count = clamp_segment_count(params.get("segment_count")) |
| 178 | update_task(task_id, progress=6, message="Writing narration from inspiration") |
| 179 | llm = LLM() |
| 180 | narrations = await generate_narrations_from_inspiration( |
| 181 | source_text, |
| 182 | segment_count, |
| 183 | llm, |
| 184 | llm_model, |
| 185 | ) |
| 186 | source_text = "\n".join(narrations) |
| 187 | else: |
| 188 | narrations = split_by_periods(source_text) |
| 189 | |
| 190 | if not narrations: |
| 191 | raise RuntimeError("No narration segments were generated.") |
| 192 | |
| 193 | title = (params.get("title") or "").strip() |
| 194 | if not title: |
| 195 | update_task(task_id, progress=8, message="Generating title") |
| 196 | if llm is None: |
| 197 | llm = LLM() |
| 198 | title = await run_blocking( |
| 199 | llm.query, |
| 200 | f"为下面的文艺短视频旁白生成一个简短中文标题，只输出标题：\n{source_text}", |
| 201 | model=llm_model, |
| 202 | ) |
| 203 | title = title.strip().splitlines()[0] |
| 204 | |
| 205 | style_control = (params.get("style_control") or params.get("negative_prompt") or DEFAULT_STYLE_CONTROL).strip() |
| 206 | video_ratio = params.get("video_ratio") or "9:16" |
| 207 | image_model = params.get("image_model") or params.get("image_workflow") |
| 208 | if not image_model: |
| 209 | raise ValueError("standard pipeline requires image_model") |
| 210 | image_resolution = params.get("image_resolution") or "1080P" |
| 211 | enable_subtitles = bool(params.get("enable_subtitles", False)) |
| 212 | subtitle_render_mode = params.get("subtitle_render_mode") or "postprocess" |
| 213 | if subtitle_render_mode not in {"postprocess", "image_model"}: |
| 214 | subtitle_render_mode = "postprocess" |
| 215 | render_subtitle_in_image = ( |
| 216 | enable_subtitles |
| 217 | and subtitle_render_mode == "image_model" |
| 218 | and not params.get("subtitle_template") |
| 219 | ) |
| 220 | subtitle_template = params.get("subtitle_template") |
| 221 | subtitle_template_fields = params.get("subtitle_template_fields") or {} |
| 222 | template_media = template_media_spec(subtitle_template, video_ratio) if subtitle_template else None |
| 223 | template_media_kind = params.get("template_media_kind") or "image" |
| 224 | if template_media_kind not in {"image", "video"}: |
| 225 | template_media_kind = "image" |
| 226 | template_video_mode = bool(subtitle_template and template_media_kind == "video") |
| 227 | if template_video_mode and template_media and not template_media.get("supports_video"): |
| 228 | raise ValueError(f"Subtitle template does not support video media: {subtitle_template}") |
| 229 | media_video_ratio = template_media["media_ratio"] if template_media else video_ratio |
| 230 | media_resolution = template_media["media_resolution"] if template_media else image_resolution |
| 231 | video_mode = params.get("video_mode") or "image_concat" |
| 232 | dynamic_video = video_mode == "dynamic_video" or bool(params.get("generate_videos", False)) |
| 233 | video_model = params.get("video_model") |
| 234 | video_resolution = params.get("video_resolution") or params.get("resolution") or "720P" |
| 235 | if (dynamic_video or template_video_mode) and not video_model: |
| 236 | raise ValueError("standard pipeline video generation requires video_model") |
| 237 | video_duration = clamp_segment_count(params.get("video_duration") or params.get("duration") or 5, default=5) |
| 238 | |
| 239 | update_task(task_id, progress=9, message="Generating image prompts") |
| 240 | if llm is None: |
| 241 | llm = LLM() |
| 242 | try: |
| 243 | image_prompts = await generate_image_prompts( |
| 244 | narrations, |
| 245 | style_control, |
| 246 | llm, |
| 247 | llm_model, |
| 248 | render_subtitle_in_image=render_subtitle_in_image, |
| 249 | ) |
| 250 | except Exception as exc: |
| 251 | logger.warning( |
| 252 | "Failed to generate structured image prompts, fallback to narration prompts: task_id=%s error=%s", |
| 253 | task_id, |
| 254 | exc, |
| 255 | ) |
| 256 | image_prompts = [ |
| 257 | build_image_prompt( |
| 258 | narration, |
| 259 | style_control, |
| 260 | subtitle=narration, |
| 261 | render_subtitle_in_image=render_subtitle_in_image, |
| 262 | ) |
| 263 | for narration in narrations |
| 264 | ] |
| 265 | |
| 266 | storyboard = { |
| 267 | "title": title, |
| 268 | "mode": mode, |
| 269 | "input_text": text, |
| 270 | "segment_count": len(narrations), |
| 271 | "video_mode": "template_video" if template_video_mode else ("dynamic_video" if dynamic_video else "image_concat"), |
| 272 | "style_control": style_control, |
| 273 | "subtitle_render_mode": subtitle_render_mode, |
| 274 | "subtitle_template": subtitle_template, |
| 275 | "subtitle_template_fields": subtitle_template_fields, |
| 276 | "template_media_kind": template_media_kind, |
| 277 | "template_media": template_media, |
| 278 | "frames": [ |
| 279 | {"index": idx + 1, "narration": narration, "image_prompt": image_prompts[idx]} |
| 280 | for idx, narration in enumerate(narrations) |
| 281 | ], |
| 282 | } |
| 283 | storyboard_path = write_json(os.path.join(output_dir, "storyboard.json"), storyboard) |
| 284 | narration_path = write_text(os.path.join(output_dir, "narration.txt"), "\n".join(narrations)) |
| 285 | |
| 286 | artifacts = [artifact(storyboard_path, "text", "storyboard"), artifact(narration_path, "text", "narration")] |
| 287 | for item in artifacts: |
| 288 | append_artifact(task_id, item) |
| 289 | |
| 290 | images = [] |
| 291 | for idx, prompt in enumerate(image_prompts, 1): |
| 292 | update_task( |
| 293 | task_id, |
| 294 | progress=10 + int(40 * idx / len(image_prompts)), |
| 295 | message=f"Generating image {idx}/{len(image_prompts)}", |
| 296 | ) |
| 297 | image_path = await run_blocking( |
| 298 | generate_image_api, |
| 299 | prompt=prompt, |
| 300 | model=image_model, |
| 301 | output_dir=output_dir, |
| 302 | task_id=task_id, |
| 303 | video_ratio=media_video_ratio, |
| 304 | resolution=media_resolution, |
| 305 | ) |
| 306 | images.append(image_path) |
| 307 | image_artifact = artifact(image_path, "image", f"image_{idx:02d}") |
| 308 | artifacts.append(image_artifact) |
| 309 | append_artifact(task_id, image_artifact) |
| 310 | storyboard["frames"][idx - 1]["image_path"] = image_path |
| 311 | write_json(storyboard_path, storyboard) |
| 312 | |
| 313 | audios = [] |
| 314 | for idx, narration in enumerate(narrations, 1): |
| 315 | update_task( |
| 316 | task_id, |
| 317 | progress=50 + int(20 * idx / len(narrations)), |
| 318 | message=f"Generating audio {idx}/{len(narrations)}", |
| 319 | ) |
| 320 | audio_path = os.path.join(output_dir, f"audio_{idx:02d}.mp3") |
| 321 | await generate_edge_tts( |
| 322 | narration, |
| 323 | output_path=audio_path, |
| 324 | voice=params.get("tts_voice", "zh-CN-YunjianNeural"), |
| 325 | speed=float(params.get("tts_speed", 1.0)), |
| 326 | ) |
| 327 | audios.append(audio_path) |
| 328 | audio_artifact = artifact(audio_path, "audio", f"audio_{idx:02d}") |
| 329 | artifacts.append(audio_artifact) |
| 330 | append_artifact(task_id, audio_artifact) |
| 331 | storyboard["frames"][idx - 1]["audio_path"] = audio_path |
| 332 | write_json(storyboard_path, storyboard) |
| 333 | |
| 334 | videos = [] |
| 335 | for idx, (image_path, audio_path) in enumerate(zip(images, audios), 1): |
| 336 | update_task( |
| 337 | task_id, |
| 338 | progress=70 + int(20 * idx / len(images)), |
| 339 | message=f"{'Rendering template video' if template_video_mode else ('Generating dynamic video' if dynamic_video else 'Creating static clip')} {idx}/{len(images)}", |
| 340 | ) |
| 341 | duration = media_duration_seconds(audio_path) or 3.0 |
| 342 | clip_image_path = image_path |
| 343 | video_path = os.path.join(output_dir, f"video_{idx:02d}.mp4") |
| 344 | if enable_subtitles or (subtitle_template and template_video_mode): |
| 345 | if subtitle_template and template_video_mode: |
| 346 | media_video_path = os.path.join(output_dir, f"template_media_{idx:02d}.mp4") |
| 347 | await run_blocking( |
| 348 | generate_video_api, |
| 349 | prompt=image_prompts[idx - 1], |
| 350 | model=video_model, |
| 351 | output_path=media_video_path, |
| 352 | image_path=image_path, |
| 353 | duration=max(video_duration, int(duration + 0.999)), |
| 354 | video_ratio=media_video_ratio, |
| 355 | video_resolution=video_resolution, |
| 356 | ) |
| 357 | storyboard["frames"][idx - 1]["template_media_video_path"] = media_video_path |
| 358 | |
| 359 | template_video_path = os.path.join(output_dir, f"video_{idx:02d}_template.mp4") |
| 360 | await run_blocking( |
| 361 | render_template_media_video, |
| 362 | media_video_path, |
| 363 | template_video_path, |
| 364 | poster_image_path=image_path, |
| 365 | subtitle=narrations[idx - 1], |
| 366 | title=title or None, |
| 367 | video_ratio=video_ratio, |
| 368 | template_id=subtitle_template, |
| 369 | template_values=subtitle_template_fields, |
| 370 | index=idx, |
| 371 | duration=duration, |
| 372 | ) |
| 373 | storyboard["frames"][idx - 1]["template_video_path"] = template_video_path |
| 374 | video_path = await run_blocking( |
| 375 | replace_video_audio, |
| 376 | template_video_path, |
| 377 | audio_path, |
| 378 | video_path, |
| 379 | ) |
| 380 | elif subtitle_template: |
| 381 | captioned_image_path = os.path.join(output_dir, f"captioned_image_{idx:02d}.jpg") |
| 382 | clip_image_path = await run_blocking( |
| 383 | render_template_text_image, |
| 384 | image_path, |
| 385 | captioned_image_path, |
| 386 | subtitle=narrations[idx - 1], |
| 387 | title=title or None, |
| 388 | video_ratio=video_ratio, |
| 389 | template_id=subtitle_template, |
| 390 | template_values=subtitle_template_fields, |
| 391 | index=idx, |
| 392 | ) |
| 393 | elif subtitle_render_mode == "postprocess": |
| 394 | captioned_image_path = os.path.join(output_dir, f"captioned_image_{idx:02d}.jpg") |
| 395 | clip_image_path = await run_blocking( |
| 396 | render_static_text_image, |
| 397 | image_path, |
| 398 | captioned_image_path, |
| 399 | subtitle=narrations[idx - 1], |
| 400 | title=title or None, |
| 401 | video_ratio=video_ratio, |
| 402 | ) |
| 403 | else: |
| 404 | storyboard["frames"][idx - 1]["captioned_image_path"] = clip_image_path |
| 405 | storyboard["frames"][idx - 1]["subtitle_rendered_in_image"] = True |
| 406 | if not template_video_mode and (subtitle_template or subtitle_render_mode == "postprocess"): |
| 407 | captioned_artifact = artifact(clip_image_path, "image", f"captioned_image_{idx:02d}") |
| 408 | artifacts.append(captioned_artifact) |
| 409 | append_artifact(task_id, captioned_artifact) |
| 410 | storyboard["frames"][idx - 1]["captioned_image_path"] = clip_image_path |
| 411 | if subtitle_template and template_video_mode: |
| 412 | pass |
| 413 | elif dynamic_video: |
| 414 | video_only_segment_path = os.path.join(output_dir, f"video_{idx:02d}_motion.mp4") |
| 415 | await run_blocking( |
| 416 | generate_video_api, |
| 417 | prompt=image_prompts[idx - 1], |
| 418 | model=video_model, |
| 419 | output_path=video_only_segment_path, |
| 420 | image_path=clip_image_path, |
| 421 | duration=max(video_duration, int(duration + 0.999)), |
| 422 | video_ratio=video_ratio, |
| 423 | video_resolution=video_resolution, |
| 424 | ) |
| 425 | motion_artifact = artifact(video_only_segment_path, "video", f"video_{idx:02d}_motion") |
| 426 | artifacts.append(motion_artifact) |
| 427 | append_artifact(task_id, motion_artifact) |
| 428 | storyboard["frames"][idx - 1]["motion_video_path"] = video_only_segment_path |
| 429 | video_path = await run_blocking( |
| 430 | replace_video_audio, |
| 431 | video_only_segment_path, |
| 432 | audio_path, |
| 433 | video_path, |
| 434 | ) |
| 435 | else: |
| 436 | await run_blocking( |
| 437 | create_static_image_clip, |
| 438 | clip_image_path, |
| 439 | audio_path, |
| 440 | video_path, |
| 441 | video_ratio=video_ratio, |
| 442 | duration=duration, |
| 443 | ) |
| 444 | videos.append(video_path) |
| 445 | video_artifact = artifact(video_path, "video", f"video_{idx:02d}") |
| 446 | artifacts.append(video_artifact) |
| 447 | append_artifact(task_id, video_artifact) |
| 448 | storyboard["frames"][idx - 1]["video_path"] = video_path |
| 449 | storyboard["frames"][idx - 1]["duration"] = duration |
| 450 | write_json(storyboard_path, storyboard) |
| 451 | |
| 452 | video_only_path = None |
| 453 | video_only_path = concat_videos(videos, os.path.join(output_dir, "final.mp4")) |
| 454 | if not video_only_path: |
| 455 | raise RuntimeError("Static short-video generation did not produce a final video.") |
| 456 | |
| 457 | final_path = video_only_path |
| 458 | |
| 459 | final_artifact = artifact(final_path, "video", "final") |
| 460 | artifacts.append(final_artifact) |
| 461 | append_artifact(task_id, final_artifact) |
| 462 | |
| 463 | write_json(storyboard_path, storyboard) |
| 464 | output = { |
| 465 | "title": title, |
| 466 | "storyboard_path": storyboard_path, |
| 467 | "narration_path": narration_path, |
| 468 | "images": images, |
| 469 | "audios": audios, |
| 470 | "videos": videos, |
| 471 | "video_mode": "template_video" if template_video_mode else ("dynamic_video" if dynamic_video else "image_concat"), |
| 472 | "video_model": video_model if (dynamic_video or template_video_mode) else None, |
| 473 | "subtitle_render_mode": subtitle_render_mode, |
| 474 | "subtitle_template": subtitle_template, |
| 475 | "subtitle_template_fields": subtitle_template_fields, |
| 476 | "template_media_kind": template_media_kind, |
| 477 | "template_media": template_media, |
| 478 | "video_only_path": video_only_path, |
| 479 | "final_video": final_path, |
| 480 | } |
| 481 | return output, artifacts |
| 482 |