| 1 | # Copyright (C) 2025 AIDC-AI |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 |
| 4 | |
| 5 | """Direct API provider media generation adapter.""" |
| 6 | |
| 7 | import asyncio |
| 8 | from copy import deepcopy |
| 9 | import os |
| 10 | from pathlib import Path |
| 11 | from typing import Any, Optional |
| 12 | |
| 13 | from loguru import logger |
| 14 | |
| 15 | from pixelle_video.config import config_manager |
| 16 | from pixelle_video.models.media import MediaResult |
| 17 | from pixelle_video.utils.os_util import get_output_path |
| 18 | |
| 19 | |
| 20 | class APIProviderMediaService: |
| 21 | """Adapter from Pixelle media calls to direct provider API clients.""" |
| 22 | |
| 23 | IMAGE_MODELS = { |
| 24 | "dashscope": [ |
| 25 | "wan2.7-image", |
| 26 | "wan2.7-image-pro", |
| 27 | "wan2.6-t2i", |
| 28 | ], |
| 29 | "openai": [ |
| 30 | "gpt-image-2", |
| 31 | ], |
| 32 | "seedream": [ |
| 33 | "doubao-seedream-5-0-260128", |
| 34 | "doubao-seedream-4-5-251128", |
| 35 | "doubao-seedream-4-0-250828", |
| 36 | ], |
| 37 | } |
| 38 | |
| 39 | VIDEO_MODELS = { |
| 40 | "dashscope": [ |
| 41 | "wan2.7-t2v", |
| 42 | "happyhorse-1.0-t2v", |
| 43 | "wan2.7-i2v", |
| 44 | "wan2.7-r2v", |
| 45 | "wan2.7-videoedit", |
| 46 | "wan2.6-i2v-flash", |
| 47 | "happyhorse-1.0-i2v", |
| 48 | "happyhorse-1.0-r2v", |
| 49 | "happyhorse-1.0-video-edit", |
| 50 | ], |
| 51 | "kling": [ |
| 52 | "kling-v3", |
| 53 | "kling-v2-6", |
| 54 | "kling-v2-5-turbo", |
| 55 | ], |
| 56 | "seedance": [ |
| 57 | "doubao-seedance-2-0-260128", |
| 58 | "doubao-seedance-2-0-fast-260128", |
| 59 | "seedance-1-0-pro", |
| 60 | "seedance-1-0-lite", |
| 61 | ], |
| 62 | } |
| 63 | |
| 64 | VIDEO_MODEL_CAPABILITIES: dict[tuple[str, str], dict[str, Any]] = { |
| 65 | ("dashscope", "wan2.7-t2v"): { |
| 66 | "ability_type": "text_to_video", |
| 67 | "ability_types": ["text_to_video", "native_audio", "multi_shot"], |
| 68 | "adapter_ability_types": ["text_to_video", "native_audio"], |
| 69 | "input_modalities": ["text"], |
| 70 | "adapter_input_modalities": ["text"], |
| 71 | "duration": {"min": 2, "max": 15, "integer": True, "verified": True}, |
| 72 | "resolutions": ["720P", "1080P"], |
| 73 | "ratios": ["16:9", "9:16", "1:1", "4:3", "3:4"], |
| 74 | "fps": 30, |
| 75 | "format": "mp4", |
| 76 | "api_contract_verified": True, |
| 77 | "source_urls": [ |
| 78 | "https://help.aliyun.com/zh/model-studio/video-generate-edit-model/", |
| 79 | "https://help.aliyun.com/zh/model-studio/text-to-video-api-reference", |
| 80 | ], |
| 81 | "contract_issues": [ |
| 82 | "Quick Create uses this model for direct text-to-video without generating a first-frame image.", |
| 83 | ], |
| 84 | }, |
| 85 | ("dashscope", "happyhorse-1.0-t2v"): { |
| 86 | "ability_type": "text_to_video", |
| 87 | "ability_types": ["text_to_video", "native_audio"], |
| 88 | "adapter_ability_types": ["text_to_video", "native_audio"], |
| 89 | "input_modalities": ["text"], |
| 90 | "adapter_input_modalities": ["text"], |
| 91 | "duration": {"min": 3, "max": 15, "integer": True, "verified": True}, |
| 92 | "resolutions": ["720P", "1080P"], |
| 93 | "fps": 24, |
| 94 | "format": "mp4", |
| 95 | "api_contract_verified": True, |
| 96 | "source_urls": [ |
| 97 | "https://help.aliyun.com/zh/model-studio/video-generate-edit-model/", |
| 98 | ], |
| 99 | "contract_issues": [ |
| 100 | "Quick Create uses this model for direct text-to-video without generating a first-frame image.", |
| 101 | ], |
| 102 | }, |
| 103 | ("dashscope", "wan2.7-i2v"): { |
| 104 | "ability_type": "image_to_video", |
| 105 | "ability_types": [ |
| 106 | "first_frame_i2v", |
| 107 | "start_end_frame_i2v", |
| 108 | "video_continuation", |
| 109 | "audio_driven_i2v", |
| 110 | "multi_shot", |
| 111 | ], |
| 112 | "adapter_ability_types": ["first_frame_i2v", "audio_driven_i2v"], |
| 113 | "input_modalities": ["text", "image", "audio", "video"], |
| 114 | "adapter_input_modalities": ["text", "image"], |
| 115 | "duration": {"min": 2, "max": 15, "integer": True, "verified": True}, |
| 116 | "resolutions": ["720P", "1080P"], |
| 117 | "ratios": ["16:9", "9:16", "1:1", "4:3", "3:4"], |
| 118 | "fps": 30, |
| 119 | "format": "mp4", |
| 120 | "api_contract_verified": True, |
| 121 | "source_urls": [ |
| 122 | "https://help.aliyun.com/zh/model-studio/video-generate-edit-model/", |
| 123 | "https://help.aliyun.com/zh/model-studio/image-to-video-general-api-reference", |
| 124 | ], |
| 125 | "contract_issues": [ |
| 126 | "Pixelle UI exposes first-frame image-to-video; asset-based workflow can optionally pass narration audio as driving_audio.", |
| 127 | "last_frame and first_clip are supported by the adapter for continuation, not character replacement.", |
| 128 | ], |
| 129 | }, |
| 130 | ("dashscope", "wan2.7-videoedit"): { |
| 131 | "ability_type": "video_editing", |
| 132 | "ability_types": ["video_editing", "action_transfer", "instruction_editing", "video_transfer"], |
| 133 | "adapter_ability_types": ["action_transfer", "video_editing"], |
| 134 | "input_modalities": ["text", "image", "video"], |
| 135 | "adapter_input_modalities": ["text", "image", "video"], |
| 136 | "duration": {"min": 2, "max": 10, "integer": True, "verified": True}, |
| 137 | "resolutions": ["720P", "1080P"], |
| 138 | "ratios": ["16:9", "9:16", "1:1", "4:3", "3:4"], |
| 139 | "fps": 30, |
| 140 | "format": "mp4", |
| 141 | "api_contract_verified": True, |
| 142 | "source_urls": [ |
| 143 | "https://help.aliyun.com/zh/model-studio/wan-video-editing-api-reference", |
| 144 | "https://help.aliyun.com/zh/model-studio/video-generate-edit-model/", |
| 145 | ], |
| 146 | "contract_issues": [ |
| 147 | "Action transfer is mapped to video plus reference_image media, following the DashScope video-edit API contract.", |
| 148 | ], |
| 149 | }, |
| 150 | ("dashscope", "wan2.7-r2v"): { |
| 151 | "ability_type": "reference_to_video", |
| 152 | "ability_types": [ |
| 153 | "reference_to_video", |
| 154 | "digital_human", |
| 155 | "multi_character", |
| 156 | "native_audio", |
| 157 | "voice_reference", |
| 158 | "multi_shot", |
| 159 | ], |
| 160 | "adapter_ability_types": ["reference_to_video", "digital_human", "voice_reference"], |
| 161 | "input_modalities": ["text", "image", "audio", "video"], |
| 162 | "adapter_input_modalities": ["text", "image", "audio"], |
| 163 | "duration": {"min": 2, "max": 10, "integer": True, "verified": True}, |
| 164 | "resolutions": ["720P", "1080P"], |
| 165 | "ratios": ["16:9", "9:16", "1:1", "4:3", "3:4"], |
| 166 | "fps": 30, |
| 167 | "format": "mp4", |
| 168 | "api_contract_verified": True, |
| 169 | "source_urls": [ |
| 170 | "https://www.alibabacloud.com/help/doc-detail/3001146.html", |
| 171 | ], |
| 172 | "contract_issues": [ |
| 173 | "Digital human uses reference_image media and optionally attaches a TTS audio file as reference_voice for the first character.", |
| 174 | ], |
| 175 | }, |
| 176 | ("dashscope", "wan2.6-i2v-flash"): { |
| 177 | "ability_type": "image_to_video", |
| 178 | "ability_types": ["first_frame_i2v", "audio_driven_i2v", "multi_shot", "fast_generation"], |
| 179 | "adapter_ability_types": ["first_frame_i2v"], |
| 180 | "input_modalities": ["text", "image", "audio"], |
| 181 | "adapter_input_modalities": ["text", "image"], |
| 182 | "duration": {"min": 2, "max": 15, "integer": True, "verified": True}, |
| 183 | "resolutions": ["720P", "1080P"], |
| 184 | "ratios": ["16:9", "9:16", "1:1", "4:3", "3:4"], |
| 185 | "fps": 30, |
| 186 | "format": "mp4", |
| 187 | "api_contract_verified": True, |
| 188 | "source_urls": [ |
| 189 | "https://help.aliyun.com/zh/model-studio/image-to-video-guide", |
| 190 | "https://help.aliyun.com/zh/model-studio/video-generate-edit-model/", |
| 191 | ], |
| 192 | "contract_issues": [ |
| 193 | "Official model supports audio input and audio sync, but the current DashScope legacy SDK call path only passes first-frame image parameters.", |
| 194 | ], |
| 195 | }, |
| 196 | ("dashscope", "happyhorse-1.0-i2v"): { |
| 197 | "ability_type": "image_to_video", |
| 198 | "ability_types": ["first_frame_i2v", "native_audio"], |
| 199 | "adapter_ability_types": ["first_frame_i2v", "native_audio"], |
| 200 | "input_modalities": ["text", "image"], |
| 201 | "adapter_input_modalities": ["text", "image"], |
| 202 | "duration": {"min": 3, "max": 15, "integer": True, "verified": True}, |
| 203 | "resolutions": ["720P", "1080P"], |
| 204 | "fps": 24, |
| 205 | "format": "mp4", |
| 206 | "api_contract_verified": True, |
| 207 | "source_urls": [ |
| 208 | "https://help.aliyun.com/zh/model-studio/video-generate-edit-model/", |
| 209 | ], |
| 210 | "contract_issues": [], |
| 211 | }, |
| 212 | ("dashscope", "happyhorse-1.0-video-edit"): { |
| 213 | "ability_type": "video_editing", |
| 214 | "ability_types": ["video_editing", "action_transfer", "instruction_editing", "native_audio"], |
| 215 | "adapter_ability_types": ["action_transfer", "video_editing"], |
| 216 | "input_modalities": ["text", "image", "video"], |
| 217 | "adapter_input_modalities": ["text", "image", "video"], |
| 218 | "duration": {"min": 3, "max": 15, "integer": True, "verified": True}, |
| 219 | "resolutions": ["720P", "1080P"], |
| 220 | "fps": 24, |
| 221 | "format": "mp4", |
| 222 | "api_contract_verified": True, |
| 223 | "source_urls": [ |
| 224 | "https://help.aliyun.com/zh/model-studio/video-generate-edit-model/", |
| 225 | "https://help.aliyun.com/zh/model-studio/wan-video-editing-api-reference", |
| 226 | ], |
| 227 | "contract_issues": [ |
| 228 | "Model capability is documented in the model list; adapter uses the same video + reference_image media contract as DashScope video-edit models.", |
| 229 | ], |
| 230 | }, |
| 231 | ("dashscope", "happyhorse-1.0-r2v"): { |
| 232 | "ability_type": "reference_to_video", |
| 233 | "ability_types": [ |
| 234 | "reference_to_video", |
| 235 | "digital_human", |
| 236 | "multi_character", |
| 237 | "native_audio", |
| 238 | "multi_shot", |
| 239 | ], |
| 240 | "adapter_ability_types": ["reference_to_video", "digital_human"], |
| 241 | "input_modalities": ["text", "image"], |
| 242 | "adapter_input_modalities": ["text", "image"], |
| 243 | "duration": {"min": 3, "max": 15, "integer": True, "verified": True}, |
| 244 | "resolutions": ["720P", "1080P"], |
| 245 | "ratios": ["16:9", "9:16", "3:4", "4:3", "1:1"], |
| 246 | "fps": 24, |
| 247 | "format": "mp4", |
| 248 | "api_contract_verified": True, |
| 249 | "source_urls": [ |
| 250 | "https://www.alibabacloud.com/help/doc-detail/3030778.html", |
| 251 | ], |
| 252 | "contract_issues": [ |
| 253 | "HappyHorse reference-to-video supports reference_image media. The public contract does not expose reference_voice in this API.", |
| 254 | ], |
| 255 | }, |
| 256 | ("kling", "kling-v3"): { |
| 257 | "ability_type": "text_to_video", |
| 258 | "ability_types": [ |
| 259 | "text_to_video", |
| 260 | "image_to_video", |
| 261 | "start_end_frame_i2v", |
| 262 | "native_audio", |
| 263 | "multi_shot", |
| 264 | "element_reference", |
| 265 | ], |
| 266 | "adapter_ability_types": ["text_to_video", "first_frame_i2v", "native_audio"], |
| 267 | "input_modalities": ["text", "image"], |
| 268 | "adapter_input_modalities": ["text", "image"], |
| 269 | "duration": {"min": 3, "max": 15, "integer": True, "verified": True}, |
| 270 | "resolutions": ["720P", "1080P"], |
| 271 | "ratios": ["16:9", "9:16", "1:1"], |
| 272 | "api_contract_verified": True, |
| 273 | "source_urls": [ |
| 274 | "https://app.klingai.com/cn/quickstart/klingai-video-3-model-user-guide", |
| 275 | "https://klingai.com/document-api/apiReference/model/textToVideo", |
| 276 | "https://klingai.com/document-api/apiReference/model/imageToVideo", |
| 277 | ], |
| 278 | "contract_issues": [ |
| 279 | "Adapter supports /v1/videos/text2video when image_path is empty and /v1/videos/image2video when image_path is provided.", |
| 280 | "Start/end frames and element references are listed as product capabilities but are not exposed by the current adapter.", |
| 281 | ], |
| 282 | }, |
| 283 | ("kling", "kling-v2-6"): { |
| 284 | "ability_type": "text_to_video", |
| 285 | "ability_types": ["text_to_video", "image_to_video", "start_end_frame_i2v", "native_audio"], |
| 286 | "adapter_ability_types": ["text_to_video", "first_frame_i2v", "native_audio"], |
| 287 | "input_modalities": ["text", "image"], |
| 288 | "adapter_input_modalities": ["text", "image"], |
| 289 | "duration": {"allowed_values": [5, 10], "integer": True, "verified": True}, |
| 290 | "resolutions": ["720P", "1080P"], |
| 291 | "ratios": ["16:9", "9:16", "1:1"], |
| 292 | "api_contract_verified": True, |
| 293 | "source_urls": [ |
| 294 | "https://app.klingai.com/cn/quickstart/klingai-video-3-model-user-guide", |
| 295 | "https://klingai.com/document-api/apiReference/model/textToVideo", |
| 296 | "https://klingai.com/document-api/apiReference/model/imageToVideo", |
| 297 | ], |
| 298 | "contract_issues": [ |
| 299 | "Adapter supports /v1/videos/text2video when image_path is empty and /v1/videos/image2video when image_path is provided.", |
| 300 | ], |
| 301 | }, |
| 302 | ("kling", "kling-v2-5-turbo"): { |
| 303 | "ability_type": "image_to_video", |
| 304 | "ability_types": ["text_to_video", "image_to_video"], |
| 305 | "adapter_ability_types": ["text_to_video", "first_frame_i2v"], |
| 306 | "input_modalities": ["text", "image"], |
| 307 | "adapter_input_modalities": ["text", "image"], |
| 308 | "duration": {"allowed_values": [5, 10], "integer": True, "verified": True}, |
| 309 | "resolutions": ["720P", "1080P"], |
| 310 | "ratios": ["16:9", "9:16", "1:1"], |
| 311 | "api_contract_verified": True, |
| 312 | "source_urls": [ |
| 313 | "https://klingai.com/document-api/apiReference/model/textToVideo", |
| 314 | "https://klingai.com/document-api/apiReference/model/imageToVideo", |
| 315 | ], |
| 316 | "contract_issues": [ |
| 317 | "Adapter supports text-to-video and first-frame image-to-video. The current adapter intentionally omits sound for this model.", |
| 318 | ], |
| 319 | }, |
| 320 | ("seedance", "doubao-seedance-2-0-260128"): { |
| 321 | "ability_type": "image_to_video", |
| 322 | "ability_types": ["text_to_video", "image_to_video"], |
| 323 | "adapter_ability_types": ["text_to_video", "first_frame_i2v", "native_audio"], |
| 324 | "input_modalities": ["text", "image"], |
| 325 | "adapter_input_modalities": ["text", "image"], |
| 326 | "duration": {"min": 2, "max": 12, "integer": True, "verified": True}, |
| 327 | "resolutions": ["720p", "1080p"], |
| 328 | "ratios": ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"], |
| 329 | "api_contract_verified": True, |
| 330 | "source_urls": [ |
| 331 | "https://www.volcengine.com/docs/82379/1520757", |
| 332 | "https://www.volcengine.com/docs/6492/2165104?lang=zh", |
| 333 | ], |
| 334 | "contract_issues": [ |
| 335 | "Current adapter supports text-to-video and first-frame image-to-video plus ratio, resolution, seed, watermark and generate_audio. Additional multi-image/video roles are not exposed.", |
| 336 | ], |
| 337 | }, |
| 338 | ("seedance", "doubao-seedance-2-0-fast-260128"): { |
| 339 | "ability_type": "image_to_video", |
| 340 | "ability_types": ["text_to_video", "image_to_video", "fast_generation"], |
| 341 | "adapter_ability_types": ["text_to_video", "first_frame_i2v", "native_audio"], |
| 342 | "input_modalities": ["text", "image"], |
| 343 | "adapter_input_modalities": ["text", "image"], |
| 344 | "duration": {"min": 2, "max": 12, "integer": True, "verified": True}, |
| 345 | "resolutions": ["720p", "1080p"], |
| 346 | "ratios": ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"], |
| 347 | "api_contract_verified": True, |
| 348 | "source_urls": [ |
| 349 | "https://www.volcengine.com/docs/82379/1520757", |
| 350 | "https://www.volcengine.com/docs/6492/2165104?lang=zh", |
| 351 | ], |
| 352 | "contract_issues": [ |
| 353 | "Current adapter supports text-to-video and first-frame image-to-video plus ratio, resolution, seed, watermark and generate_audio. Additional multi-image/video roles are not exposed.", |
| 354 | ], |
| 355 | }, |
| 356 | ("seedance", "seedance-1-0-pro"): { |
| 357 | "ability_type": "image_to_video", |
| 358 | "ability_types": [], |
| 359 | "adapter_ability_types": ["first_frame_i2v"], |
| 360 | "api_contract_verified": False, |
| 361 | "source_urls": [], |
| 362 | "contract_issues": [ |
| 363 | "Exact official model ID was not found. Volcengine docs refer to doubao-seedance model IDs, so this alias must be confirmed before relying on it.", |
| 364 | ], |
| 365 | }, |
| 366 | ("seedance", "seedance-1-0-lite"): { |
| 367 | "ability_type": "image_to_video", |
| 368 | "ability_types": [], |
| 369 | "adapter_ability_types": ["first_frame_i2v"], |
| 370 | "api_contract_verified": False, |
| 371 | "source_urls": [], |
| 372 | "contract_issues": [ |
| 373 | "Exact official model ID was not found. Volcengine docs refer to doubao-seedance model IDs, so this alias must be confirmed before relying on it.", |
| 374 | ], |
| 375 | }, |
| 376 | } |
| 377 | |
| 378 | def __init__(self, config: dict, core=None): |
| 379 | self.config = config |
| 380 | self.core = core |
| 381 | |
| 382 | def list_workflows(self) -> list[dict]: |
| 383 | """Return API models in the same shape as Comfy workflow metadata.""" |
| 384 | workflows = [] |
| 385 | |
| 386 | for provider, models in self.IMAGE_MODELS.items(): |
| 387 | for model in models: |
| 388 | workflows.append(self._workflow_info(provider, model, "image")) |
| 389 | |
| 390 | for provider, models in self.VIDEO_MODELS.items(): |
| 391 | for model in models: |
| 392 | workflows.append(self._workflow_info(provider, model, "video")) |
| 393 | |
| 394 | return workflows |
| 395 | |
| 396 | def _workflow_info(self, provider: str, model: str, media_type: str) -> dict: |
| 397 | key = f"api/{provider}/{model}" |
| 398 | info = { |
| 399 | "name": model, |
| 400 | "display_name": f"{model} - API {provider.title()}", |
| 401 | "source": "api", |
| 402 | "provider": provider, |
| 403 | "model": model, |
| 404 | "media_type": media_type, |
| 405 | "path": key, |
| 406 | "key": key, |
| 407 | } |
| 408 | if media_type == "video": |
| 409 | capabilities = self._video_capabilities(provider, model) |
| 410 | info["capabilities"] = capabilities |
| 411 | info["ability_type"] = capabilities.get("ability_type") |
| 412 | info["ability_types"] = capabilities.get("ability_types", []) |
| 413 | info["adapter_ability_types"] = capabilities.get("adapter_ability_types", []) |
| 414 | info["api_contract_verified"] = capabilities.get("api_contract_verified", False) |
| 415 | info["contract_issues"] = capabilities.get("contract_issues", []) |
| 416 | return info |
| 417 | |
| 418 | def resolve_workflow(self, workflow: str) -> dict: |
| 419 | """Resolve an api/provider/model key to model metadata.""" |
| 420 | for info in self.list_workflows(): |
| 421 | if info["key"] == workflow: |
| 422 | return info |
| 423 | available = ", ".join(info["key"] for info in self.list_workflows()) |
| 424 | raise ValueError(f"API workflow '{workflow}' not found. Available API workflows: {available}") |
| 425 | |
| 426 | async def __call__( |
| 427 | self, |
| 428 | prompt: str, |
| 429 | workflow: str, |
| 430 | media_type: str = "image", |
| 431 | width: Optional[int] = None, |
| 432 | height: Optional[int] = None, |
| 433 | duration: Optional[float] = None, |
| 434 | output_path: Optional[str] = None, |
| 435 | image_path: Optional[str] = None, |
| 436 | **params, |
| 437 | ) -> MediaResult: |
| 438 | info = self.resolve_workflow(workflow) |
| 439 | provider = info["provider"] |
| 440 | model = info["model"] |
| 441 | resolved_media_type = info.get("media_type") or media_type |
| 442 | |
| 443 | if resolved_media_type == "image": |
| 444 | image_paths = params.pop("image_paths", None) |
| 445 | return await self._generate_image( |
| 446 | provider=provider, |
| 447 | model=model, |
| 448 | prompt=prompt, |
| 449 | width=width, |
| 450 | height=height, |
| 451 | output_path=output_path, |
| 452 | image_paths=image_paths, |
| 453 | **params, |
| 454 | ) |
| 455 | |
| 456 | resolved_image_path = image_path or params.pop("image_path", None) |
| 457 | return await self._generate_video( |
| 458 | provider=provider, |
| 459 | model=model, |
| 460 | prompt=prompt, |
| 461 | image_path=resolved_image_path, |
| 462 | output_path=output_path, |
| 463 | duration=duration, |
| 464 | width=width, |
| 465 | height=height, |
| 466 | **params, |
| 467 | ) |
| 468 | |
| 469 | async def _generate_image( |
| 470 | self, |
| 471 | provider: str, |
| 472 | model: str, |
| 473 | prompt: str, |
| 474 | width: Optional[int], |
| 475 | height: Optional[int], |
| 476 | output_path: Optional[str], |
| 477 | image_paths: Optional[list[str]] = None, |
| 478 | **params, |
| 479 | ) -> MediaResult: |
| 480 | from pixelle_video.services.api_services.image_client import ImageClient |
| 481 | |
| 482 | client = self._create_image_client() |
| 483 | save_dir = self._save_dir(output_path, "api_images") |
| 484 | ratio = self._ratio(width, height) |
| 485 | resolution = self._resolution(width, height) |
| 486 | session_id = params.get("session_id") or "pixelle" |
| 487 | |
| 488 | logger.info(f"Generating image via API provider={provider}, model={model}") |
| 489 | paths = await asyncio.to_thread( |
| 490 | client.generate_image, |
| 491 | prompt=prompt, |
| 492 | image_paths=image_paths, |
| 493 | model=model, |
| 494 | save_dir=save_dir, |
| 495 | session_id=session_id, |
| 496 | video_ratio=ratio, |
| 497 | resolution=resolution, |
| 498 | ) |
| 499 | |
| 500 | if not paths: |
| 501 | raise RuntimeError(f"API image generation returned no result: provider={provider}, model={model}") |
| 502 | |
| 503 | result_path = paths[0] |
| 504 | if output_path and os.path.exists(result_path) and os.path.abspath(result_path) != os.path.abspath(output_path): |
| 505 | os.makedirs(os.path.dirname(output_path), exist_ok=True) |
| 506 | os.replace(result_path, output_path) |
| 507 | result_path = output_path |
| 508 | |
| 509 | return MediaResult(media_type="image", url=result_path) |
| 510 | |
| 511 | async def _generate_video( |
| 512 | self, |
| 513 | provider: str, |
| 514 | model: str, |
| 515 | prompt: str, |
| 516 | image_path: Optional[str], |
| 517 | output_path: Optional[str], |
| 518 | duration: Optional[float], |
| 519 | width: Optional[int], |
| 520 | height: Optional[int], |
| 521 | **params, |
| 522 | ) -> MediaResult: |
| 523 | from pixelle_video.services.api_services.video_client import VideoClient |
| 524 | |
| 525 | first_clip_path = params.get("first_clip_path") or params.get("first_video_path") |
| 526 | reference_image_path = params.get("reference_image_path") |
| 527 | reference_image_paths = params.get("reference_image_paths") or [] |
| 528 | reference_video_paths = params.get("reference_video_paths") or [] |
| 529 | has_reference_inputs = bool(reference_image_path or reference_image_paths or reference_video_paths) |
| 530 | capabilities = self._video_capabilities(provider, model) |
| 531 | supports_text_to_video = "text_to_video" in set(capabilities.get("adapter_ability_types") or []) |
| 532 | if not image_path and not first_clip_path and not has_reference_inputs and not supports_text_to_video: |
| 533 | raise ValueError( |
| 534 | "API video models require image_path, first_clip_path, or reference media inputs. " |
| 535 | "Use an image template first or pass input image/video/reference media when calling media generation." |
| 536 | ) |
| 537 | if first_clip_path and not image_path and provider != "dashscope": |
| 538 | raise ValueError(f"first_clip_path is only supported for DashScope wan2.7 models, not provider={provider}.") |
| 539 | |
| 540 | client = self._create_video_client() |
| 541 | save_path = output_path or os.path.join(self._save_dir(None, "api_videos"), "video.mp4") |
| 542 | ratio = params.get("video_ratio") or params.get("ratio") or self._ratio(width, height) |
| 543 | requested_duration = int(duration or params.get("duration") or 5) |
| 544 | safe_duration = self._video_duration(provider, model, requested_duration) |
| 545 | resolution = params.get("resolution") or self._video_resolution(provider, width, height) |
| 546 | video_options = self._video_options(provider, model, params, resolution) |
| 547 | |
| 548 | prompt_to_use = prompt |
| 549 | max_safety_retries = int(params.get("prompt_safety_retries", 1)) |
| 550 | for attempt in range(max_safety_retries + 1): |
| 551 | try: |
| 552 | logger.info( |
| 553 | f"Generating video via API provider={provider}, model={model}" |
| 554 | + (f" (safety retry {attempt})" if attempt else "") |
| 555 | ) |
| 556 | await asyncio.to_thread( |
| 557 | client.generate_video, |
| 558 | prompt=prompt_to_use, |
| 559 | image_path=image_path, |
| 560 | save_path=save_path, |
| 561 | model=model, |
| 562 | duration=safe_duration, |
| 563 | video_ratio=ratio, |
| 564 | **video_options, |
| 565 | ) |
| 566 | break |
| 567 | except Exception as exc: |
| 568 | if attempt >= max_safety_retries or not self._is_content_inspection_error(exc): |
| 569 | raise |
| 570 | |
| 571 | logger.warning( |
| 572 | "API video generation failed content inspection; " |
| 573 | f"neutralizing prompt and retrying once. provider={provider}, model={model}, error={exc}" |
| 574 | ) |
| 575 | prompt_to_use = await self._neutralize_video_prompt(prompt_to_use) |
| 576 | |
| 577 | if not os.path.exists(save_path): |
| 578 | raise RuntimeError(f"API video generation did not create file: {save_path}") |
| 579 | |
| 580 | return MediaResult(media_type="video", url=save_path, duration=safe_duration) |
| 581 | |
| 582 | def _is_content_inspection_error(self, exc: Exception) -> bool: |
| 583 | """Return True when a provider rejects input because of content inspection.""" |
| 584 | message = str(exc).lower() |
| 585 | return any( |
| 586 | marker in message |
| 587 | for marker in ( |
| 588 | "datainspectionfailed", |
| 589 | "inappropriate content", |
| 590 | "green net check failed", |
| 591 | "content inspection", |
| 592 | "safety inspection", |
| 593 | "risk control", |
| 594 | ) |
| 595 | ) |
| 596 | |
| 597 | async def _neutralize_video_prompt(self, prompt: str) -> str: |
| 598 | """Use the configured LLM to rewrite a video prompt into a safer neutral prompt.""" |
| 599 | if not prompt or not prompt.strip(): |
| 600 | return prompt |
| 601 | |
| 602 | rewrite_instruction = f""" |
| 603 | 请将下面的视频生成提示词改写为更中性、安全、适合公开视频生成模型审核的画面描述。 |
| 604 | |
| 605 | 要求： |
| 606 | 1. 保留原本的积极含义、画面主题和视觉风格。 |
| 607 | 2. 去掉可能触发审核的暴力、危险、恐惧、政治、成人、歧视、极端情绪、自伤、违法、攻击性表达。 |
| 608 | 3. 不要提及“审核”“违规”“敏感词”等元信息。 |
| 609 | 4. 只输出改写后的提示词，不要解释。 |
| 610 | 5. 输出优先使用英文，画面描述要具体、平和、正向。 |
| 611 | |
| 612 | 原提示词： |
| 613 | {prompt} |
| 614 | """.strip() |
| 615 | |
| 616 | try: |
| 617 | from pixelle_video.services.llm_service import LLMService |
| 618 | |
| 619 | llm = LLMService(config_manager.config.model_dump()) |
| 620 | rewritten = await llm( |
| 621 | rewrite_instruction, |
| 622 | temperature=0.2, |
| 623 | max_tokens=500, |
| 624 | ) |
| 625 | rewritten = self._clean_rewritten_prompt(str(rewritten)) |
| 626 | if rewritten: |
| 627 | logger.info(f"Neutralized API video prompt: {rewritten[:200]}") |
| 628 | return rewritten |
| 629 | except Exception as exc: |
| 630 | logger.warning(f"Failed to neutralize video prompt with LLM; using fallback sanitizer: {exc}") |
| 631 | |
| 632 | return self._fallback_neutralize_prompt(prompt) |
| 633 | |
| 634 | def _clean_rewritten_prompt(self, text: str) -> str: |
| 635 | """Clean common LLM formatting around a rewritten prompt.""" |
| 636 | cleaned = text.strip() |
| 637 | if cleaned.startswith("```"): |
| 638 | cleaned = cleaned.strip("`").strip() |
| 639 | if cleaned.lower().startswith(("text", "prompt", "english")): |
| 640 | cleaned = cleaned.split("\n", 1)[-1].strip() |
| 641 | for prefix in ("改写后的提示词：", "改写后：", "Prompt:", "Rewritten prompt:"): |
| 642 | if cleaned.startswith(prefix): |
| 643 | cleaned = cleaned[len(prefix):].strip() |
| 644 | return cleaned.strip().strip('"').strip("'") |
| 645 | |
| 646 | def _fallback_neutralize_prompt(self, prompt: str) -> str: |
| 647 | """Conservative fallback if the configured LLM is unavailable.""" |
| 648 | sanitized = prompt |
| 649 | replacements = { |
| 650 | "害怕": "平静", |
| 651 | "恐惧": "沉思", |
| 652 | "危险": "未知", |
| 653 | "挣脱": "走向", |
| 654 | "崩溃": "调整", |
| 655 | "压迫": "压力", |
| 656 | "攻击": "互动", |
| 657 | "血": "红色", |
| 658 | "死亡": "离别", |
| 659 | } |
| 660 | for source, target in replacements.items(): |
| 661 | sanitized = sanitized.replace(source, target) |
| 662 | return ( |
| 663 | "A calm, positive, cinematic scene with gentle natural light, peaceful atmosphere, " |
| 664 | "safe public setting, no violence, no danger, no sensitive content. " |
| 665 | f"Original theme adapted neutrally: {sanitized}" |
| 666 | ) |
| 667 | |
| 668 | def _create_image_client(self): |
| 669 | from pixelle_video.services.api_services.image_client import ImageClient |
| 670 | |
| 671 | cfg = config_manager.get_api_providers_config() |
| 672 | local_proxy = cfg["common"].get("local_proxy") or None |
| 673 | return ImageClient( |
| 674 | dashscope_api_key=cfg["dashscope"].get("api_key") or None, |
| 675 | dashscope_base_url=cfg["dashscope"].get("base_url") or None, |
| 676 | dashscope_local_proxy=local_proxy if cfg["dashscope"].get("use_proxy") else None, |
| 677 | gpt_api_key=cfg["openai"].get("api_key") or None, |
| 678 | gpt_base_url=cfg["openai"].get("base_url") or None, |
| 679 | local_proxy=local_proxy if cfg["openai"].get("use_proxy") else None, |
| 680 | ark_api_key=cfg["ark"].get("api_key") or None, |
| 681 | ark_base_url=cfg["ark"].get("base_url") or None, |
| 682 | ark_local_proxy=local_proxy if cfg["ark"].get("use_proxy") else None, |
| 683 | ) |
| 684 | |
| 685 | def _create_video_client(self): |
| 686 | from pixelle_video.services.api_services.video_client import VideoClient |
| 687 | |
| 688 | cfg = config_manager.get_api_providers_config() |
| 689 | local_proxy = cfg["common"].get("local_proxy") or None |
| 690 | return VideoClient( |
| 691 | dashscope_api_key=cfg["dashscope"].get("api_key") or None, |
| 692 | dashscope_base_url=cfg["dashscope"].get("base_url") or None, |
| 693 | dashscope_local_proxy=local_proxy if cfg["dashscope"].get("use_proxy") else None, |
| 694 | kling_access_key=cfg["kling"].get("access_key") or None, |
| 695 | kling_secret_key=cfg["kling"].get("secret_key") or None, |
| 696 | kling_base_url=cfg["kling"].get("base_url") or None, |
| 697 | kling_local_proxy=local_proxy if cfg["kling"].get("use_proxy") else None, |
| 698 | ark_api_key=cfg["ark"].get("api_key") or None, |
| 699 | ark_base_url=cfg["ark"].get("base_url") or None, |
| 700 | ark_local_proxy=local_proxy if cfg["ark"].get("use_proxy") else None, |
| 701 | ) |
| 702 | |
| 703 | def _save_dir(self, output_path: Optional[str], fallback_name: str) -> str: |
| 704 | if output_path: |
| 705 | return str(Path(output_path).parent) |
| 706 | return get_output_path(fallback_name) |
| 707 | |
| 708 | def _ratio(self, width: Optional[int], height: Optional[int]) -> str: |
| 709 | if not width or not height: |
| 710 | return "16:9" |
| 711 | if width == height: |
| 712 | return "1:1" |
| 713 | return "9:16" if height > width else "16:9" |
| 714 | |
| 715 | def _resolution(self, width: Optional[int], height: Optional[int]) -> str: |
| 716 | largest = max(width or 0, height or 0) |
| 717 | if largest >= 3600: |
| 718 | return "4K" |
| 719 | if largest >= 2000: |
| 720 | return "2K" |
| 721 | return "1080P" |
| 722 | |
| 723 | def _video_resolution(self, provider: str, width: Optional[int], height: Optional[int]) -> str: |
| 724 | resolution = self._resolution(width, height) |
| 725 | if provider == "seedance": |
| 726 | return "1080p" if resolution in {"1080P", "2K", "4K"} else "720p" |
| 727 | return "1080P" if resolution in {"1080P", "2K", "4K"} else "720P" |
| 728 | |
| 729 | def _video_duration(self, provider: str, model: str, duration: int) -> int: |
| 730 | """Normalize requested duration to ranges accepted by common providers.""" |
| 731 | capabilities = self._video_capabilities(provider, model) |
| 732 | duration_contract = capabilities.get("duration") or {} |
| 733 | |
| 734 | if duration_contract.get("verified"): |
| 735 | if duration_contract.get("allowed_values"): |
| 736 | allowed = sorted(duration_contract["allowed_values"]) |
| 737 | return min(allowed, key=lambda value: abs(value - duration)) |
| 738 | |
| 739 | min_duration = int(duration_contract.get("min", duration)) |
| 740 | max_duration = int(duration_contract.get("max", duration)) |
| 741 | return min(max(duration, min_duration), max_duration) |
| 742 | |
| 743 | model_lower = model.lower() |
| 744 | |
| 745 | if provider == "dashscope": |
| 746 | return 10 if duration >= 8 else 5 |
| 747 | |
| 748 | if provider == "kling": |
| 749 | if "v3" in model_lower: |
| 750 | return min(max(duration, 3), 15) |
| 751 | return 10 if duration >= 8 else 5 |
| 752 | |
| 753 | if provider == "seedance": |
| 754 | return min(max(duration, 5), 10) |
| 755 | |
| 756 | return max(duration, 1) |
| 757 | |
| 758 | def _video_capabilities(self, provider: str, model: str) -> dict[str, Any]: |
| 759 | """Return provider/model capability metadata backed by official docs when available.""" |
| 760 | default = { |
| 761 | "ability_type": "image_to_video", |
| 762 | "ability_types": [], |
| 763 | "adapter_ability_types": ["first_frame_i2v"], |
| 764 | "api_contract_verified": False, |
| 765 | "source_urls": [], |
| 766 | "contract_issues": ["No official API contract metadata has been added for this model."], |
| 767 | } |
| 768 | return deepcopy(self.VIDEO_MODEL_CAPABILITIES.get((provider, model), default)) |
| 769 | |
| 770 | def _video_options( |
| 771 | self, |
| 772 | provider: str, |
| 773 | model: str, |
| 774 | params: dict[str, Any], |
| 775 | resolution: str, |
| 776 | ) -> dict[str, Any]: |
| 777 | """Map Pixelle's generic video params to provider client options.""" |
| 778 | options: dict[str, Any] = { |
| 779 | "resolution": resolution, |
| 780 | "negative_prompt": params.get("negative_prompt"), |
| 781 | "watermark": params.get("watermark"), |
| 782 | "seed": params.get("seed"), |
| 783 | } |
| 784 | |
| 785 | if provider == "dashscope": |
| 786 | options.update( |
| 787 | { |
| 788 | "last_image_path": params.get("last_image_path") or params.get("last_frame_path"), |
| 789 | "first_clip_path": params.get("first_clip_path") or params.get("first_video_path"), |
| 790 | "reference_image_path": params.get("reference_image_path"), |
| 791 | "reference_image_paths": params.get("reference_image_paths"), |
| 792 | "reference_video_paths": params.get("reference_video_paths"), |
| 793 | "reference_audio_path": params.get("reference_audio_path") or params.get("reference_voice_path"), |
| 794 | "audio": params.get("audio"), |
| 795 | "audio_path": params.get("audio_path") or params.get("driving_audio_path"), |
| 796 | "prompt_extend": params.get("prompt_extend"), |
| 797 | "shot_type": params.get("shot_type", "multi"), |
| 798 | } |
| 799 | ) |
| 800 | elif provider == "kling": |
| 801 | options.update( |
| 802 | { |
| 803 | "sound": params.get("sound", ""), |
| 804 | "mode": params.get("mode", "pro"), |
| 805 | "cfg_scale": params.get("cfg_scale", 0.5), |
| 806 | } |
| 807 | ) |
| 808 | elif provider == "seedance": |
| 809 | options.update( |
| 810 | { |
| 811 | "generate_audio": params.get("generate_audio"), |
| 812 | } |
| 813 | ) |
| 814 | |
| 815 | return {key: value for key, value in options.items() if value is not None} |
| 816 |