| 1 | import type { DashscopeVideoAiLog } from '../video-ai-log.interface' |
| 2 | import type { UserVideoGenerationRequestDto } from '../video.dto' |
| 3 | import { BadRequestException, Injectable, Logger } from '@nestjs/common' |
| 4 | import { AssetsService, StorageProvider, VideoMetadataService } from '@yikart/assets' |
| 5 | import { AppException, FileUtil, ResponseCode } from '@yikart/common' |
| 6 | import { AiLogChannel, AiLogRepository, AiLogStatus, AiLogType, AssetType } from '@yikart/mongodb' |
| 7 | import { TaskStatus } from '../../../../common' |
| 8 | import { AiAvailabilityService } from '../../../ai-availability/ai-availability.service' |
| 9 | import { |
| 10 | DashscopeCreateVideoTaskRequest, |
| 11 | DashscopeService as DashscopeLibService, |
| 12 | DashscopeQueryVideoTaskResponse, |
| 13 | DashscopeTaskStatus, |
| 14 | } from '../../libs/dashscope' |
| 15 | import { ModelsConfigService } from '../../models-config/models-config.service' |
| 16 | |
| 17 | export interface DashscopeVideoCallbackDto { |
| 18 | id: string |
| 19 | status: DashscopeTaskStatus |
| 20 | requestId?: string |
| 21 | providerModel?: string |
| 22 | videoUrl?: string |
| 23 | usage?: DashscopeQueryVideoTaskResponse['usage'] |
| 24 | error?: string |
| 25 | } |
| 26 | |
| 27 | interface DashscopeModelConfig { |
| 28 | name: string |
| 29 | defaults: { |
| 30 | resolution?: string |
| 31 | aspectRatio?: string |
| 32 | duration?: number |
| 33 | } |
| 34 | durations: number[] |
| 35 | maxInputImages: number |
| 36 | modes: string[] |
| 37 | runtimeModels?: Array<{ |
| 38 | model: string |
| 39 | mode?: string |
| 40 | resolution?: string |
| 41 | }> |
| 42 | } |
| 43 | |
| 44 | @Injectable() |
| 45 | export class DashscopeVideoService { |
| 46 | private readonly logger = new Logger(DashscopeVideoService.name) |
| 47 | |
| 48 | constructor( |
| 49 | private readonly dashscopeLibService: DashscopeLibService, |
| 50 | private readonly aiLogRepo: AiLogRepository, |
| 51 | private readonly assetsService: AssetsService, |
| 52 | private readonly storageProvider: StorageProvider, |
| 53 | private readonly modelsConfigService: ModelsConfigService, |
| 54 | private readonly videoMetadataService: VideoMetadataService, |
| 55 | private readonly aiAvailability: AiAvailabilityService, |
| 56 | ) {} |
| 57 | |
| 58 | private getModelConfig(model: string): DashscopeModelConfig { |
| 59 | const modelConfig = this.modelsConfigService.config.video.generation.find(m => m.name === model) |
| 60 | if (!modelConfig) { |
| 61 | throw new AppException(ResponseCode.InvalidModel) |
| 62 | } |
| 63 | return modelConfig as DashscopeModelConfig |
| 64 | } |
| 65 | |
| 66 | private getProviderModel(modelConfig: DashscopeModelConfig, mode: string, resolution: string | undefined): string { |
| 67 | const runtimeModel = modelConfig.runtimeModels |
| 68 | ?.filter(item => (item.mode == null || item.mode === mode) && (item.resolution == null || item.resolution === resolution)) |
| 69 | .sort((a, b) => Number(b.mode != null) + Number(b.resolution != null) - Number(a.mode != null) - Number(a.resolution != null))[0] |
| 70 | if (!runtimeModel) { |
| 71 | throw new AppException(ResponseCode.InvalidModel) |
| 72 | } |
| 73 | return runtimeModel.model |
| 74 | } |
| 75 | |
| 76 | private async toAccessibleUrl(url: string): Promise<string> { |
| 77 | const parsed = this.storageProvider.parsePathFromUrl(url) |
| 78 | if (parsed.startsWith('http')) { |
| 79 | return url |
| 80 | } |
| 81 | return this.storageProvider.toPresignedUrl(url) |
| 82 | } |
| 83 | |
| 84 | private collectImageUrls(request: UserVideoGenerationRequestDto): string[] { |
| 85 | const imageUrls: string[] = [] |
| 86 | if (request.image) { |
| 87 | if (Array.isArray(request.image)) { |
| 88 | imageUrls.push(...request.image) |
| 89 | } |
| 90 | else { |
| 91 | imageUrls.push(request.image) |
| 92 | } |
| 93 | } |
| 94 | if (request.images) { |
| 95 | imageUrls.push(...request.images) |
| 96 | } |
| 97 | return imageUrls |
| 98 | } |
| 99 | |
| 100 | private resolveMode(request: UserVideoGenerationRequestDto, modelConfig: DashscopeModelConfig, imageUrls: string[]): string { |
| 101 | if (request.video_url || request.videos?.[0]) { |
| 102 | return 'video2video' |
| 103 | } |
| 104 | |
| 105 | if (request.mode) { |
| 106 | if (!modelConfig.modes.includes(request.mode)) { |
| 107 | throw new AppException(ResponseCode.InvalidModel) |
| 108 | } |
| 109 | return request.mode |
| 110 | } |
| 111 | |
| 112 | if (imageUrls.length === 0) { |
| 113 | return 'text2video' |
| 114 | } |
| 115 | |
| 116 | return imageUrls.length === 1 ? 'image2video' : 'multi-image2video' |
| 117 | } |
| 118 | |
| 119 | private getMaxDuration(modelConfig: DashscopeModelConfig): number { |
| 120 | const maxDuration = Math.max(...modelConfig.durations) |
| 121 | if (!Number.isFinite(maxDuration)) { |
| 122 | throw new AppException(ResponseCode.InvalidModel) |
| 123 | } |
| 124 | return maxDuration |
| 125 | } |
| 126 | |
| 127 | private async resolveVideoDuration(request: UserVideoGenerationRequestDto, modelConfig: DashscopeModelConfig, videoUrl: string | undefined): Promise<number | undefined> { |
| 128 | if (request.duration != null) { |
| 129 | return request.duration |
| 130 | } |
| 131 | |
| 132 | if (!videoUrl) { |
| 133 | return modelConfig.defaults.duration |
| 134 | } |
| 135 | |
| 136 | const metadata = await this.videoMetadataService.probeVideoMetadata(FileUtil.buildUrl(videoUrl)) |
| 137 | const metadataDuration = Number(metadata.duration) |
| 138 | if (!Number.isFinite(metadataDuration) || metadataDuration <= 0) { |
| 139 | throw new BadRequestException('video duration is required') |
| 140 | } |
| 141 | |
| 142 | return Math.min(Math.ceil(metadataDuration), this.getMaxDuration(modelConfig)) |
| 143 | } |
| 144 | |
| 145 | private async buildPayload(request: UserVideoGenerationRequestDto, modelConfig: DashscopeModelConfig): Promise<{ |
| 146 | providerModel: string |
| 147 | mode: string |
| 148 | payload: DashscopeCreateVideoTaskRequest |
| 149 | duration?: number |
| 150 | resolution?: string |
| 151 | }> { |
| 152 | if (request.image_tail) { |
| 153 | throw new BadRequestException('DashScope HappyHorse does not support image_tail') |
| 154 | } |
| 155 | |
| 156 | const imageUrls = this.collectImageUrls(request) |
| 157 | const videoUrl = request.video_url ?? request.videos?.[0] |
| 158 | const mode = this.resolveMode(request, modelConfig, imageUrls) |
| 159 | const resolution = request.resolution |
| 160 | const providerModel = this.getProviderModel(modelConfig, mode, resolution) |
| 161 | const ratio = request.ratio |
| 162 | const duration = await this.resolveVideoDuration(request, modelConfig, mode === 'video2video' ? videoUrl : undefined) |
| 163 | const prompt = request.prompt |
| 164 | const parameters: NonNullable<DashscopeCreateVideoTaskRequest['parameters']> = {} |
| 165 | if (resolution) { |
| 166 | parameters.resolution = resolution |
| 167 | } |
| 168 | parameters.watermark = request.watermark ?? false |
| 169 | if (request.seed != null) { |
| 170 | parameters.seed = request.seed |
| 171 | } |
| 172 | const generationParameters = duration != null ? { ...parameters, duration } : parameters |
| 173 | const generationParametersWithRatio = ratio ? { ...generationParameters, ratio } : generationParameters |
| 174 | |
| 175 | if (mode === 'text2video') { |
| 176 | return { |
| 177 | providerModel, |
| 178 | mode, |
| 179 | duration, |
| 180 | resolution, |
| 181 | payload: { |
| 182 | model: providerModel, |
| 183 | input: { prompt }, |
| 184 | parameters: generationParametersWithRatio, |
| 185 | }, |
| 186 | } |
| 187 | } |
| 188 | |
| 189 | if (mode === 'image2video') { |
| 190 | if (imageUrls.length !== 1) { |
| 191 | throw new AppException(ResponseCode.InvalidModel) |
| 192 | } |
| 193 | return { |
| 194 | providerModel, |
| 195 | mode, |
| 196 | duration, |
| 197 | resolution, |
| 198 | payload: { |
| 199 | model: providerModel, |
| 200 | input: { |
| 201 | prompt, |
| 202 | media: [{ type: 'first_frame', url: await this.toAccessibleUrl(imageUrls[0]) }], |
| 203 | }, |
| 204 | parameters: generationParameters, |
| 205 | }, |
| 206 | } |
| 207 | } |
| 208 | |
| 209 | if (mode === 'multi-image2video') { |
| 210 | if (imageUrls.length < 1 || imageUrls.length > modelConfig.maxInputImages) { |
| 211 | throw new AppException(ResponseCode.InvalidModel) |
| 212 | } |
| 213 | return { |
| 214 | providerModel, |
| 215 | mode, |
| 216 | duration, |
| 217 | resolution, |
| 218 | payload: { |
| 219 | model: providerModel, |
| 220 | input: { |
| 221 | prompt, |
| 222 | media: await Promise.all(imageUrls.map(async url => ({ type: 'reference_image', url: await this.toAccessibleUrl(url) }))), |
| 223 | }, |
| 224 | parameters: generationParametersWithRatio, |
| 225 | }, |
| 226 | } |
| 227 | } |
| 228 | |
| 229 | if (mode === 'video2video') { |
| 230 | if (!videoUrl) { |
| 231 | throw new AppException(ResponseCode.InvalidModel) |
| 232 | } |
| 233 | if (imageUrls.length > 5) { |
| 234 | throw new AppException(ResponseCode.InvalidModel) |
| 235 | } |
| 236 | const media = [ |
| 237 | { type: 'video', url: await this.toAccessibleUrl(videoUrl) }, |
| 238 | ...await Promise.all(imageUrls.map(async url => ({ type: 'reference_image', url: await this.toAccessibleUrl(url) }))), |
| 239 | ] |
| 240 | return { |
| 241 | providerModel, |
| 242 | mode, |
| 243 | duration, |
| 244 | resolution, |
| 245 | payload: { |
| 246 | model: providerModel, |
| 247 | input: { prompt, media }, |
| 248 | parameters, |
| 249 | }, |
| 250 | } |
| 251 | } |
| 252 | |
| 253 | throw new AppException(ResponseCode.InvalidModel) |
| 254 | } |
| 255 | |
| 256 | async createFromRequest(request: UserVideoGenerationRequestDto): Promise<{ id: string }> { |
| 257 | const modelConfig = this.getModelConfig(request.model) |
| 258 | const { providerModel, mode, payload, duration, resolution } = await this.buildPayload(request, modelConfig) |
| 259 | |
| 260 | const startedAt = new Date() |
| 261 | const result = await this.aiAvailability.executeAsync( |
| 262 | { provider: 'dashscope', operation: 'videoGeneration', model: providerModel }, |
| 263 | () => this.dashscopeLibService.createVideoTask(payload), |
| 264 | response => response.output?.task_id ?? '', |
| 265 | ) |
| 266 | |
| 267 | const taskId = result.output?.task_id |
| 268 | if (!taskId) { |
| 269 | throw new BadRequestException(result.message || result.code || 'DashScope task id is missing') |
| 270 | } |
| 271 | |
| 272 | this.logger.log({ request, payload, result }, 'Video generation submitted to provider model') |
| 273 | |
| 274 | const aiLog = await this.aiLogRepo.create({ |
| 275 | userId: request.userId, |
| 276 | userType: request.userType, |
| 277 | taskId, |
| 278 | model: request.model, |
| 279 | channel: AiLogChannel.Dashscope, |
| 280 | startedAt, |
| 281 | type: AiLogType.Video, |
| 282 | request: { |
| 283 | model: request.model, |
| 284 | providerModel, |
| 285 | mode, |
| 286 | prompt: request.prompt, |
| 287 | images: this.collectImageUrls(request), |
| 288 | videoUrl: request.video_url ?? request.videos?.[0], |
| 289 | resolution, |
| 290 | ratio: request.ratio, |
| 291 | duration, |
| 292 | watermark: request.watermark, |
| 293 | seed: request.seed, |
| 294 | }, |
| 295 | response: { |
| 296 | id: taskId, |
| 297 | requestId: result.request_id, |
| 298 | providerModel, |
| 299 | status: result.output?.task_status ?? DashscopeTaskStatus.Pending, |
| 300 | }, |
| 301 | status: AiLogStatus.Generating, |
| 302 | }) |
| 303 | |
| 304 | return { id: aiLog.id } |
| 305 | } |
| 306 | |
| 307 | private async failTask(aiLog: DashscopeVideoAiLog, taskId: string, errorMessage: string, elapsedMs: number, status = DashscopeTaskStatus.Failed): Promise<DashscopeVideoCallbackDto> { |
| 308 | const callbackData: DashscopeVideoCallbackDto = { |
| 309 | id: taskId, |
| 310 | status, |
| 311 | providerModel: aiLog.request.providerModel, |
| 312 | error: errorMessage, |
| 313 | } |
| 314 | |
| 315 | const updatedAiLog = await this.aiLogRepo.updateByIdAndStatus(aiLog.id, AiLogStatus.Generating, { |
| 316 | $set: { |
| 317 | status: AiLogStatus.Failed, |
| 318 | response: callbackData, |
| 319 | duration: elapsedMs, |
| 320 | errorMessage, |
| 321 | }, |
| 322 | }) |
| 323 | |
| 324 | if (!updatedAiLog) { |
| 325 | return aiLog.response! |
| 326 | } |
| 327 | |
| 328 | await this.aiAvailability.recordAsyncComplete( |
| 329 | taskId, |
| 330 | { provider: 'dashscope', operation: 'videoGeneration', model: aiLog.request.providerModel ?? aiLog.model }, |
| 331 | { success: false, latencyMs: elapsedMs, errorMessage }, |
| 332 | ) |
| 333 | |
| 334 | return callbackData |
| 335 | } |
| 336 | |
| 337 | async callback(queryResult: DashscopeQueryVideoTaskResponse): Promise<DashscopeVideoCallbackDto> { |
| 338 | const taskId = queryResult.output.task_id |
| 339 | const aiLog = await this.aiLogRepo.getByTaskId(taskId) |
| 340 | if (!aiLog || aiLog.channel !== AiLogChannel.Dashscope) { |
| 341 | throw new AppException(ResponseCode.InvalidAiTaskId) |
| 342 | } |
| 343 | const dashscopeAiLog = aiLog as DashscopeVideoAiLog |
| 344 | |
| 345 | if (dashscopeAiLog.status !== AiLogStatus.Generating) { |
| 346 | return dashscopeAiLog.response! |
| 347 | } |
| 348 | |
| 349 | const status = queryResult.output.task_status |
| 350 | if (status === DashscopeTaskStatus.Pending || status === DashscopeTaskStatus.Running) { |
| 351 | return { |
| 352 | id: taskId, |
| 353 | status, |
| 354 | requestId: queryResult.request_id, |
| 355 | providerModel: dashscopeAiLog.request.providerModel, |
| 356 | } |
| 357 | } |
| 358 | |
| 359 | const elapsedMs = Date.now() - dashscopeAiLog.startedAt.getTime() |
| 360 | |
| 361 | if (status === DashscopeTaskStatus.Succeeded && queryResult.output.video_url) { |
| 362 | const uploaded = await this.assetsService.uploadFromUrl(dashscopeAiLog.userId, { |
| 363 | url: queryResult.output.video_url, |
| 364 | type: AssetType.AiVideo, |
| 365 | }, dashscopeAiLog.model) |
| 366 | |
| 367 | const callbackData: DashscopeVideoCallbackDto = { |
| 368 | id: taskId, |
| 369 | status, |
| 370 | requestId: queryResult.request_id, |
| 371 | providerModel: dashscopeAiLog.request.providerModel, |
| 372 | videoUrl: uploaded.asset.path, |
| 373 | usage: queryResult.usage, |
| 374 | } |
| 375 | |
| 376 | const updatedAiLog = await this.aiLogRepo.updateByIdAndStatus( |
| 377 | dashscopeAiLog.id, |
| 378 | AiLogStatus.Generating, |
| 379 | { |
| 380 | $set: { |
| 381 | status: AiLogStatus.Success, |
| 382 | response: callbackData, |
| 383 | duration: elapsedMs, |
| 384 | }, |
| 385 | }, |
| 386 | ) |
| 387 | |
| 388 | if (!updatedAiLog) { |
| 389 | return dashscopeAiLog.response! |
| 390 | } |
| 391 | |
| 392 | await this.aiAvailability.recordAsyncComplete( |
| 393 | taskId, |
| 394 | { provider: 'dashscope', operation: 'videoGeneration', model: dashscopeAiLog.request.providerModel ?? dashscopeAiLog.model }, |
| 395 | { success: true, latencyMs: elapsedMs }, |
| 396 | ) |
| 397 | |
| 398 | return callbackData |
| 399 | } |
| 400 | |
| 401 | const errorMessage = queryResult.output.message |
| 402 | || queryResult.output.code |
| 403 | || (status === DashscopeTaskStatus.Succeeded ? 'DashScope task succeeded but no video URL returned' : `DashScope task ${status}`) |
| 404 | return status === DashscopeTaskStatus.Succeeded |
| 405 | ? this.failTask(dashscopeAiLog, taskId, errorMessage, elapsedMs) |
| 406 | : this.failTask(dashscopeAiLog, taskId, errorMessage, elapsedMs, status) |
| 407 | } |
| 408 | |
| 409 | getTaskResult(result: DashscopeVideoCallbackDto) { |
| 410 | const status = { |
| 411 | [DashscopeTaskStatus.Succeeded]: TaskStatus.Success, |
| 412 | [DashscopeTaskStatus.Failed]: TaskStatus.Failure, |
| 413 | [DashscopeTaskStatus.Canceled]: TaskStatus.Failure, |
| 414 | [DashscopeTaskStatus.Unknown]: TaskStatus.Failure, |
| 415 | [DashscopeTaskStatus.Pending]: TaskStatus.InProgress, |
| 416 | [DashscopeTaskStatus.Running]: TaskStatus.InProgress, |
| 417 | }[result.status] |
| 418 | |
| 419 | return { |
| 420 | status, |
| 421 | videoUrl: result.videoUrl ? FileUtil.buildUrl(result.videoUrl) : undefined, |
| 422 | error: result.error ? { message: result.error } : undefined, |
| 423 | } |
| 424 | } |
| 425 | |
| 426 | extractInput(request: DashscopeVideoAiLog['request']) { |
| 427 | const image = !request.images?.length |
| 428 | ? undefined |
| 429 | : request.images.length === 1 |
| 430 | ? request.images[0] |
| 431 | : request.images |
| 432 | |
| 433 | return { |
| 434 | prompt: request.prompt || '', |
| 435 | image, |
| 436 | images: request.images, |
| 437 | videoUrl: request.videoUrl, |
| 438 | duration: request.duration, |
| 439 | resolution: request.resolution, |
| 440 | aspectRatio: request.ratio, |
| 441 | watermark: request.watermark, |
| 442 | } |
| 443 | } |
| 444 | } |
| 445 |