| 1 | import { createSdkMcpServer, McpSdkServerConfigWithInstance } from '@anthropic-ai/claude-agent-sdk' |
| 2 | import { Injectable, Logger } from '@nestjs/common' |
| 3 | import { AssetsService } from '@yikart/assets' |
| 4 | import { UserType } from '@yikart/common' |
| 5 | import { AiLogChannel, AiLogRepository, AiLogStatus, AiLogType, AssetType } from '@yikart/mongodb' |
| 6 | import { z } from 'zod' |
| 7 | import { AiAvailabilityService } from '../../../ai-availability' |
| 8 | import { VolcengineService } from '../../../ai/libs/volcengine' |
| 9 | import { DirectEditApplicationType, DirectEditParam } from '../../../ai/libs/volcengine/volcengine.interface' |
| 10 | import { McpServerName } from '../../agent.constants' |
| 11 | import { errorResult, successResult, wrapTool } from '../mcp.utils' |
| 12 | import { |
| 13 | getVideoEditTaskStatusSchema, |
| 14 | submitDirectEditTaskSchema, |
| 15 | VideoEditToolName, |
| 16 | } from './common' |
| 17 | import { VolcengineVideoUtils } from './volcengine.utils' |
| 18 | |
| 19 | @Injectable() |
| 20 | export class VideoEditMcp { |
| 21 | private readonly logger = new Logger(VideoEditMcp.name) |
| 22 | |
| 23 | constructor( |
| 24 | private readonly volcengineService: VolcengineService, |
| 25 | private readonly assetsService: AssetsService, |
| 26 | private readonly aiAvailability: AiAvailabilityService, |
| 27 | private readonly aiLogRepo: AiLogRepository, |
| 28 | ) {} |
| 29 | |
| 30 | /** |
| 31 | * 从 Track 中推导 Canvas 尺寸 |
| 32 | * 若 Canvas 已提供则直接返回，否则从 Track 中提取第一个 vid:// 视频源的实际尺寸 |
| 33 | */ |
| 34 | private async resolveCanvas( |
| 35 | canvas: { Width: number, Height: number } | undefined, |
| 36 | track: z.infer<typeof submitDirectEditTaskSchema>['Track'], |
| 37 | ): Promise<{ Width: number, Height: number }> { |
| 38 | if (canvas) { |
| 39 | return canvas |
| 40 | } |
| 41 | |
| 42 | let vid: string | undefined |
| 43 | for (const layer of track) { |
| 44 | for (const element of layer) { |
| 45 | if (element.Type === 'video' && element.Source?.startsWith('vid://')) { |
| 46 | vid = element.Source.replace('vid://', '') |
| 47 | break |
| 48 | } |
| 49 | } |
| 50 | if (vid) |
| 51 | break |
| 52 | } |
| 53 | |
| 54 | if (!vid) { |
| 55 | throw new Error('Canvas not provided and no vid:// video source found in Track. Please provide Canvas dimensions or use vid:// video sources.') |
| 56 | } |
| 57 | |
| 58 | const mediaInfos = await this.volcengineService.getMediaInfos({ Vids: vid }) |
| 59 | const sourceInfo = mediaInfos.MediaInfoList?.[0]?.SourceInfo |
| 60 | |
| 61 | if (!sourceInfo?.Width || !sourceInfo?.Height) { |
| 62 | throw new Error(`Canvas not provided and failed to retrieve video dimensions for vid://${vid}. Please provide Canvas dimensions explicitly.`) |
| 63 | } |
| 64 | |
| 65 | return { Width: sourceInfo.Width, Height: sourceInfo.Height } |
| 66 | } |
| 67 | |
| 68 | /** |
| 69 | * 提交视频编辑任务（直接使用 Track 结构） |
| 70 | */ |
| 71 | createSubmitDirectEditTaskTool(userId: string, userType: UserType) { |
| 72 | return wrapTool( |
| 73 | this.logger, |
| 74 | VideoEditToolName.SubmitDirectEditTask, |
| 75 | `Submit a video editing task using Volcengine Track structure. |
| 76 | |
| 77 | **CRITICAL - ALL PosX/PosY Rules**: |
| 78 | - ALL PosX/PosY values are TOP-LEFT corner coordinates, NOT center point |
| 79 | - This applies to: transform, crop, delogo, and any other filter with PosX/PosY |
| 80 | - For full-screen video: ALWAYS use PosX: 0, PosY: 0 |
| 81 | - WRONG: Using canvas center (640, 360) or (360, 640) - this causes video to go off-canvas |
| 82 | - CORRECT: Use (0, 0) for top-left alignment when video should fill the canvas |
| 83 | |
| 84 | **Canvas** (optional, recommended to omit): |
| 85 | - If omitted, auto-detected from the primary video source in Track (recommended) |
| 86 | - Only provide when you need a custom canvas size (cropping, letterboxing, rotation) |
| 87 | - If provided: Width = horizontal pixels, Height = vertical pixels (from getVideoInfo) |
| 88 | - DO NOT swap Width and Height |
| 89 | |
| 90 | **IMPORTANT**: Before using this tool, you must read the "editing-videos" skill document to understand the complete EditParam structure and available resource IDs. |
| 91 | |
| 92 | **Time unit**: All time values are in MILLISECONDS (1 second = 1000 milliseconds) |
| 93 | |
| 94 | **Workflow**: |
| 95 | 1. Call getVideoInfo to get source video VID and dimensions |
| 96 | 2. Read the "editing-videos" skill for EditParam reference |
| 97 | 3. Build your Track structure with correct PosX/PosY values (top-left corner!) |
| 98 | 4. Submit with this tool (omit Canvas for auto-detection) |
| 99 | 5. Poll getVideoEditTaskStatus for results`, |
| 100 | submitDirectEditTaskSchema.shape, |
| 101 | async ({ Canvas: inputCanvas, Output, Track }) => { |
| 102 | const startedAt = new Date() |
| 103 | const timestamp = Date.now().toString(16) |
| 104 | |
| 105 | const canvas = await this.resolveCanvas(inputCanvas, Track) |
| 106 | |
| 107 | const editParam: DirectEditParam = { |
| 108 | Canvas: { |
| 109 | Width: canvas.Width, |
| 110 | Height: canvas.Height, |
| 111 | }, |
| 112 | Output: Output |
| 113 | ? { |
| 114 | Fps: Output.Fps, |
| 115 | DisableVideo: Output.DisableVideo, |
| 116 | DisableAudio: Output.DisableAudio, |
| 117 | Alpha: Output.Alpha, |
| 118 | Codec: Output.Codec |
| 119 | ? { |
| 120 | VideoCodec: Output.Codec.VideoCodec, |
| 121 | AudioCodec: Output.Codec.AudioCodec, |
| 122 | VideoBitrate: Output.Codec.VideoBitRate, |
| 123 | AudioBitrate: Output.Codec.AudioBitrate, |
| 124 | } |
| 125 | : undefined, |
| 126 | } |
| 127 | : undefined, |
| 128 | Track: Track as DirectEditParam['Track'], |
| 129 | Upload: { |
| 130 | SpaceName: this.volcengineService['config'].spaceName, |
| 131 | VideoName: `direct_edit_${timestamp}`, |
| 132 | FileName: `direct_edit_${timestamp}.mp4`, |
| 133 | }, |
| 134 | } |
| 135 | |
| 136 | const result = await this.volcengineService.submitDirectEditTaskAsync({ |
| 137 | Application: DirectEditApplicationType.VideoTrackToB, |
| 138 | EditParam: editParam, |
| 139 | }) |
| 140 | |
| 141 | const aiLog = await this.aiLogRepo.create({ |
| 142 | userId, |
| 143 | userType, |
| 144 | taskId: result.ReqId, |
| 145 | model: 'video-edit', |
| 146 | channel: AiLogChannel.Volcengine, |
| 147 | startedAt, |
| 148 | type: AiLogType.VideoEdit, |
| 149 | request: { Canvas: canvas, Output, tracksCount: Track.length }, |
| 150 | status: AiLogStatus.Generating, |
| 151 | }) |
| 152 | |
| 153 | return successResult(`Video edit task submitted. Task ID: ${aiLog.id}`) |
| 154 | }, |
| 155 | this.aiAvailability, |
| 156 | ) |
| 157 | } |
| 158 | |
| 159 | /** |
| 160 | * 查询视频剪辑任务状态 |
| 161 | */ |
| 162 | createGetVideoEditTaskStatusTool(userId: string, _userType: UserType) { |
| 163 | return wrapTool( |
| 164 | this.logger, |
| 165 | VideoEditToolName.GetVideoEditTaskStatus, |
| 166 | `Check the status of a video editing task. |
| 167 | |
| 168 | **Status values**: |
| 169 | - pending/start/processing: Task is running |
| 170 | - success: Task completed, returns output URL |
| 171 | - failed/failed_run: Task failed with error message |
| 172 | |
| 173 | Returns detailed error information on failure.`, |
| 174 | getVideoEditTaskStatusSchema.shape, |
| 175 | async ({ taskId }) => { |
| 176 | const aiLog = await this.aiLogRepo.getById(taskId) |
| 177 | if (!aiLog) { |
| 178 | return errorResult('Task not found. Please check the task ID.') |
| 179 | } |
| 180 | |
| 181 | const volcTaskId = aiLog.taskId |
| 182 | if (!volcTaskId) { |
| 183 | return errorResult('Task record is invalid. Missing Volcengine task ID.') |
| 184 | } |
| 185 | |
| 186 | if (aiLog.status === AiLogStatus.Success) { |
| 187 | const outputUrl = aiLog.response?.outputUrl |
| 188 | return successResult( |
| 189 | outputUrl |
| 190 | ? `Task completed successfully! Output Video URL: ${outputUrl}. The video has been processed and is now available.` |
| 191 | : 'Task completed successfully.', |
| 192 | ) |
| 193 | } |
| 194 | |
| 195 | if (aiLog.status === AiLogStatus.Failed) { |
| 196 | return errorResult(aiLog.response?.error ?? 'Task failed.') |
| 197 | } |
| 198 | |
| 199 | const result = await this.volcengineService.getDirectEditResult({ |
| 200 | ReqIds: [volcTaskId], |
| 201 | }) |
| 202 | |
| 203 | if (result.Status === 'success') { |
| 204 | const outputVid = result.OutputVid |
| 205 | |
| 206 | if (!outputVid) { |
| 207 | return errorResult('Task completed but no output video found.') |
| 208 | } |
| 209 | |
| 210 | this.logger.debug({ taskId, volcTaskId, outputVid }, '[VideoEdit] 任务完成，开始下载上传') |
| 211 | |
| 212 | const outputUrl = await VolcengineVideoUtils.saveVideoFromVid( |
| 213 | outputVid, |
| 214 | userId, |
| 215 | 'edited', |
| 216 | 'video-edit', |
| 217 | this.volcengineService, |
| 218 | this.assetsService, |
| 219 | this.logger, |
| 220 | AssetType.VideoEdit, |
| 221 | ) |
| 222 | |
| 223 | if (!outputUrl) { |
| 224 | this.logger.error({ taskId, volcTaskId, outputVid }, '[VideoEdit] 视频上传失败') |
| 225 | return errorResult('Task completed but failed to upload video. Please try again.') |
| 226 | } |
| 227 | |
| 228 | this.logger.debug({ taskId, outputUrl }, '[VideoEdit] 视频上传成功') |
| 229 | |
| 230 | const updatedAiLog = await this.aiLogRepo.updateByIdAndStatus(aiLog.id, AiLogStatus.Generating, { |
| 231 | $set: { |
| 232 | status: AiLogStatus.Success, |
| 233 | finishedAt: new Date(), |
| 234 | response: { outputUrl, outputVid }, |
| 235 | }, |
| 236 | }) |
| 237 | |
| 238 | if (!updatedAiLog) { |
| 239 | return errorResult('Task is no longer running. Please check the task status again.') |
| 240 | } |
| 241 | |
| 242 | return successResult( |
| 243 | `Task completed successfully! Output Video URL: ${outputUrl}. The video has been processed and is now available.`, |
| 244 | ) |
| 245 | } |
| 246 | else if (result.Status === 'processing' || result.Status === 'pending' || result.Status === 'start') { |
| 247 | return successResult('Task is still processing. Please continue to wait and check again...') |
| 248 | } |
| 249 | else { |
| 250 | const errorMessage = result.Message |
| 251 | ? `Task failed: ${result.Message}` |
| 252 | : 'Task failed. Please check the video source and parameters, then try again.' |
| 253 | |
| 254 | const updatedAiLog = await this.aiLogRepo.updateByIdAndStatus(aiLog.id, AiLogStatus.Generating, { |
| 255 | $set: { |
| 256 | status: AiLogStatus.Failed, |
| 257 | finishedAt: new Date(), |
| 258 | response: { error: errorMessage }, |
| 259 | }, |
| 260 | }) |
| 261 | |
| 262 | if (!updatedAiLog) { |
| 263 | return errorResult('Task is no longer running. Please check the task status again.') |
| 264 | } |
| 265 | |
| 266 | return errorResult(errorMessage) |
| 267 | } |
| 268 | }, |
| 269 | this.aiAvailability, |
| 270 | ) |
| 271 | } |
| 272 | |
| 273 | createServer(userId: string, userType: UserType): McpSdkServerConfigWithInstance { |
| 274 | return createSdkMcpServer({ |
| 275 | name: McpServerName.VideoEdit, |
| 276 | version: '1.0.0', |
| 277 | tools: [ |
| 278 | this.createSubmitDirectEditTaskTool(userId, userType), |
| 279 | this.createGetVideoEditTaskStatusTool(userId, userType), |
| 280 | ], |
| 281 | }) |
| 282 | } |
| 283 | } |
| 284 | |
| 285 | export { VideoEditToolName } from './common' |
| 286 |