| 1 | import { createSdkMcpServer, McpSdkServerConfigWithInstance } from '@anthropic-ai/claude-agent-sdk' |
| 2 | import { Injectable, Logger } from '@nestjs/common' |
| 3 | import { UserType } from '@yikart/common' |
| 4 | import { z } from 'zod' |
| 5 | import { AiAvailabilityService } from '../../../ai-availability' |
| 6 | import { AideoService } from '../../../ai/aideo' |
| 7 | import { AideoTaskStatus } from '../../../ai/libs/volcengine' |
| 8 | import { McpServerName } from '../../agent.constants' |
| 9 | import { errorResult, successResult, wrapTool } from '../mcp.utils' |
| 10 | |
| 11 | const submitAideoTaskPromptSchema = z.object({ |
| 12 | prompt: z.string(), |
| 13 | multiInputs: z.array(z.string()), |
| 14 | }) |
| 15 | |
| 16 | const getAideoTaskStatusSchema = z.object({ |
| 17 | taskId: z.string(), |
| 18 | }) |
| 19 | |
| 20 | export enum AideoToolName { |
| 21 | SubmitAideoTask = 'submitAideoTask', |
| 22 | GetAideoTaskStatus = 'getAideoTaskStatus', |
| 23 | } |
| 24 | |
| 25 | @Injectable() |
| 26 | export class AideoMcp { |
| 27 | private readonly logger = new Logger(AideoMcp.name) |
| 28 | |
| 29 | constructor( |
| 30 | private readonly aideoService: AideoService, |
| 31 | private readonly aiAvailability: AiAvailabilityService, |
| 32 | ) { } |
| 33 | |
| 34 | createSubmitAideoTaskTool(userId: string, userType: UserType) { |
| 35 | return wrapTool( |
| 36 | this.logger, |
| 37 | AideoToolName.SubmitAideoTask, |
| 38 | `Submit an AI-powered video processing task using natural language instructions. Provide prompt describing the desired operation, and multiInputs (array of video/image/audio URLs). |
| 39 | |
| 40 | **Capabilities**: |
| 41 | - Multiple Images → Video: Stitch/combine images into video with transitions |
| 42 | - Multiple Videos → Single Video: Concatenate/merge video clips |
| 43 | - Video + Audio: Merge video with background music |
| 44 | - Highlight Extraction: Extract exciting clips for trailers |
| 45 | - Video Translation: Translate subtitles, clone voice, sync lip movements |
| 46 | - Subtitle Removal: Remove hardcoded subtitles using AI inpainting |
| 47 | - Video Understanding: Generate summaries, keywords, plot analysis (up to 2 hours) |
| 48 | |
| 49 | **Video Translation - Subtitle Recognition**: |
| 50 | - Default: ASR (speech recognition) - extracts text from audio track, suitable for videos without hardcoded subtitles |
| 51 | - OCR: Add "ocr" or "硬字幕" or "hardcoded" in prompt - extracts text from video frames, use when video has burned-in subtitles |
| 52 | - Vision: Add "vision" in prompt - AI-based understanding of both visual and audio content |
| 53 | |
| 54 | Note: For video style transfer (comic/anime), use submitVideoStyleTransfer instead. Processing time: ~10 minutes per 1-minute video. Returns taskId for status tracking.`, |
| 55 | submitAideoTaskPromptSchema.shape, |
| 56 | async ({ prompt, multiInputs }) => { |
| 57 | const result = await this.aideoService.submitAideoTask({ |
| 58 | userId, |
| 59 | userType, |
| 60 | prompt, |
| 61 | multiInputs, |
| 62 | }) |
| 63 | return successResult(`Aideo task submitted successfully, taskId: ${result.taskId}`) |
| 64 | }, |
| 65 | this.aiAvailability, |
| 66 | ) |
| 67 | } |
| 68 | |
| 69 | createGetAideoTaskStatusTool(userId: string, userType: UserType) { |
| 70 | return wrapTool( |
| 71 | this.logger, |
| 72 | AideoToolName.GetAideoTaskStatus, |
| 73 | `Get AI video processing task status and results. |
| 74 | |
| 75 | **Returns by Task Type**: |
| 76 | - Translation: Complete translated video URL |
| 77 | - Erase: Clean video without subtitles |
| 78 | - Highlight: Array of extracted highlight clip URLs |
| 79 | - VCreative: Edited video URL |
| 80 | - Vision: Summary, keywords, plot analysis (JSON)`, |
| 81 | getAideoTaskStatusSchema.shape, |
| 82 | async ({ taskId }) => { |
| 83 | const result = await this.aideoService.getAideoTask({ userId, userType, taskId }) |
| 84 | if (result.status === AideoTaskStatus.Failed) { |
| 85 | return errorResult(`Aideo task failed. Error: ${result.errorMessage || 'Unknown error'}`) |
| 86 | } |
| 87 | |
| 88 | if (result.status === AideoTaskStatus.Completed) { |
| 89 | let outputInfo = `Task completed successfully!\n\nStatus: ${result.status}\n` |
| 90 | |
| 91 | if (result.apiResponses && result.apiResponses.length > 0) { |
| 92 | const apiResponse = result.apiResponses[0] |
| 93 | const aiTranslation = apiResponse['AITranslation'] |
| 94 | const erase = apiResponse['Erase'] |
| 95 | const highlight = apiResponse['Highlight'] |
| 96 | const vCreative = apiResponse['VCreative'] |
| 97 | const vision = apiResponse['Vision'] |
| 98 | |
| 99 | if (aiTranslation?.ProjectInfo?.OutputVideo?.Url) { |
| 100 | outputInfo += `\nOutput Video URL: ${aiTranslation.ProjectInfo.OutputVideo.Url}` |
| 101 | if (aiTranslation.ProjectInfo.OutputVideo.Vid) { |
| 102 | outputInfo += `\nOutput VID: ${aiTranslation.ProjectInfo.OutputVideo.Vid}` |
| 103 | } |
| 104 | } |
| 105 | else if (erase?.Output?.Task?.Erase?.File?.url) { |
| 106 | outputInfo += `\nOutput Video URL: ${erase.Output.Task.Erase.File.url}` |
| 107 | } |
| 108 | else if (highlight?.Edits && highlight.Edits.length > 0) { |
| 109 | outputInfo += `\n\nHighlight Clips:` |
| 110 | highlight.Edits.forEach((edit: { url?: string }, index: number) => { |
| 111 | if (edit.url) { |
| 112 | outputInfo += `\n Clip ${index + 1}: ${edit.url}` |
| 113 | } |
| 114 | }) |
| 115 | } |
| 116 | else if (vCreative?.OutputJson) { |
| 117 | const outputJson = vCreative.OutputJson |
| 118 | if (typeof outputJson !== 'string') { |
| 119 | const url = outputJson.Result?.url || outputJson.url |
| 120 | if (url) { |
| 121 | outputInfo += `\nOutput Video URL: ${url}` |
| 122 | } |
| 123 | } |
| 124 | } |
| 125 | else if (vision) { |
| 126 | outputInfo += `\n\nVision Analysis Results:\n${vision.Content}` |
| 127 | } |
| 128 | } |
| 129 | |
| 130 | return successResult(outputInfo) |
| 131 | } |
| 132 | |
| 133 | return successResult(`Task status: ${result.status}. Please continue polling.`) |
| 134 | }, |
| 135 | this.aiAvailability, |
| 136 | ) |
| 137 | } |
| 138 | |
| 139 | createServer(userId: string, userType: UserType): McpSdkServerConfigWithInstance { |
| 140 | return createSdkMcpServer({ |
| 141 | name: McpServerName.Aideo, |
| 142 | version: '1.0.0', |
| 143 | tools: [ |
| 144 | this.createSubmitAideoTaskTool(userId, userType), |
| 145 | this.createGetAideoTaskStatusTool(userId, userType), |
| 146 | ], |
| 147 | }) |
| 148 | } |
| 149 | } |
| 150 |