1063 lines
34 KiB
TypeScript
1063 lines
34 KiB
TypeScript
import { OptionRealmService } from '@/define/db/service/optionService'
|
||
import { OptionKeyName } from '@/define/enum/option'
|
||
import { SettingModal } from '@/define/model/setting'
|
||
import { TaskModal } from '@/define/model/task'
|
||
import { optionSerialization } from '../option/optionSerialization'
|
||
import {
|
||
CheckFileOrDirExist,
|
||
CheckFolderExistsOrCreate,
|
||
CopyFileOrFolder
|
||
} from '@/define/Tools/file'
|
||
import fs from 'fs'
|
||
import { ValidateJson } from '@/define/Tools/validate'
|
||
import axios from 'axios'
|
||
import { cloneDeep, isEmpty } from 'lodash'
|
||
import { BookBackTaskStatus, BookTaskStatus, OperateBookType } from '@/define/enum/bookEnum'
|
||
import { SendReturnMessage, successMessage } from '@/public/generalTools'
|
||
import { Book } from '@/define/model/book/book'
|
||
import { MJAction } from '@/define/enum/mjEnum'
|
||
import { ImageGenerateMode } from '@/define/data/mjData'
|
||
import path from 'path'
|
||
import { getProjectPath } from '../option/optionCommonService'
|
||
import { SDServiceHandle } from './sdServiceHandle'
|
||
import { t } from '@/i18n'
|
||
import { WorkflowModel } from '@/define/model/workflow'
|
||
import { BookTaskDetail } from '@/define/model/book/bookTaskDetail'
|
||
import { VideoStatus } from '@/define/enum/video'
|
||
import { ResponseMessageType } from '@/define/enum/softwareEnum'
|
||
import { ComfyUIWorkflowType } from '@/define/enum/comfyuiEnum'
|
||
import { RetryWithBackoff } from '@/define/Tools/common'
|
||
|
||
export class ComfyUIServiceHandle extends SDServiceHandle {
|
||
constructor() {
|
||
super()
|
||
}
|
||
|
||
//#region 生图主流程
|
||
/**
|
||
* 提交并跟进单条任务的ComfyUI生图流程，串联初始化、提示词合并、请求发送与结果监听。
|
||
* @param task 等待执行的任务实体。
|
||
*/
|
||
ComfyUIImageGenerate = async (task: TaskModal.Task) => {
|
||
try {
|
||
await this.InitSDBasic()
|
||
let comfyuiSimpleSetting = await this.GetComfyUISetting()
|
||
let bookTaskDetail = await this.bookTaskDetailService.GetBookTaskDetailDataById(
|
||
task.bookTaskDetailId as string, true
|
||
)
|
||
|
||
let book = await this.bookService.GetBookDataById(bookTaskDetail.bookId as string, true)
|
||
|
||
let bookTask = await this.bookTaskService.GetBookTaskDataById(
|
||
bookTaskDetail.bookTaskId as string, true
|
||
)
|
||
|
||
|
||
// 调用方法合并提示词
|
||
let mergeRes = await this.MergeSDPrompt(
|
||
task.bookTaskDetailId as string,
|
||
OperateBookType.BOOKTASKDETAIL
|
||
)
|
||
if (mergeRes.code == 0) {
|
||
throw new Error(mergeRes.message)
|
||
}
|
||
// 获取提示词
|
||
bookTaskDetail.prompt = mergeRes.data[0].prompt
|
||
|
||
let prompt = bookTaskDetail.prompt
|
||
let negativePrompt = comfyuiSimpleSetting.negativePrompt
|
||
|
||
let imageWorkflow = await this.GetComfyUIWorkflow(comfyuiSimpleSetting.selectedWorkflow as string);
|
||
|
||
// 开始组合请求体
|
||
let body = await this.GetComfyUIImageAPIBody(
|
||
prompt ?? '',
|
||
negativePrompt ?? '',
|
||
imageWorkflow.workflowFilePath
|
||
)
|
||
|
||
// 开始发送请求
|
||
let resData = await this.SubmitComfyUITask(body, comfyuiSimpleSetting)
|
||
|
||
// 修改任务状态
|
||
await this.bookTaskDetailService.ModifyBookTaskDetailById(task.bookTaskDetailId as string, {
|
||
status: BookTaskStatus.IMAGE
|
||
})
|
||
this.taskListService.UpdateTaskStatus({
|
||
id: task.id as string,
|
||
status: BookBackTaskStatus.RUNNING
|
||
})
|
||
SendReturnMessage(
|
||
{
|
||
code: 1,
|
||
message: t('任务已提交'),
|
||
id: task.bookTaskDetailId as string,
|
||
data: {
|
||
status: 'submited',
|
||
message: t('任务已提交'),
|
||
id: task.bookTaskDetailId as string
|
||
} as any
|
||
},
|
||
task.messageName as string
|
||
)
|
||
|
||
await this.FetchComfyUIImageTask(
|
||
task,
|
||
resData.prompt_id,
|
||
book,
|
||
bookTask,
|
||
bookTaskDetail,
|
||
comfyuiSimpleSetting
|
||
)
|
||
} catch (error: any) {
|
||
let errorMsg = t("ComfyUI生图失败，{error}", {
|
||
error: error.message()
|
||
})
|
||
this.taskListService.UpdateTaskStatus({
|
||
id: task.id as string,
|
||
status: BookBackTaskStatus.FAIL,
|
||
errorMessage: errorMsg
|
||
})
|
||
this.bookTaskDetailService.UpdateBookTaskDetailMjMessage(
|
||
task.bookTaskDetailId as string,
|
||
{
|
||
mjApiUrl: '',
|
||
progress: 0,
|
||
category: ImageGenerateMode.ComfyUI,
|
||
imageClick: '',
|
||
imageShow: '',
|
||
messageId: '',
|
||
action: MJAction.IMAGINE,
|
||
status: 'error',
|
||
message: errorMsg
|
||
}
|
||
)
|
||
|
||
SendReturnMessage(
|
||
{
|
||
code: 0,
|
||
message: errorMsg,
|
||
id: task.bookTaskDetailId as string,
|
||
data: {
|
||
status: 'error',
|
||
message: errorMsg,
|
||
id: task.bookTaskDetailId
|
||
}
|
||
},
|
||
task.messageName as string
|
||
)
|
||
throw error
|
||
}
|
||
}
|
||
//#endregion
|
||
|
||
|
||
//#region 生成视频主流程
|
||
|
||
ComfyUIVideoGenerate = async (task: TaskModal.Task) => {
|
||
try {
|
||
await this.InitSDBasic()
|
||
let bookTaskDetail = await this.bookTaskDetailService.GetBookTaskDetailDataById(
|
||
task.bookTaskDetailId as string, true
|
||
)
|
||
|
||
let { comfyuiOptions, videoMessage } = await this.GetComfyuiOptions(bookTaskDetail)
|
||
|
||
let comfyuiSimpleSetting = await this.GetComfyUISetting()
|
||
|
||
let workflow_file = comfyuiOptions.workflow_file;
|
||
if (isEmpty(workflow_file)) {
|
||
workflow_file = comfyuiSimpleSetting.imageToVideoSelectWorkflow || "";
|
||
}
|
||
|
||
let prompt = videoMessage.prompt || comfyuiOptions.prompt || '';
|
||
let negativePrompt = comfyuiOptions.negative_prompt || '';
|
||
let firstFrameImage = videoMessage.imageUrl ?? comfyuiOptions.first_frame_image ?? "";
|
||
let lastFrameImage = comfyuiOptions.last_frame_image ?? "";
|
||
let resolution = comfyuiOptions.resolution ?? 768;
|
||
let duration = comfyuiOptions.duration ?? 5;
|
||
let fps = comfyuiOptions.fps ?? 24;
|
||
|
||
|
||
let workflow = await this.GetComfyUIWorkflow(workflow_file);
|
||
|
||
// 检查图片是否存在并且上传
|
||
let firstFrameImageName: string;
|
||
if (isEmpty(firstFrameImage)) {
|
||
throw new Error(t('ComfyUI首帧参考图像不能为空，请检查分镜的ComfyUI参数配置'))
|
||
} else {
|
||
if (!await CheckFileOrDirExist(firstFrameImage)) {
|
||
throw new Error(t('ComfyUI首帧参考图像不存在，请检查分镜的ComfyUI参数配置'))
|
||
}
|
||
|
||
firstFrameImageName = await this.ComfyUIUploadImage(firstFrameImage, comfyuiSimpleSetting);
|
||
}
|
||
|
||
let lastFrameImageName: string = "";
|
||
if (!isEmpty(lastFrameImage)) {
|
||
if (!await CheckFileOrDirExist(lastFrameImage)) {
|
||
throw new Error(t('ComfyUI尾帧参考图像文件不存在，请检查分镜的ComfyUI参数配置'))
|
||
}
|
||
lastFrameImageName = await this.ComfyUIUploadImage(lastFrameImage, comfyuiSimpleSetting);
|
||
}
|
||
|
||
// 开始处理body
|
||
let bodyString = await this.GetComfyUIVideoAPIBody({
|
||
prompt: prompt,
|
||
negativePrompt: negativePrompt,
|
||
workflowFilePath: workflow.workflowFilePath,
|
||
firstFrameImageName: firstFrameImageName,
|
||
lastFrameImageName: lastFrameImageName ?? "",
|
||
resolution: resolution,
|
||
duration: duration,
|
||
fps: fps
|
||
})
|
||
|
||
// 开始发送请求
|
||
let resData = await this.SubmitComfyUITask(bodyString, comfyuiSimpleSetting)
|
||
let taskId = resData.prompt_id;
|
||
|
||
// 修改Task, 将数据写入
|
||
this.taskListService.UpdateBackTaskData(task.id as string, {
|
||
taskId: taskId as string,
|
||
taskMessage: JSON.stringify(resData)
|
||
})
|
||
|
||
// 修改videoMessage
|
||
videoMessage.taskId = taskId
|
||
videoMessage.status = VideoStatus.SUBMITTED
|
||
videoMessage.messageData = JSON.stringify(resData)
|
||
videoMessage.msg = ''
|
||
delete videoMessage.imageUrl // 不要修改原本的图片地址
|
||
|
||
this.bookTaskDetailService.UpdateBookTaskDetailVideoMessage(
|
||
task.bookTaskDetailId as string,
|
||
videoMessage
|
||
)
|
||
|
||
// 添加任务成功 返回前端任务事件
|
||
SendReturnMessage(
|
||
{
|
||
code: 1,
|
||
id: task.bookTaskDetailId as string,
|
||
message: t('已成功提交{type}图转视频任务，任务ID：{taskId}', { type: t("ComfyUI"), taskId: taskId }),
|
||
type: ResponseMessageType.COMFYUI_VIDEO,
|
||
data: JSON.stringify(videoMessage)
|
||
},
|
||
task.messageName as string
|
||
)
|
||
|
||
await this.FetchComfyUIVideoTask(task, taskId, bookTaskDetail, comfyuiSimpleSetting)
|
||
return successMessage(
|
||
t('ComfyUI视频生成任务完成！'),
|
||
)
|
||
|
||
} catch (error) {
|
||
throw new Error(t('ComfyUI视频生成任务失败，失败信息：{error}', { error: (error as Error).message }));
|
||
}
|
||
}
|
||
|
||
//#endregion
|
||
|
||
//#region comfyUI上传图片
|
||
|
||
private async ComfyUIUploadImage(imagePath: string, comfyuiSimpleSetting: SettingModal.ComfyUISimpleSettingModel): Promise<string> {
|
||
var FormData = require('form-data');
|
||
var data = new FormData();
|
||
data.append('image', fs.createReadStream(imagePath));
|
||
data.append('type', 'input');
|
||
|
||
let url = comfyuiSimpleSetting.requestUrl?.replace(
|
||
'localhost',
|
||
'127.0.0.1'
|
||
)
|
||
if (url.endsWith('/')) {
|
||
url = url + 'api/upload/image'
|
||
} else {
|
||
url = url + '/api/upload/image'
|
||
}
|
||
|
||
var config = {
|
||
method: 'post',
|
||
url: url,
|
||
headers: {
|
||
'User-Agent': 'Apifox/1.0.0 (https://apifox.com)',
|
||
...data.getHeaders()
|
||
},
|
||
data: data
|
||
};
|
||
|
||
let res = await RetryWithBackoff(async () => await axios(config), 5, 2000);
|
||
|
||
let resData = res.data;
|
||
|
||
if (isEmpty(resData.name)) {
|
||
throw new Error(t('ComfyUI图片上传失败，未获取到图片名称，请检查ComfyUI设置是否正确'))
|
||
}
|
||
return resData.name;
|
||
}
|
||
|
||
//#endregion
|
||
|
||
//#region 获取ComfyuiOptions
|
||
|
||
private async GetComfyuiOptions(bookTaskDetail: Book.SelectBookTaskDetail)
|
||
: Promise<{ comfyuiOptions: BookTaskDetail.ComfyUIOptions, videoMessage: BookTaskDetail.VideoMessage }> {
|
||
let videoMessage = bookTaskDetail.videoMessage;
|
||
if (videoMessage == null || videoMessage == undefined) {
|
||
throw new Error(t('小说批次任务的分镜数据的转视频配置为空，请检查'))
|
||
}
|
||
|
||
let comfyuiOptions: BookTaskDetail.ComfyUIOptions;
|
||
let comfyuiOptionsString = videoMessage.comfyUIOptions as string;
|
||
|
||
if (!ValidateJson(comfyuiOptionsString)) {
|
||
throw new Error(t('当前分镜数据的ComfyUI图转视频参数为空或参数校验失败，请检查'))
|
||
}
|
||
comfyuiOptions = JSON.parse(comfyuiOptionsString) as BookTaskDetail.ComfyUIOptions;
|
||
|
||
return { comfyuiOptions, videoMessage };
|
||
}
|
||
|
||
//#endregion
|
||
|
||
//#region 获取设置
|
||
/**
|
||
* 获取ComfyUI的设置
|
||
* @returns
|
||
*/
|
||
private async GetComfyUISetting(): Promise<SettingModal.ComfyUISimpleSettingModel> {
|
||
let optionRealmService = await OptionRealmService.getInstance()
|
||
|
||
// 获取ComfyUI的基础设置
|
||
let comfyuiSimpleSettingOption = optionRealmService.GetOptionByKey(
|
||
OptionKeyName.SD.ComfyUISimpleSetting
|
||
)
|
||
let res = optionSerialization<SettingModal.ComfyUISimpleSettingModel>(
|
||
comfyuiSimpleSettingOption,
|
||
t("设置 -> ComfyUI 设置")
|
||
)
|
||
return res;
|
||
}
|
||
|
||
private async GetComfyUIWorkflow(workflowId: string): Promise<WorkflowModel.Workflow> {
|
||
if (isEmpty(workflowId)) {
|
||
throw new Error(t('未设置选中的工作流，请检查是否正确设置！！'))
|
||
}
|
||
|
||
// 查找工作流
|
||
let workflowRes: WorkflowModel.Workflow = this.workflowRealmService.GetWorkFlowById(workflowId as string, true);
|
||
|
||
// 判断工作流对应的文件是不是存在
|
||
if (!(await CheckFileOrDirExist(workflowRes.workflowFilePath))) {
|
||
throw new Error(t('本地未找到选中的工作流文件地址，请检查是否正确设置！！'))
|
||
}
|
||
return workflowRes
|
||
}
|
||
|
||
//#endregion
|
||
|
||
//#region 生成视频的请求体
|
||
|
||
private async GetComfyUIVideoAPIBody(params: SettingModal.ComfyUIVideoAPIBodyGenerateQuery): Promise<string> {
|
||
|
||
let jsonContentString = await fs.promises.readFile(params.workflowFilePath, 'utf-8')
|
||
if (!ValidateJson(jsonContentString)) {
|
||
throw new Error(t('工作流文件内容不是有效的JSON格式，请检查是否正确设置！！'))
|
||
}
|
||
let jsonContent = JSON.parse(jsonContentString)
|
||
// 判断是否是对象
|
||
if (jsonContent !== null && typeof jsonContent === 'object' && !Array.isArray(jsonContent)) {
|
||
// 遍历对象属性
|
||
for (const key in jsonContent) {
|
||
let element = jsonContent[key]
|
||
// 处理正向提示词和反向提示词
|
||
if (element && element.class_type === 'CLIPTextEncode') {
|
||
if (element._meta?.title === '正向提示词' || element._meta?.title === 'Positive Prompt') {
|
||
jsonContent[key].inputs.text = params.prompt
|
||
}
|
||
if (element._meta?.title === '反向提示词' || element._meta?.title === 'Negative Prompt') {
|
||
jsonContent[key].inputs.text = params.negativePrompt
|
||
}
|
||
}
|
||
|
||
// 处理首尾帧图像传入
|
||
if (element && element.class_type === 'LoadImage') {
|
||
if (element._meta?.title === '加载首帧图像' || element._meta?.title === 'Load First Frame Image') {
|
||
jsonContent[key].inputs.image = params.firstFrameImageName;
|
||
}
|
||
|
||
if (element._meta?.title === '加载尾帧图像' || element._meta?.title === 'Load Last Frame Image') {
|
||
jsonContent[key].inputs.image = params.lastFrameImageName;
|
||
}
|
||
}
|
||
|
||
// 处理视频时长，分辨率
|
||
if (element && element.class_type === 'Int') {
|
||
if (element._meta?.title === '视频时长' || element._meta?.title === 'Video Duration') {
|
||
jsonContent[key].inputs.Number = params.duration * 16 + 1;
|
||
}
|
||
if (element._meta?.title === '分辨率' || element._meta?.title === 'Resolution') {
|
||
jsonContent[key].inputs.Number = params.resolution.toString();
|
||
}
|
||
}
|
||
|
||
// 处理随机seed
|
||
if (element && element.class_type === 'KSampler') {
|
||
let seed = this.GenerateRandomSeed();
|
||
|
||
jsonContent[key].inputs.seed = seed;
|
||
} else if (element && element.class_type === 'KSamplerAdvanced') {
|
||
let seed = this.GenerateRandomSeed();
|
||
|
||
jsonContent[key].inputs.noise_seed = seed
|
||
}
|
||
}
|
||
} else {
|
||
throw new Error(t('工作流文件内容不是有效的JSON对象格式，请检查是否正确设置！！'))
|
||
}
|
||
|
||
let res = JSON.stringify({
|
||
prompt: jsonContent
|
||
})
|
||
return res
|
||
}
|
||
|
||
//#endregion
|
||
|
||
//#region 生成随机seed
|
||
|
||
private GenerateRandomSeed(): string {
|
||
const crypto = require('crypto')
|
||
const buffer = crypto.randomBytes(8)
|
||
let seed = BigInt('0x' + buffer.toString('hex'))
|
||
return seed.toString();
|
||
}
|
||
|
||
//#endregion
|
||
|
||
//#region 组合图片的请求体
|
||
/**
|
||
* 组合ComfyUI生成图片的的请求体
|
||
* @param prompt 正向提示词
|
||
* @param negativePrompt 反向提示词
|
||
* @param workflowPath 工作流地址
|
||
*/
|
||
private async GetComfyUIImageAPIBody(
|
||
prompt: string,
|
||
negativePrompt: string,
|
||
workflowPath: string
|
||
): Promise<string> {
|
||
let jsonContentString = await fs.promises.readFile(workflowPath, 'utf-8')
|
||
|
||
if (!ValidateJson(jsonContentString)) {
|
||
throw new Error(t('工作流文件内容不是有效的JSON格式，请检查是否正确设置！！'))
|
||
}
|
||
|
||
let jsonContent = JSON.parse(jsonContentString)
|
||
// 判断是否是对象
|
||
if (jsonContent !== null && typeof jsonContent === 'object' && !Array.isArray(jsonContent)) {
|
||
// 遍历对象属性
|
||
for (const key in jsonContent) {
|
||
let element = jsonContent[key]
|
||
if (element && element.class_type === 'CLIPTextEncode') {
|
||
if (element._meta?.title === '正向提示词' || element._meta?.title === 'Positive Prompt') {
|
||
jsonContent[key].inputs.text = prompt
|
||
}
|
||
if (element._meta?.title === '反向提示词' || element._meta?.title === 'Negative Prompt') {
|
||
jsonContent[key].inputs.text = negativePrompt
|
||
}
|
||
}
|
||
if (element && element.class_type === 'KSampler') {
|
||
let seed = this.GenerateRandomSeed();
|
||
|
||
jsonContent[key].inputs.seed = seed;
|
||
} else if (element && element.class_type === 'KSamplerAdvanced') {
|
||
let seed = this.GenerateRandomSeed();
|
||
|
||
jsonContent[key].inputs.noise_seed = seed
|
||
}
|
||
}
|
||
} else {
|
||
throw new Error(t('工作流文件内容不是有效的JSON对象格式，请检查是否正确设置！！'))
|
||
}
|
||
let result = JSON.stringify({
|
||
prompt: jsonContent
|
||
})
|
||
return result
|
||
}
|
||
|
||
//#endregion
|
||
|
||
//#region 提交执行任务
|
||
/**
|
||
* 将组合后的请求体提交给ComfyUI服务器，返回用于轮询的响应结构。
|
||
* @param body 序列化后的请求报文。
|
||
* @param comfyUISettingCollection ComfyUI 的配置集合。
|
||
*/
|
||
private async SubmitComfyUITask(
|
||
body: string,
|
||
comfyuiSimpleSetting: SettingModal.ComfyUISimpleSettingModel
|
||
): Promise<any> {
|
||
let url = comfyuiSimpleSetting.requestUrl?.replace(
|
||
'localhost',
|
||
'127.0.0.1'
|
||
)
|
||
if (url.endsWith('/')) {
|
||
url = url + 'api/prompt'
|
||
} else {
|
||
url = url + '/api/prompt'
|
||
}
|
||
var config = {
|
||
method: 'post',
|
||
url: url,
|
||
headers: {
|
||
'User-Agent': 'Apifox/1.0.0 (https://apifox.com)',
|
||
'Content-Type': 'application/json'
|
||
},
|
||
data: body
|
||
}
|
||
|
||
let res = await RetryWithBackoff(async () => await axios(config), 5, 2000);
|
||
let resData = res.data
|
||
// 判断是不是失败
|
||
if (resData.error) {
|
||
let errorNode = ''
|
||
if (resData.node_errors) {
|
||
for (const key in resData.node_errors) {
|
||
errorNode += key + ', '
|
||
}
|
||
}
|
||
let msg = t("错误信息：{error}，错误节点：{node}", {
|
||
error: resData.error.message,
|
||
node: errorNode
|
||
})
|
||
throw new Error(msg)
|
||
}
|
||
// 没有错误 判断是不是成功
|
||
if (resData.prompt_id && !isEmpty(resData.prompt_id)) {
|
||
// 成功
|
||
return resData
|
||
} else {
|
||
throw new Error(t('未知错误，未获取到请求ID，请检查是否正确设置！！'))
|
||
}
|
||
}
|
||
|
||
//#endregion
|
||
|
||
//#region 循环获取视频任务
|
||
|
||
async FetchComfyUIVideoTask(
|
||
task: TaskModal.Task,
|
||
promptId: string,
|
||
bookTaskDetail: Book.SelectBookTaskDetail,
|
||
comfyuiSimpleSetting: SettingModal.ComfyUISimpleSettingModel) {
|
||
|
||
while (true) {
|
||
try {
|
||
let resData = await this.GetComfyUITaskOnce(promptId, comfyuiSimpleSetting, ComfyUIWorkflowType.IMAGE_TO_VIDEO);
|
||
|
||
// 判断他的状态是不是成功
|
||
if (resData.status == 'error') {
|
||
// 修改小说分镜的 videoMessage
|
||
let videoMessage = cloneDeep(bookTaskDetail.videoMessage) ?? {}
|
||
|
||
videoMessage.status = VideoStatus.FAIL
|
||
videoMessage.msg = resData.message
|
||
videoMessage.taskId = promptId
|
||
videoMessage.messageData = JSON.stringify(resData)
|
||
delete videoMessage.imageUrl
|
||
|
||
// 修改 videoMessage数据
|
||
this.bookTaskDetailService.UpdateBookTaskDetailVideoMessage(
|
||
bookTaskDetail.id as string,
|
||
videoMessage
|
||
)
|
||
|
||
// 修改TASK
|
||
this.taskListService.UpdateBackTaskData(task.id as string, {
|
||
taskId: promptId,
|
||
taskMessage: JSON.stringify(resData)
|
||
})
|
||
|
||
// 返回前端数据
|
||
SendReturnMessage(
|
||
{
|
||
code: 0,
|
||
id: bookTaskDetail.id as string,
|
||
message: t("ComfyUI视频任务失败，失败信息：{error}", {
|
||
error: resData.message
|
||
}),
|
||
type: ResponseMessageType.COMFYUI_VIDEO,
|
||
data: JSON.stringify(videoMessage)
|
||
},
|
||
task.messageName as string
|
||
)
|
||
throw new Error(resData.message)
|
||
} else if (resData.status == 'in_progress') {
|
||
// 任务执行中或者是提交成功
|
||
let videoMessage = cloneDeep(bookTaskDetail.videoMessage) ?? {}
|
||
videoMessage.status = VideoStatus.PROCESSING
|
||
videoMessage.taskId = promptId;
|
||
videoMessage.messageData = JSON.stringify(resData)
|
||
delete videoMessage.imageUrl
|
||
this.bookTaskDetailService.UpdateBookTaskDetailVideoMessage(
|
||
task.bookTaskDetailId as string,
|
||
videoMessage
|
||
)
|
||
|
||
SendReturnMessage(
|
||
{
|
||
code: 1,
|
||
id: bookTaskDetail.id as string,
|
||
message: t('ComfyUI视频任务正在执行中...'),
|
||
type: ResponseMessageType.COMFYUI_VIDEO,
|
||
data: JSON.stringify(videoMessage)
|
||
},
|
||
task.messageName as string
|
||
)
|
||
await new Promise((resolve) => setTimeout(resolve, 20000))
|
||
}
|
||
else {
|
||
// 任务成功 修改 videoMessage
|
||
let videoMessage = cloneDeep(bookTaskDetail.videoMessage) ?? {}
|
||
videoMessage.status = VideoStatus.SUCCESS
|
||
videoMessage.taskId = promptId
|
||
|
||
videoMessage.messageData = JSON.stringify(resData)
|
||
delete videoMessage.imageUrl
|
||
|
||
this.bookTaskDetailService.UpdateBookTaskDetailVideoMessage(
|
||
task.bookTaskDetailId as string,
|
||
videoMessage
|
||
)
|
||
|
||
// 修改小说分镜状态
|
||
this.bookTaskDetailService.ModifyBookTaskDetailById(task.bookTaskDetailId as string, {
|
||
status: BookTaskStatus.IMAGE_TO_VIDEO_SUCCESS
|
||
})
|
||
|
||
// 修改任务状态
|
||
this.taskListService.UpdateBackTaskData(task.id as string, {
|
||
status: BookBackTaskStatus.DONE,
|
||
taskId: promptId,
|
||
taskMessage: JSON.stringify(resData)
|
||
})
|
||
|
||
let videoUrls: string[] = [];
|
||
videoMessage.videoUrls = [];
|
||
for (let i = 0; resData.fileInfo && i < resData.fileInfo.length; i++) {
|
||
const element = resData.fileInfo[i];
|
||
if (!isEmpty(element.filename)) {
|
||
let videoUrl = this.GetComfyUIVideoUrl(element, comfyuiSimpleSetting);
|
||
videoUrls.push(videoUrl);
|
||
videoMessage.videoUrls.push(videoUrl);
|
||
}
|
||
}
|
||
|
||
await this.DownloadVideoUrls(videoMessage.videoUrls || [], task, bookTaskDetail, promptId);
|
||
SendReturnMessage(
|
||
{
|
||
code: 1,
|
||
id: bookTaskDetail.id as string,
|
||
message: t('ComfyUI视频任务已完成！'),
|
||
type: ResponseMessageType.COMFYUI_VIDEO,
|
||
data: JSON.stringify(videoMessage)
|
||
},
|
||
task.messageName as string
|
||
)
|
||
break;
|
||
}
|
||
|
||
} catch (error) {
|
||
throw error;
|
||
}
|
||
}
|
||
}
|
||
|
||
//#endregion
|
||
|
||
//#region 拼接视频地址
|
||
|
||
private GetComfyUIVideoUrl(fileInfo: any, comfyuiSimpleSetting: SettingModal.ComfyUISimpleSettingModel): string {
|
||
let url = comfyuiSimpleSetting.requestUrl?.replace(
|
||
'localhost',
|
||
'127.0.0.1'
|
||
)
|
||
if (url.endsWith('/')) {
|
||
url = url + 'api/view'
|
||
} else {
|
||
url = url + '/api/view'
|
||
}
|
||
|
||
const queryParams = new URLSearchParams();
|
||
|
||
for (const [key, value] of Object.entries(fileInfo)) {
|
||
if (value !== null && value !== undefined) {
|
||
queryParams.append(key, String(value));
|
||
}
|
||
}
|
||
|
||
const queryString = queryParams.toString();
|
||
if (isEmpty(queryString)) {
|
||
throw new Error(t('ComfyUI视频文件信息不完整，无法获取视频地址，请检查'))
|
||
}
|
||
let videoUrl = `${url}?${queryString}`;
|
||
return videoUrl;
|
||
}
|
||
|
||
//#endregion
|
||
|
||
//#region 循环获取出图任务
|
||
/**
|
||
* 循环拉取任务状态并在完成后处理文件保存与状态回写。
|
||
* @param task 当前任务实体。
|
||
* @param promptId ComfyUI返回的任务ID。
|
||
* @param book 作品信息。
|
||
* @param bookTask 作品任务信息。
|
||
* @param bookTaskDetail 任务明细信息。
|
||
* @param comfyUISettingCollection ComfyUI 配置集合。
|
||
*/
|
||
async FetchComfyUIImageTask(
|
||
task: TaskModal.Task,
|
||
promptId: string,
|
||
book: Book.SelectBook,
|
||
bookTask: Book.SelectBookTask,
|
||
bookTaskDetail: Book.SelectBookTaskDetail,
|
||
comfyuiSimpleSetting: SettingModal.ComfyUISimpleSettingModel
|
||
) {
|
||
while (true) {
|
||
try {
|
||
let resData = await this.GetComfyUITaskOnce(promptId, comfyuiSimpleSetting, ComfyUIWorkflowType.IMAGE)
|
||
|
||
// 判断他的状态是不是成功
|
||
if (resData.status == 'error') {
|
||
// 生图失败
|
||
await this.bookTaskDetailService.ModifyBookTaskDetailById(
|
||
task.bookTaskDetailId as string,
|
||
{
|
||
status: BookTaskStatus.IMAGE_FAIL
|
||
}
|
||
)
|
||
let errorMsg = t("ComfyUI生图失败，{error}", {
|
||
error: resData.message
|
||
})
|
||
this.bookTaskDetailService.UpdateBookTaskDetailMjMessage(
|
||
task.bookTaskDetailId as string,
|
||
{
|
||
mjApiUrl: comfyuiSimpleSetting.requestUrl,
|
||
progress: 100,
|
||
category: ImageGenerateMode.ComfyUI,
|
||
imageClick: '',
|
||
imageShow: '',
|
||
messageId: promptId,
|
||
action: MJAction.IMAGINE,
|
||
status: 'error',
|
||
message: errorMsg
|
||
}
|
||
)
|
||
this.taskListService.UpdateTaskStatus({
|
||
id: task.id as string,
|
||
status: BookBackTaskStatus.FAIL,
|
||
errorMessage: errorMsg
|
||
})
|
||
SendReturnMessage(
|
||
{
|
||
code: 0,
|
||
message: errorMsg,
|
||
id: task.bookTaskDetailId as string,
|
||
data: {
|
||
status: 'error',
|
||
message: errorMsg,
|
||
id: task.bookTaskDetailId
|
||
}
|
||
},
|
||
task.messageName as string
|
||
)
|
||
return
|
||
} else if (resData.status == 'in_progress') {
|
||
// 生图中
|
||
this.bookTaskDetailService.UpdateBookTaskDetailMjMessage(
|
||
task.bookTaskDetailId as string,
|
||
{
|
||
mjApiUrl: comfyuiSimpleSetting.requestUrl,
|
||
progress: 0,
|
||
category: ImageGenerateMode.ComfyUI,
|
||
imageClick: '',
|
||
imageShow: '',
|
||
messageId: promptId,
|
||
action: MJAction.IMAGINE,
|
||
status: 'running',
|
||
message: t('任务正在执行中')
|
||
}
|
||
)
|
||
|
||
SendReturnMessage(
|
||
{
|
||
code: 1,
|
||
message: 'running',
|
||
id: task.bookTaskDetailId as string,
|
||
data: {
|
||
status: 'running',
|
||
message: t('任务正在执行中'),
|
||
id: task.bookTaskDetailId
|
||
}
|
||
},
|
||
task.messageName as string
|
||
)
|
||
} else {
|
||
let res = await this.DownloadFileUrl(
|
||
resData.fileInfo.filter((item: any) => !isEmpty(item.filename)).map((item: any) => item.filename),
|
||
comfyuiSimpleSetting,
|
||
book,
|
||
bookTask,
|
||
bookTaskDetail
|
||
)
|
||
let projectPath = await getProjectPath()
|
||
// 修改数据库数据
|
||
// 修改数据库
|
||
await this.bookTaskDetailService.ModifyBookTaskDetailById(bookTaskDetail.id as string, {
|
||
outImagePath: path.relative(projectPath, res.outImagePath),
|
||
subImagePath: res.subImagePath.map((item) => path.relative(projectPath, item))
|
||
})
|
||
|
||
this.taskListService.UpdateTaskStatus({
|
||
id: task.id as string,
|
||
status: BookBackTaskStatus.DONE
|
||
})
|
||
|
||
this.bookTaskDetailService.UpdateBookTaskDetailMjMessage(
|
||
task.bookTaskDetailId as string,
|
||
{
|
||
mjApiUrl: comfyuiSimpleSetting.requestUrl,
|
||
progress: 100,
|
||
category: ImageGenerateMode.ComfyUI,
|
||
imageClick: '',
|
||
imageShow: '',
|
||
messageId: promptId,
|
||
action: MJAction.IMAGINE,
|
||
status: 'success',
|
||
message: t("ComfyUI 生成图片成功！")
|
||
}
|
||
)
|
||
|
||
SendReturnMessage(
|
||
{
|
||
code: 1,
|
||
message: t("ComfyUI 生成图片成功！"),
|
||
id: task.bookTaskDetailId as string,
|
||
data: {
|
||
status: 'success',
|
||
message: t("ComfyUI 生成图片成功！"),
|
||
id: task.bookTaskDetailId,
|
||
outImagePath: res.outImagePath + '?t=' + new Date().getTime(),
|
||
subImagePath: res.subImagePath.map((item) => item + '?t=' + new Date().getTime())
|
||
}
|
||
},
|
||
task.messageName as string
|
||
)
|
||
break
|
||
}
|
||
await new Promise((resolve) => setTimeout(resolve, 3000))
|
||
} catch (error) {
|
||
throw error
|
||
}
|
||
}
|
||
}
|
||
|
||
//#endregion
|
||
|
||
//#region 单次获取任务
|
||
|
||
/**
|
||
* 获取ComfyUI任务
|
||
* @param promptId
|
||
* @param comfyUISettingCollection
|
||
*/
|
||
private async GetComfyUITaskOnce(
|
||
promptId: string,
|
||
comfyuiSimpleSetting: SettingModal.ComfyUISimpleSettingModel,
|
||
type: ComfyUIWorkflowType
|
||
): Promise<any> {
|
||
if (isEmpty(promptId)) {
|
||
throw new Error(t("未获取到请求ID，请检查是否正确设置！！"))
|
||
}
|
||
if (isEmpty(comfyuiSimpleSetting.requestUrl)) {
|
||
throw new Error(t('未获取到ComfyUI的请求地址，请检查是否正确设置！！'))
|
||
}
|
||
|
||
let url = comfyuiSimpleSetting.requestUrl?.replace(
|
||
'localhost',
|
||
'127.0.0.1'
|
||
)
|
||
if (url.endsWith('/')) {
|
||
url = url + 'api/history'
|
||
} else {
|
||
url = url + '/api/history'
|
||
}
|
||
|
||
var config = {
|
||
method: 'get',
|
||
url: `${url}/${promptId}`,
|
||
headers: {
|
||
'User-Agent': 'Apifox/1.0.0 (https://apifox.com)'
|
||
}
|
||
}
|
||
|
||
let res = await RetryWithBackoff(async () => await axios.request(config), 5, 2000);
|
||
|
||
let resData = res.data
|
||
// 判断状态是失败还是成功
|
||
let data = resData[promptId]
|
||
if (data == null) {
|
||
// 还在执行中 或者是任务不存在
|
||
return {
|
||
progress: 0,
|
||
status: 'in_progress',
|
||
message: t('任务正在执行中')
|
||
}
|
||
}
|
||
let completed = data.status?.completed
|
||
let outputs = data.outputs
|
||
if (completed && outputs) {
|
||
let fileInfo: string[] = []
|
||
for (const key in outputs) {
|
||
let outputNode = outputs[key]
|
||
|
||
if (type == ComfyUIWorkflowType.IMAGE) {
|
||
if (outputNode && outputNode?.images && outputNode?.images.length > 0) {
|
||
for (let i = 0; i < outputNode?.images.length; i++) {
|
||
const element = outputNode?.images[i]
|
||
if (!isEmpty(element.filename)) {
|
||
fileInfo.push(element)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if (type == ComfyUIWorkflowType.IMAGE_TO_VIDEO) {
|
||
if (outputNode && outputNode?.gifs && outputNode?.gifs.length > 0) {
|
||
for (let i = 0; i < outputNode?.gifs.length; i++) {
|
||
const element = outputNode?.gifs[i]
|
||
if (!isEmpty(element.filename)) {
|
||
fileInfo.push(element)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if (fileInfo.length == 0) {
|
||
return {
|
||
progress: 0,
|
||
status: 'error',
|
||
message: t('未找到返回的文件信息')
|
||
}
|
||
}
|
||
|
||
return {
|
||
progress: 100,
|
||
status: 'success',
|
||
fileInfo: fileInfo
|
||
}
|
||
} else {
|
||
return {
|
||
progress: 0,
|
||
status: 'error',
|
||
message: t('详细失败信息看启动器控制台')
|
||
}
|
||
}
|
||
}
|
||
|
||
//#endregion
|
||
|
||
//#region 请求下载图片
|
||
|
||
/**
|
||
* 请求下载对应的图片
|
||
* @param url
|
||
* @param path
|
||
*/
|
||
private async DownloadFileUrl(
|
||
imageNames: string[],
|
||
comfyuiSimpleSetting: SettingModal.ComfyUISimpleSettingModel,
|
||
book: Book.SelectBook,
|
||
bookTask: Book.SelectBookTask,
|
||
bookTaskDetail: Book.SelectBookTaskDetail
|
||
): Promise<{
|
||
outImagePath: string
|
||
subImagePath: string[]
|
||
}> {
|
||
let url = comfyuiSimpleSetting.requestUrl?.replace(
|
||
'localhost',
|
||
'127.0.0.1'
|
||
)
|
||
if (url.endsWith('/')) {
|
||
url = url + 'api/view'
|
||
} else {
|
||
url = url + '/api/view'
|
||
}
|
||
let outImagePath = ''
|
||
let subImagePath: string[] = []
|
||
for (let i = 0; i < imageNames.length; i++) {
|
||
const imageName = imageNames[i]
|
||
|
||
var config = {
|
||
method: 'get',
|
||
url: `${url}?filename=${imageName}&nocache=${Date.now()}`,
|
||
headers: {
|
||
'User-Agent': 'Apifox/1.0.0 (https://apifox.com)'
|
||
},
|
||
responseType: 'arraybuffer' as 'arraybuffer' // 明确指定类型
|
||
}
|
||
|
||
let res = await RetryWithBackoff(async () => axios.request(config), 5, 2000);
|
||
|
||
// 检查响应状态和类型
|
||
console.log(`图片下载状态: ${res.status}, 内容类型: ${res.headers['content-type']}`)
|
||
|
||
// 确保得到的是图片数据
|
||
if (!res.headers['content-type']?.includes('image/')) {
|
||
console.error(`响应不是图片: ${res.headers['content-type']}`)
|
||
continue
|
||
}
|
||
|
||
let resData = res.data
|
||
console.log(resData)
|
||
|
||
let subImageFolderPath = path.join(
|
||
bookTask.imageFolder as string,
|
||
`subImage/${bookTaskDetail.name}`
|
||
)
|
||
await CheckFolderExistsOrCreate(subImageFolderPath)
|
||
let outputFolder = bookTask.imageFolder as string
|
||
await CheckFolderExistsOrCreate(outputFolder)
|
||
let inputFolder = path.join(book.bookFolderPath as string, 'tmp/input')
|
||
await CheckFolderExistsOrCreate(inputFolder)
|
||
|
||
// 包含info信息的图片地址
|
||
let infoImgPath = path.join(subImageFolderPath, `${new Date().getTime()}_${i}.png`)
|
||
|
||
// 直接将二进制数据写入文件
|
||
await fs.promises.writeFile(infoImgPath, Buffer.from(resData))
|
||
|
||
if (i == 0) {
|
||
// 复制到对应的文件夹里面
|
||
let outPath = path.join(outputFolder, `${bookTaskDetail.name}.png`)
|
||
await CopyFileOrFolder(infoImgPath, outPath)
|
||
outImagePath = outPath
|
||
}
|
||
subImagePath.push(infoImgPath as string)
|
||
}
|
||
console.log(outImagePath)
|
||
console.log(subImagePath)
|
||
|
||
// 将获取的数据返回
|
||
return {
|
||
outImagePath: outImagePath,
|
||
subImagePath: subImagePath
|
||
}
|
||
}
|
||
|
||
//#endregion
|
||
}
|