lq1405
224ee47984
2. 重构软件鉴权，添加自动判断机器码，可以不在手动点击确定 3. 新增软件启动时，提示用户是不是还存在等待中的任务，还有的话，由用户判断是不是丢弃，丢弃，会将所有的等待中的任务设置为失败 4. 聚合推文 提示词合并，人物、风格和场景 添加判无，修复部分情况无法合并 5. 新增 预设（人物，场景，风格）对提示词相关的提示 6. 修复 文案处理，流式输出内容重复问题，删除豆包（请求格式不兼容） 7. 新增 文案处理 输出内容一键格式化（注意：由于GPT输出的格式化有太多的不确定，不一定可以完全格式化，需要手动检查） 8. 聚合推文原创，新增时默认设置 修脸参数，初始和SD设置中的一致 9. 修复聚合推文选图覆盖 bug
671 lines
25 KiB
TypeScript
671 lines
25 KiB
TypeScript
import { isEmpty } from 'lodash'
|
||
import { errorMessage, successMessage } from '../../Public/generalTools'
|
||
import { FfmpegOptions } from '../ffmpegOptions'
|
||
import { SubtitleSavePositionType } from '../../../define/enum/waterMarkAndSubtitle'
|
||
import { define } from '../../../define/define'
|
||
import path from 'path'
|
||
import {
|
||
CheckFileOrDirExist,
|
||
CheckFolderExistsOrCreate,
|
||
DeleteFolderAllFile,
|
||
GetFilesWithExtensions
|
||
} from '../../../define/Tools/file'
|
||
import { shell } from 'electron'
|
||
import { Book } from '../../../model/book'
|
||
import fs from 'fs'
|
||
import { GeneralResponse } from '../../../model/generalResponse'
|
||
import { BookServiceBasic } from '../ServiceBasic/bookServiceBasic'
|
||
import { LoggerStatus, OtherData, ResponseMessageType } from '../../../define/enum/softwareEnum'
|
||
import { TaskScheduler } from '../task/taskScheduler'
|
||
import { SubtitleModel } from '../../../model/subtitle'
|
||
import { BookTaskStatus, OperateBookType } from '../../../define/enum/bookEnum'
|
||
import axios from 'axios'
|
||
import { GptService } from '../GPT/gpt'
|
||
import FormData from 'form-data'
|
||
import { RetryWithBackoff } from '../../../define/Tools/common'
|
||
import { DEFINE_STRING } from '../../../define/define_string'
|
||
const util = require('util')
|
||
const { exec } = require('child_process')
|
||
const execAsync = util.promisify(exec)
|
||
const fspromises = fs.promises
|
||
|
||
|
||
/**
|
||
* 去除水印和获取字幕相关操作
|
||
*/
|
||
export class Subtitle {
|
||
ffmpegOptions: FfmpegOptions
|
||
bookServiceBasic: BookServiceBasic
|
||
taskScheduler: TaskScheduler
|
||
gptService: GptService
|
||
|
||
constructor() {
|
||
this.bookServiceBasic = new BookServiceBasic()
|
||
this.taskScheduler = new TaskScheduler()
|
||
this.ffmpegOptions = new FfmpegOptions()
|
||
this.gptService = new GptService()
|
||
}
|
||
|
||
//#region 通用方法
|
||
|
||
/**
|
||
* 拆分视频总帧数，每秒多少帧，平分视频总帧数，后截取
|
||
* @param {*} videoDurationMs 视频的总时长（毫秒）
|
||
* @param {*} framesPerSecond 每秒截取多少帧
|
||
* @returns
|
||
*/
|
||
GenerateFrameTimes(videoDurationMs: number, framesPerSecond: number): number[] {
|
||
// 直接使用视频总时长（毫秒），不进行向下取整
|
||
const videoDurationSec = videoDurationMs / 1000
|
||
|
||
// 计算总共需要抽取的帧数，考虑到视频时长可能不是完整秒数，使用 Math.ceil 来确保至少获取到最后一秒内的帧
|
||
const totalFrames = Math.ceil(videoDurationSec * framesPerSecond)
|
||
|
||
// 计算两帧之间的时间间隔（毫秒）
|
||
const interval = 1000 / framesPerSecond
|
||
|
||
// 生成对应的时间点数组
|
||
const frameTimes = []
|
||
for (let i = 0; i < totalFrames; i++) {
|
||
// 使用 Math.min 确保最后一个时间点不会超过视频总时长
|
||
let timePoint = Math.min(Math.round(interval * i), videoDurationMs)
|
||
frameTimes.push(timePoint)
|
||
}
|
||
return frameTimes
|
||
}
|
||
|
||
/**
|
||
* 加载指定的的小说相关的所有的数据
|
||
* @param bookId 小说ID
|
||
* @param bookTaskId 小说任务ID
|
||
* @returns
|
||
*/
|
||
async GetBookAllData(bookId: string, bookTaskId: string = null): Promise<{ book: Book.SelectBook, bookTask: Book.SelectBookTask, bookTaskDetails: Book.SelectBookTaskDetail[] }> {
|
||
let { book, bookTask } = await this.bookServiceBasic.GetBookAndTask(bookId, bookTaskId ? bookTaskId : 'output_00001')
|
||
if (isEmpty(book.subtitlePosition)) {
|
||
throw new Error("请先设置小说的字幕位置")
|
||
}
|
||
// 获取所有的分镜数据
|
||
let bookTaskDetails = await this.bookServiceBasic.GetBookTaskDetailData({
|
||
bookId: bookId,
|
||
bookTaskId: bookTask.id
|
||
})
|
||
if (bookTaskDetails.length <= 0) {
|
||
throw new Error("没有找到对应的分镜数据，请先执行对应的操作")
|
||
}
|
||
return { book, bookTask, bookTaskDetails }
|
||
}
|
||
|
||
/**
|
||
* 通用的小说获取分案的返回方法
|
||
* @param content 获取的文案内容
|
||
* @param book 小说实体类
|
||
* @param bookTask 小说任务实体类
|
||
* @param bookTaskDetail 小说任务分镜实体类
|
||
*/
|
||
async GetSubtitleLoggerAndResponse(content: string, progress: GeneralResponse.ProgressResponse, book: Book.SelectBook, bookTask: Book.SelectBookTask, bookTaskDetail: Book.SelectBookTaskDetail): Promise<void> {
|
||
// 修改数据
|
||
await this.bookServiceBasic.UpdateBookTaskDetail(bookTaskDetail.id, {
|
||
word: content,
|
||
afterGpt: content
|
||
});
|
||
|
||
// let res = await this.basicReverse.GetCopywritingFunc(book, item);
|
||
// 将当前的数据实时返回，前端进行修改
|
||
this.bookServiceBasic.sendReturnMessage({
|
||
code: 1,
|
||
id: bookTaskDetail.id,
|
||
type: ResponseMessageType.GET_TEXT,
|
||
data: {
|
||
content: content,
|
||
progress: progress
|
||
} as GeneralResponse.SubtitleProgressResponse // 返回识别到的文案
|
||
}, DEFINE_STRING.BOOK.GET_COPYWRITING_RETURN)
|
||
|
||
// 添加日志
|
||
await this.taskScheduler.AddLogToDB(
|
||
book.id,
|
||
book.type,
|
||
`${bookTaskDetail.name} 识别文案成功`,
|
||
bookTask.id,
|
||
LoggerStatus.SUCCESS
|
||
)
|
||
}
|
||
|
||
//#endregion
|
||
|
||
//#region 获取字幕位置信息相关操作
|
||
|
||
/**
|
||
* 获取当前帧的文字信息
|
||
* @param {*} value 需要的参数的对象，必须包含以下参数
|
||
* @param {*} value.id 小说ID/小说分镜详细信息ID/null
|
||
* @param {*} value.type 保存的类型（主视频/分镜视频/后续会添加外部单独的视频提取）
|
||
*/
|
||
async GetCurrentFrameText(value: { id: any; type?: SubtitleSavePositionType; imageFolder: any }): Promise<GeneralResponse.ErrorItem | GeneralResponse.SuccessItem> {
|
||
try {
|
||
let iamgePaths = []
|
||
let imageFolder = value.imageFolder
|
||
? value.imageFolder
|
||
: path.join(define.project_path, `${value.id}/data/subtitle/${value.id}`)
|
||
let imageFolderIsExist = await CheckFileOrDirExist(imageFolder)
|
||
if (!imageFolderIsExist) {
|
||
throw new Error('请先保存位置信息')
|
||
}
|
||
|
||
let images = await GetFilesWithExtensions(imageFolder, ['.png'])
|
||
let regex = /.*frame_.*\.png$/
|
||
images.forEach((element) => {
|
||
// 使用正则表达式测试文件名
|
||
if (regex.test(element)) {
|
||
iamgePaths.push(element)
|
||
}
|
||
})
|
||
|
||
// 开始识别
|
||
for (let i = 0; i < iamgePaths.length; i++) {
|
||
const imagePath = iamgePaths[i]
|
||
let scriptPath = path.join(define.scripts_path, 'LaiOcr/LaiOcr.exe')
|
||
let script = `cd "${path.dirname(scriptPath)}" && "${scriptPath}" "${imagePath}"`
|
||
let scriptRes = await execAsync(script, { maxBuffer: 1024 * 1024 * 10, encoding: 'utf-8' })
|
||
console.log(scriptRes)
|
||
if (scriptRes.error) {
|
||
throw new Error(scriptRes.error)
|
||
}
|
||
}
|
||
|
||
// 处理所有的图片完毕，遍历所有的数据返回
|
||
let textData = []
|
||
let jsonPath = await GetFilesWithExtensions(imageFolder, ['.json'])
|
||
for (let i = 0; i < jsonPath.length; i++) {
|
||
const element = jsonPath[i]
|
||
// 开始拼接
|
||
let texts = JSON.parse(await fspromises.readFile(element, 'utf-8'))
|
||
for (let j = 0; j < texts.length; j++) {
|
||
const text = texts[j][1][0]
|
||
textData.includes(text) ? null : textData.push(text)
|
||
}
|
||
}
|
||
return successMessage(
|
||
textData.join('\n'),
|
||
'获取当前帧的文字信息成功',
|
||
'WatermarkAndSubtitle_GetCurrentFrameText'
|
||
)
|
||
} catch (error) {
|
||
return errorMessage(
|
||
'获取当前帧的文字信息失败，错误消息如下：' + error.toString(),
|
||
'WatermarkAndSubtitle_GetCurrentFrameText'
|
||
)
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 打开对应的ID的字幕提取的图片文件夹
|
||
* @param {*} value 需要的参数的对象，必须包含以下参数
|
||
* @param {*} value.id 小说ID/小说分镜详细信息ID/null
|
||
* @param {*} value.type 保存的类型（主视频/分镜视频/后续会添加外部单独的视频提取）
|
||
*/
|
||
async OpenBookSubtitlePositionScreenshot(value: { type: SubtitleSavePositionType; id: any }) {
|
||
try {
|
||
let folder
|
||
if (
|
||
value.type == SubtitleSavePositionType.MAIN_VIDEO ||
|
||
value.type == SubtitleSavePositionType.SETTING
|
||
) {
|
||
folder = path.join(define.project_path, `${value.id}/data/subtitle/${value.id}`)
|
||
} else if (value.type == SubtitleSavePositionType.STORYBOARD_VIDEO) {
|
||
folder = path.join(define.project_path, `${value.id}/data/subtitle/${value.id}`)
|
||
}
|
||
|
||
// 判断文件夹是不是存在
|
||
let folderIsExist = await CheckFileOrDirExist(folder)
|
||
if (!folderIsExist) {
|
||
throw new Error('文件夹不存在，请先保存字幕位置信息')
|
||
}
|
||
|
||
// 打开文件夹
|
||
shell.openPath(folder)
|
||
return successMessage(
|
||
null,
|
||
'打开对应的文件夹成功',
|
||
'WatermarkAndSubtitle_OpenBookSubtitlePositionScreenshot'
|
||
)
|
||
} catch (error) {
|
||
return errorMessage(
|
||
'打开字幕位置信息失败，错误消息如下：' + error.toString(),
|
||
'WatermarkAndSubtitle_OpenBookSubtitlePositionScreenshot'
|
||
)
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 保存反推的视频的文案位置信息（可以保存多个）
|
||
* @param {*} value 需要的参数的对象，必须包含以下参数
|
||
* @param {*} value.id 小说ID/小说分镜详细信息ID/null
|
||
* @param {*} value.bookSubtitlePosition 小说文案对应的位置
|
||
* @param {*} value.currentTime 视频当前保存的时间
|
||
* @param {*} value.type 保存的类型（主视频/分镜视频/后续会添加外部单独的视频提取）
|
||
* @returns
|
||
*/
|
||
async SaveBookSubtitlePosition(value: { type: SubtitleSavePositionType; id: string; bookSubtitlePosition: string | any[]; currentTime: number }) {
|
||
try {
|
||
let saveData = []
|
||
let videoPath
|
||
let outImagePath
|
||
// 小说视频保存
|
||
this.ffmpegOptions = new FfmpegOptions()
|
||
if (
|
||
value.type == SubtitleSavePositionType.MAIN_VIDEO ||
|
||
value.type == SubtitleSavePositionType.SETTING
|
||
) {
|
||
if (value.id == null) {
|
||
throw new Error('小说ID不能为空')
|
||
}
|
||
// 获取指定的小说
|
||
let bookRes = await this.bookServiceBasic.GetBookDataById(value.id)
|
||
if (bookRes == null) {
|
||
throw new Error('没有找到小说信息')
|
||
}
|
||
let book = bookRes
|
||
|
||
if (value.bookSubtitlePosition.length <= 0) {
|
||
throw new Error('没有获取到字幕信息')
|
||
}
|
||
|
||
videoPath = book.oldVideoPath
|
||
if (isEmpty(videoPath)) {
|
||
throw new Error('没有获取到视频路径')
|
||
}
|
||
outImagePath = path.join(book.bookFolderPath, `data/subtitle/${book.id}/frame.png`)
|
||
|
||
// 获取视频的宽高数据
|
||
let videoSizeRes = await this.ffmpegOptions.FfmpegGetVideoSize(videoPath)
|
||
if (videoSizeRes.code == 0) {
|
||
throw new Error(videoSizeRes.message)
|
||
}
|
||
let videoSize = videoSizeRes.data
|
||
|
||
// 开始计算比例
|
||
let videoWidth = videoSize.width
|
||
let videoHeight = videoSize.height
|
||
for (let i = 0; i < value.bookSubtitlePosition.length; i++) {
|
||
const element = value.bookSubtitlePosition[i]
|
||
let widthRate = videoWidth / element.videoWidth // 宽度比例
|
||
let heightRate = videoHeight / element.videoHeight // 高度比例
|
||
// 计算比例
|
||
let newStartX = widthRate * element.startX
|
||
let newStartY = heightRate * element.startY
|
||
let newWidth = widthRate * element.width
|
||
let newHeight = heightRate * element.height
|
||
saveData.push({
|
||
startX: newStartX,
|
||
startY: newStartY,
|
||
width: newWidth,
|
||
height: newHeight,
|
||
videoWidth: videoWidth,
|
||
videoHeight: videoHeight
|
||
})
|
||
}
|
||
|
||
// 数据保存
|
||
let saveRes = await this.bookServiceBasic.UpdateBookData(value.id, {
|
||
subtitlePosition: JSON.stringify(saveData)
|
||
})
|
||
} else if (value.type == SubtitleSavePositionType.STORYBOARD_VIDEO) {
|
||
// 小说分镜详细信息保存
|
||
if (value.id == null) {
|
||
throw new Error('小说分镜详细信息ID不能为空')
|
||
}
|
||
// 获取指定的小说分镜详细信息
|
||
let bookStoryboard = await this.bookServiceBasic.GetBookTaskDetailDataById(value.id)
|
||
if (bookStoryboard == null) {
|
||
throw new Error('没有找到小说分镜信息')
|
||
}
|
||
|
||
if (value.bookSubtitlePosition.length <= 0) {
|
||
throw new Error('没有获取到字幕信息')
|
||
}
|
||
|
||
videoPath = bookStoryboard.videoPath
|
||
if (isEmpty(videoPath)) {
|
||
throw new Error('没有获取到视频路径')
|
||
}
|
||
|
||
outImagePath = path.join(
|
||
define.project_path,
|
||
`${bookStoryboard.bookId}/data/subtitle/${bookStoryboard.id}/frame.png`
|
||
)
|
||
|
||
// 获取视频的宽高数据
|
||
this.ffmpegOptions = new FfmpegOptions()
|
||
let videoSizeRes = await this.ffmpegOptions.FfmpegGetVideoSize(videoPath)
|
||
if (videoSizeRes.code == 0) {
|
||
throw new Error(videoSizeRes.message)
|
||
}
|
||
let videoSize = videoSizeRes.data
|
||
|
||
// 开始计算比例
|
||
let videoWidth = videoSize.width
|
||
let videoHeight = videoSize.height
|
||
for (let i = 0; i < value.bookSubtitlePosition.length; i++) {
|
||
const element = value.bookSubtitlePosition[i]
|
||
let widthRate = videoWidth / element.videoWidth // 宽度比例
|
||
let heightRate = videoHeight / element.videoHeight // 高度比例
|
||
// 计算比例
|
||
let newStartX = widthRate * element.startX
|
||
let newStartY = heightRate * element.startY
|
||
let newWidth = widthRate * element.width
|
||
let newHeight = heightRate * element.height
|
||
saveData.push({
|
||
startX: newStartX,
|
||
startY: newStartY,
|
||
width: newWidth,
|
||
height: newHeight,
|
||
videoWidth: videoWidth,
|
||
videoHeight: videoHeight
|
||
})
|
||
}
|
||
// 数据保存
|
||
let saveRes = this.bookServiceBasic.UpdateBookTaskDetail(bookStoryboard.id, {
|
||
subtitlePosition: JSON.stringify(saveData)
|
||
})
|
||
}
|
||
|
||
// 开始设置裁剪出来的图片位置
|
||
// 裁剪一个示例图片
|
||
let saveImagePath = await this.ffmpegOptions.FfmpegGetVideoFramdAndClip(
|
||
videoPath,
|
||
value.currentTime * 1000,
|
||
outImagePath,
|
||
saveData
|
||
)
|
||
if (saveImagePath.code == 0) {
|
||
throw new Error(saveImagePath.message)
|
||
}
|
||
return successMessage(
|
||
saveImagePath.data,
|
||
'保存字幕位置信息成功',
|
||
'WatermarkAndSubtitle_SaveBookSubtitlePosition'
|
||
)
|
||
} catch (error) {
|
||
return errorMessage(
|
||
'保存字幕位置信息失败，错误消息如下：' + error.toString(),
|
||
'WatermarkAndSubtitle_SaveBookSubtitlePosition'
|
||
)
|
||
}
|
||
}
|
||
//#endregion
|
||
|
||
//#region 本地OCR识别字幕相关操作
|
||
|
||
/**
|
||
* 获取当前视频中所有的字幕信息
|
||
* @param {*} value 需要的参数的对象，包含下面的参数
|
||
* @param {*} value.id 小说ID/小说分镜详细信息ID/null
|
||
* @param {*} value.type 保存的类型（主视频/分镜视频/后续会添加外部单独的视频提取）
|
||
* @param {*} value.videoPath 视频路径
|
||
* @param {*} value.subtitlePosition 字幕位置信息
|
||
*/
|
||
async GetVideoFrameText(value: Book.GetVideoFrameTextParams): Promise<GeneralResponse.ErrorItem | GeneralResponse.SuccessItem> {
|
||
try {
|
||
let videoPath = undefined
|
||
let tempImageFolder = undefined
|
||
let position = undefined
|
||
if (value.type == SubtitleSavePositionType.MAIN_VIDEO) {
|
||
let bookRes = await this.bookServiceBasic.GetBookDataById(value.id)
|
||
if (bookRes == null) {
|
||
throw new Error('没有找到小说对应的的视频地址')
|
||
}
|
||
let book = bookRes
|
||
tempImageFolder = path.join(define.project_path, `${book.id}/data/subtitle/${book.id}/temp`)
|
||
if (isEmpty(book.subtitlePosition)) {
|
||
throw new Error('请先保存位置信息')
|
||
}
|
||
position = JSON.parse(book.subtitlePosition)
|
||
videoPath = book.oldVideoPath
|
||
} else if (value.type == SubtitleSavePositionType.STORYBOARD_VIDEO) {
|
||
let bookTaskDetail = await this.bookServiceBasic.GetBookTaskDetailDataById(value.id);
|
||
if (bookTaskDetail == null) {
|
||
throw new Error("没有找到小说分镜详细信息")
|
||
}
|
||
|
||
tempImageFolder = path.join(define.project_path, `${bookTaskDetail.bookId}/data/subtitle/${bookTaskDetail.name}_${bookTaskDetail.id}/temp`)
|
||
if (isEmpty(value.subtitlePosition)) {
|
||
throw new Error('请先保存位置信息')
|
||
}
|
||
position = JSON.parse(value.subtitlePosition)
|
||
videoPath = bookTaskDetail.videoPath
|
||
} else {
|
||
throw new Error("不支持的操作");
|
||
}
|
||
|
||
await CheckFolderExistsOrCreate(tempImageFolder)
|
||
// 判断文件夹是不是存在，存在的话，将里面的所有文件删除
|
||
await DeleteFolderAllFile(tempImageFolder)
|
||
|
||
// 将视频进行抽帧，（目前是每秒1帧，时间小于一秒，抽一帧）
|
||
let getDurationRes = await this.ffmpegOptions.FfmpegGetVideoDuration(videoPath)
|
||
if (getDurationRes.code == 0) {
|
||
throw new Error(getDurationRes.message)
|
||
}
|
||
let videoDuration = getDurationRes.data
|
||
let frameTime = this.GenerateFrameTimes(videoDuration, 1)
|
||
for (let i = 0; i < frameTime.length; i++) {
|
||
const item = frameTime[i];
|
||
let name = i.toString().padStart(6, '0')
|
||
let imagePath = path.join(tempImageFolder, `frame_${name}.png`)
|
||
// 开始抽帧
|
||
let res = await this.ffmpegOptions.FfmpegGetVideoFramdAndClip(
|
||
videoPath,
|
||
item,
|
||
imagePath,
|
||
position
|
||
)
|
||
if (res.code == 0) {
|
||
throw new Error(res.message)
|
||
}
|
||
}
|
||
|
||
// 开始识别
|
||
let textRes = await this.GetCurrentFrameText({
|
||
id: value.id,
|
||
type: value.type,
|
||
imageFolder: tempImageFolder
|
||
})
|
||
|
||
let allTextData = [] as string[]
|
||
// 开始获取所有的数据
|
||
let jsonPaths = await GetFilesWithExtensions(tempImageFolder, ['.json'])
|
||
for (let i = 0; i < jsonPaths.length; i++) {
|
||
const element = jsonPaths[i]
|
||
// 开始拼接
|
||
let texts = JSON.parse(await fspromises.readFile(element, 'utf-8'))
|
||
for (let j = 0; j < texts.length; j++) {
|
||
const text = texts[j][1][0]
|
||
allTextData.includes(text) ? null : allTextData.push(text)
|
||
}
|
||
}
|
||
// 这边计算相似度，返回过于相似的数据
|
||
// let res = await RemoveSimilarTexts(allTextData)
|
||
|
||
return successMessage(
|
||
allTextData.join('，'),
|
||
'获取视频的的文案信息成功',
|
||
'WatermarkAndSubtitle_GetVideoFrameText'
|
||
)
|
||
} catch (error) {
|
||
return errorMessage(
|
||
'提取视频的的文案信息失败，错误消息如下：' + error.toString(),
|
||
'WatermarkAndSubtitle_GetCurrentFrameText'
|
||
)
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 使用本地OCR识别字幕文案
|
||
* @param bookId 小说ID
|
||
* @param bookTaskId 小说任务ID
|
||
* @returns
|
||
*/
|
||
async GetCopywritingByLocalOcr(book: Book.SelectBook, bookTask: Book.SelectBookTask, bookTaskDetails: Book.SelectBookTaskDetail[]): Promise<GeneralResponse.ErrorItem | GeneralResponse.SuccessItem> {
|
||
try {
|
||
for (let i = 0; i < bookTaskDetails.length; i++) {
|
||
const item = bookTaskDetails[i];
|
||
let res = await this.GetVideoFrameText({
|
||
id: item.id,
|
||
videoPath: item.videoPath,
|
||
type: SubtitleSavePositionType.STORYBOARD_VIDEO,
|
||
subtitlePosition: book.subtitlePosition
|
||
})
|
||
if (res.code == 0) {
|
||
throw new Error(res.message)
|
||
}
|
||
// 修改数据，并返回
|
||
await this.GetSubtitleLoggerAndResponse(res.data, {
|
||
total: bookTaskDetails.length,
|
||
current: i + 1
|
||
}, book, bookTask, item)
|
||
}
|
||
return successMessage(null, "识别是所有文案成功", "Subtitle_GetCopywriting")
|
||
} catch (error) {
|
||
return errorMessage("获取分镜数据失败，失败信息如下：" + error.message, 'Subtitle_GetCopywriting')
|
||
}
|
||
}
|
||
|
||
//#endregion
|
||
|
||
//#region Lai_WHISPER识别字幕相关操作
|
||
|
||
/**
|
||
* 单个分离音频的方法
|
||
* @param book 小说数据
|
||
* @param bookTask 小说任务数据
|
||
* @param bookTaskDetail 小说任务详细信息数据
|
||
* @returns
|
||
*/
|
||
async SplitAudio(book: Book.SelectBook, bookTask: Book.SelectBookTask, bookTaskDetail: Book.SelectBookTaskDetail) {
|
||
// 开始分离音频
|
||
let videoPath = bookTaskDetail.videoPath
|
||
let audioPath = path.join(path.dirname(videoPath), bookTaskDetail.name + '.mp3');
|
||
let audioRes = await this.ffmpegOptions.FfmpegExtractAudio(bookTaskDetail.videoPath, audioPath)
|
||
if (audioRes.code == 0) {
|
||
let errorMessage = `分离音频失败，错误信息如下：${audioRes.message}`
|
||
await this.bookServiceBasic.UpdateBookTaskStatus(
|
||
bookTask.id,
|
||
BookTaskStatus.AUDIO_FAIL,
|
||
errorMessage
|
||
)
|
||
throw new Error(audioRes.message)
|
||
}
|
||
this.bookServiceBasic.UpdateBookTaskDetail(bookTaskDetail.id, {
|
||
audioPath: path.relative(define.project_path, audioPath)
|
||
})
|
||
|
||
// 推送成功消息
|
||
await this.taskScheduler.AddLogToDB(
|
||
book.id,
|
||
book.type,
|
||
`${bookTaskDetail.name}分离音频成功，输出地址：${audioPath}`,
|
||
OtherData.DEFAULT,
|
||
LoggerStatus.SUCCESS
|
||
)
|
||
// 修改状态为分离音频成功
|
||
this.bookServiceBasic.UpdateBookTaskStatus(bookTask.id, BookTaskStatus.AUDIO_DONE)
|
||
return audioPath;
|
||
}
|
||
|
||
/**
|
||
* 使用LAI Whisper 进行文本识别，然后将繁体转换为简体
|
||
* @param audioPath 要识别的音频地址
|
||
* @param subtitleSetting 识别字幕设置
|
||
*/
|
||
async LaiWhisperApi(audioPath: string, subtitleSetting: SubtitleModel.subtitleSettingModel): Promise<string> {
|
||
// 开始调用LAI API识别
|
||
let formdata = new FormData()
|
||
formdata.append("file", fs.createReadStream(audioPath)); // 如果是Node.js环境，可以使用fs.createReadStream方法
|
||
formdata.append("model", "whisper-1");
|
||
formdata.append("response_format", "srt");
|
||
formdata.append("temperature", "0");
|
||
formdata.append("language", "zh");
|
||
formdata.append("prompt", isEmpty(subtitleSetting.laiWhisper.prompt) ? "eiusmod nulla" : subtitleSetting.laiWhisper.prompt);
|
||
let url = subtitleSetting.laiWhisper.url
|
||
if (!url.endsWith('/')) {
|
||
url = url + '/'
|
||
}
|
||
const config = {
|
||
method: 'post',
|
||
url: url + 'v1/audio/transcriptions',
|
||
headers: {
|
||
'Accept': 'application/json',
|
||
'Authorization': subtitleSetting.laiWhisper.apiKey,
|
||
'Content-Type': 'multipart/form-data',
|
||
...formdata.getHeaders() // 在Node.js环境中需要添加这一行
|
||
},
|
||
data: formdata
|
||
};
|
||
|
||
// laiwhisper 要做重试机制
|
||
let res = await RetryWithBackoff(async () => {
|
||
return await axios(config)
|
||
}, 5, 2000)
|
||
let text = res.data.text;
|
||
// 但是这边是繁体，需要转化为简体
|
||
// 请求也要做重试
|
||
let simpleText = await RetryWithBackoff(async () => {
|
||
return await this.gptService.ChineseTraditionalToSimplified(text, subtitleSetting.laiWhisper.apiKey, url);
|
||
}, 5, 2000);
|
||
|
||
console.log(res.data)
|
||
return simpleText;
|
||
}
|
||
|
||
/**
|
||
* 使用LAI Whisper识别字幕
|
||
* @param bookId 小说ID
|
||
* @param bookTaskId 小说任务ID
|
||
* @param subtitleSetting 提取文案相关设置
|
||
* @returns
|
||
*/
|
||
async GetCopywritingByLaiWhisper(book: Book.SelectBook, bookTask: Book.SelectBookTask, bookTaskDetails: Book.SelectBookTaskDetail[], subtitleSetting: SubtitleModel.subtitleSettingModel): Promise<GeneralResponse.ErrorItem | GeneralResponse.SuccessItem> {
|
||
try {
|
||
|
||
let emptyVideoPaths = [] as string[]
|
||
for (let i = 0; i < bookTaskDetails.length; i++) {
|
||
const element = bookTaskDetails[i];
|
||
// 将所有的分镜视频音频分开
|
||
if (isEmpty(element.videoPath)) {
|
||
emptyVideoPaths.push(element.name)
|
||
}
|
||
}
|
||
if (emptyVideoPaths.length > 0) {
|
||
throw new Error(`以下分镜视频没有找到对应的视频路径：${emptyVideoPaths.join("，")} \n 请先计算分镜`)
|
||
}
|
||
// 拆分音频和视频
|
||
for (let i = 0; i < bookTaskDetails.length; i++) {
|
||
const bookTaskDetail = bookTaskDetails[i];
|
||
// 开始分离音频
|
||
let audioPath = await this.SplitAudio(book, bookTask, bookTaskDetail)
|
||
let fileExist = await CheckFileOrDirExist(audioPath)
|
||
if (!fileExist) {
|
||
throw new Error('没有找到对应的音频文件');
|
||
}
|
||
// 开始调用LAI API识别
|
||
let content = await this.LaiWhisperApi(audioPath, subtitleSetting);
|
||
// 向前端发送数据
|
||
await this.GetSubtitleLoggerAndResponse(content, {
|
||
total: bookTaskDetails.length,
|
||
current: i + 1
|
||
}, book, bookTask, bookTaskDetail)
|
||
}
|
||
return successMessage(
|
||
null,
|
||
`所有音频识别成功`,
|
||
'Subtitle_GetCopywritingByLaiWhisper'
|
||
)
|
||
} catch (error) {
|
||
return errorMessage("获取分镜数据失败，失败信息如下：" + error.message, 'Subtitle_GetCopywritingByLaiWhisper')
|
||
}
|
||
}
|
||
//#endregion
|
||
}
|