diff --git a/app/component/label/video/libav.worker.ts b/app/component/label/video/libav.worker.ts deleted file mode 100644 index 979cde2..0000000 --- a/app/component/label/video/libav.worker.ts +++ /dev/null @@ -1,410 +0,0 @@ -// app/component/label/video/libav.worker.ts -import type { - LibAVWrapper, - WorkerMessage, - WorkerResponse, -} from "@/app/component/label/video/types/libav" - -const ctx: Worker = self as any -let libav: any = null // Use any to access full API -const PROXY_FILE = "proxy.avi" - -ctx.onmessage = async (event: MessageEvent) => { - const { type } = event.data - - try { - switch (type) { - case "LOAD": - await loadLibAV(event.data.config.corePath) - break - case "TRANSCODE": - if (!libav) throw new Error("LibAV not loaded") - if ("file" in event.data) { - await handleTranscode(event.data.file) - } - break - case "EXTRACT_FRAME": - if (!libav) throw new Error("LibAV not loaded") - if ("time" in event.data) { - await handleExtractFrame(event.data.time) - } - break - } - } catch (err: any) { - console.error("Worker Error:", err) - ctx.postMessage({ - type: "ERROR", - error: err.message || "Unknown error", - } as WorkerResponse) - } -} - -async function loadLibAV(corePath: string) { - if (libav) { - ctx.postMessage({ type: "READY" } as WorkerResponse) - return - } - - try { - importScripts(corePath) - - const LibAVWrapper = (self as any).LibAV as LibAVWrapper - if (!LibAVWrapper || typeof LibAVWrapper.LibAV !== "function") { - throw new Error("LibAV factory not found.") - } - - const baseUrl = corePath.substring(0, corePath.lastIndexOf("/") + 1) - // Dynamic WASM URL based on the JS file name (standard convention) - const wasmUrl = corePath.replace(".js", ".wasm.wasm") - - libav = await LibAVWrapper.LibAV({ - noworker: true, - base: baseUrl, - wasmurl: wasmUrl, - }) - - ctx.postMessage({ type: "READY" } as WorkerResponse) - } catch (error) { - throw new Error(`Failed to load libav: ${error}`) - } -} - -async function handleTranscode(file: File) { - const ext = file.name.match(/\.([^.]+)$/)?.[0] || ".mp4" - const inputFile = `input_video${ext}` - - try { - // 1. Write Input File - const fileBuffer = await file.arrayBuffer() - const u8Arr = new Uint8Array(fileBuffer) - console.log(`[Worker] Received file: ${file.name}, size: ${u8Arr.length}`) - - await libav.writeFile(inputFile, u8Arr) - - // Note: libav.js (webcodecs-avf) does not support readdir/stat via API directly - // but readFile works. We trust writeFile worked. - - // 2. Init Demuxer (Input) - // webcodecs-avf SHOULD have the demuxer for MP4. - let fmt_ctx, streams - try { - ;[fmt_ctx, streams] = await libav.ff_init_demuxer_file(inputFile) - } catch (e) { - console.error("Demuxer init failed:", e) - throw new Error( - `Demuxer failed. Libav variant might lack MP4 support. Error: ${e}` - ) - } - - // Find Video Stream - let videoStreamIdx = -1 - let videoStream = null - for (let i = 0; i < streams.length; i++) { - if (streams[i].codec_type === 0) { - // 0 is Video - videoStreamIdx = i - videoStream = streams[i] - break - } - } - if (videoStreamIdx === -1) throw new Error("No video stream found") - - console.log( - `[Worker] Found video stream #${videoStreamIdx}, codec_id=${videoStream.codec_id}` - ) - - // 3. Init Decoder (Input) - // For obsolete variant, we might have software H.264 decoders. - // Try standard init first. - - let c, pkt, frame - try { - // Try standard init first - ;[, c, pkt, frame] = await libav.ff_init_decoder( - videoStream.codec_id, - videoStream.codecpar - ) - } catch (e) { - console.warn( - "[Worker] Standard decoder init failed, trying explicit H264...", - e - ) - try { - // Try explicit H264 - ;[, c, pkt, frame] = await libav.ff_init_decoder( - "h264", - videoStream.codecpar - ) - } catch (e2) { - console.error("[Worker] All decoder init attempts failed", e2) - throw new Error("Codec not found (decoder init failed).") - } - } - - // Get input properties - let width = await libav.AVCodecParameters_width(videoStream.codecpar) - let height = await libav.AVCodecParameters_height(videoStream.codecpar) - - console.log(`[Worker] Video dimensions: ${width}x${height}`) - - // 4. Init Encoder (Output: MJPEG) - const codec_name = "mjpeg" - const [, enc_c, enc_frame, enc_pkt] = await libav.ff_init_encoder( - codec_name, - { - ctx: { - width: width, - height: height, - pix_fmt: 0, // AV_PIX_FMT_YUV420P - time_base: [1, 25], - framerate: [25, 1], - }, - } - ) - - // 5. Init Muxer (Output: AVI container) - const [oc, , pb, [stream_ctx]] = await libav.ff_init_muxer( - { format_name: "avi", filename: PROXY_FILE, open: true }, - [[enc_c, 1, 25]] - ) - - console.log(stream_ctx) - - // Write Header - await libav.avformat_write_header(oc, 0) - - ctx.postMessage({ - type: "LOG", - message: "Transcoding started...", - } as WorkerResponse) - - // 6. Loop: Read -> Decode -> Encode -> Write - let frameCount = 0 - - // We need to use "video" copyout mode for WebCodecs to work efficiently? - // Actually, ff_decode_multi returns Frame objects. - // If WebCodecs is used, the Frame might be backed by VideoFrame. - - while (true) { - const [res, packets] = await libav.ff_read_frame_multi(fmt_ctx, pkt) - - if (res === libav.AVERROR_EOF) break - if (res < 0 && res !== -11) break - - const streamPackets = packets[videoStreamIdx] - if (!streamPackets || streamPackets.length === 0) continue - - // Decode - const decodedFrames = await libav.ff_decode_multi( - c, - pkt, - frame, - streamPackets - ) - - for (const f of decodedFrames) { - console.log(f) - - // Encode - const encodedPackets = await libav.ff_encode_multi( - enc_c, - enc_frame, - enc_pkt - ) - - for (const p of encodedPackets) { - p.stream_index = 0 - } - - await libav.ff_write_multi(oc, enc_pkt, encodedPackets) - frameCount++ - } - - if (frameCount % 30 === 0) { - ctx.postMessage({ - type: "LOG", - message: `Transcoded ${frameCount} frames...`, - } as WorkerResponse) - } - } - - // Flush Encoder - const flushedPackets = await libav.ff_encode_multi( - enc_c, - enc_frame, - enc_pkt, - [], - true - ) - if (flushedPackets.length > 0) { - for (const p of flushedPackets) p.stream_index = 0 - await libav.ff_write_multi(oc, enc_pkt, flushedPackets) - } - - // Write Trailer - await libav.av_write_trailer(oc) - - // Cleanup - await libav.ff_free_muxer(oc, pb) - await libav.ff_free_encoder(enc_c, enc_frame, enc_pkt) - await libav.ff_free_decoder(c, pkt, frame) - await libav.avformat_close_input_js(fmt_ctx) - await libav.unlink(inputFile) - - ctx.postMessage({ - type: "DONE", - data: `Transcoding complete. ${frameCount} frames ready.`, - } as WorkerResponse) - } catch (e: any) { - console.error(e) - throw e - } -} - -async function handleExtractFrame(time: number) { - if (!libav) return - - try { - // 1. Open Proxy File - // Note: For efficiency in a real app, keep this open. - // But for safety here, we open/close per request or keep it open in global? - // Let's open it fresh. AVI parsing is fast. - const [fmt_ctx, streams] = await libav.ff_init_demuxer_file(PROXY_FILE) - - const streamIdx = 0 // We know we only have 1 video stream - const stream = streams[streamIdx] - console.log(stream) - - // 2. Init Decoder (MJPEG) - const [, c, pkt, frame] = await libav.ff_init_decoder("mjpeg") - - // 3. Seek - // Calculate timestamp in AV_TIME_BASE or stream time base - // av_seek_frame(ctx, stream_index, timestamp, flags) - // If stream_index is -1, timestamp is in AV_TIME_BASE (microseconds) - // We'll use stream index 0. We need to know its timebase. - // But easier: Use -1 and AV_TIME_BASE - const targetTS = Math.floor(time * 1000000) // seconds -> microseconds - - // seek flags: 1 (BACKWARD) is usually good to find keyframe before time - await libav.av_seek_frame(fmt_ctx, -1, targetTS, 1) - - // 4. Read & Decode until we hit the frame - // Since MJPEG is all-intra, the first frame we read after seek should be the one (or close) - let foundFrame = null - - // Limit attempts - for (let i = 0; i < 10; i++) { - const [res, packets] = await libav.ff_read_frame_multi(fmt_ctx, pkt, { - limit: 1, - }) - if (res === libav.AVERROR_EOF) break - - const streamPackets = packets[streamIdx] - if (!streamPackets || streamPackets.length === 0) continue - - // Decode - const frames = await libav.ff_decode_multi(c, pkt, frame, streamPackets) - if (frames && frames.length > 0) { - foundFrame = frames[0] - break // Just take the first one found after seek - } - } - - if (foundFrame) { - // 5. Convert to Blob (BMP is simplest uncompressed format libav supports encoding to?) - // Or use libav to encode to JPEG? - // Actually, since we are in MJPEG, the packet *is* a JPEG. - // But we just decoded it. - // Let's re-encode the single frame to JPEG for the UI. - // Or just return the raw packet? - // Let's re-encode to be safe (ensure headers etc). - - // Init single-frame encoder - // const [, jpeg_c, jpeg_frame, jpeg_pkt] = await libav.ff_init_encoder("mjpeg", { - // ctx: { width: foundFrame.width, height: foundFrame.height, ... } - // }) - // This is heavy. - - // Alternative: "copyoutFrame" as "ImageData" (only in main thread). - // In Worker, we can use "video_packed" to get RGB/RGBA buffer. - // libav.js Frame object has data. - // Let's stick to the prompt: "extract single images". - // I'll return a Blob of a JPEG. - // Since I have a valid frame, I can use a simple JS JPEG encoder or libav. - // Using libav is robust. - - // Let's try to grab the PACKET directly from the read step? - // If the packet is a full JPEG, we can just dump it. - // MJPEG stream packets usually miss Huffman tables (DHT). Browsers often fail to render them directly. - // So safe bet: Decode -> Encode to JPEG (single frame). - - // Reuse the decoder context? No, that's for decoding. - // Create a throwaway encoder for the snapshot. - const width = await libav.AVCodecContext_width(c) - const height = await libav.AVCodecContext_height(c) - - // We need to know the pixel format of the decoded frame. - // usually YUVJ420P. - - const [, enc_c, enc_frame, enc_pkt] = await libav.ff_init_encoder( - "mjpeg", - { - ctx: { - width, - height, - pix_fmt: 0, // YUV420P - time_base: [1, 1], - framerate: [1, 1], - }, - options: { - // Force standard JPEG tables - qscale: "2", - }, - } - ) - - const encodedPackets = await libav.ff_encode_multi( - enc_c, - enc_frame, - enc_pkt, - [foundFrame] - ) - - // Combine packets if multiple (usually 1 for JPEG) - // But actually, ff_encode_multi returns packets. - // We can concat their data. - - const chunks = [] - for (const p of encodedPackets) { - // p.data is the pointer. We need to copy it out. - // Wait, ff_encode_multi returns objects with .data usually if copyoutPacket is default? - // No, it returns Packet objects which refer to memory. - // We need to copy data out. - const data = await libav.ff_copyout_packet(p.ptr) // or just p.data if it's already copied? - // libav.js docs: "Most functions will instead copy out their data into libav.js Frame or Packet objects." - // So `encodedPackets` are JS objects with `data` (Uint8Array). - console.log(data) - chunks.push(p.data) - } - - const blob = new Blob(chunks, { type: "image/jpeg" }) - - ctx.postMessage({ - type: "FRAME", - data: blob, - time: time, - } as WorkerResponse) - - await libav.ff_free_encoder(enc_c, enc_frame, enc_pkt) - } else { - console.warn("Frame not found after seek") - } - - // Cleanup extraction - await libav.ff_free_decoder(c, pkt, frame) - await libav.avformat_close_input_js(fmt_ctx) - } catch (e) { - console.error("Extraction error", e) - } -} diff --git a/app/component/label/video/page.tsx b/app/component/label/video/page.tsx deleted file mode 100644 index 1d63891..0000000 --- a/app/component/label/video/page.tsx +++ /dev/null @@ -1,127 +0,0 @@ -"use client" - -import { ChangeEvent, useEffect, useRef, useState } from "react" -import type { - WorkerMessage, - WorkerResponse, -} from "@/app/component/label/video/types/libav" - -export default function Home() { - const workerRef = useRef(null) - const [ready, setReady] = useState(false) - const [logs, setLogs] = useState([]) - const [videoUrl, setVideoUrl] = useState(null) - const [processing, setProcessing] = useState(false) - const addLog = (msg: string) => { - setLogs((prev) => [...prev.slice(-10), msg]) // 只保留最近10条 - } - useEffect(() => { - // 初始化 Worker - // Next.js 会自动检测到这个语法并单独打包 Worker - workerRef.current = new Worker( - new URL("./libav.worker.ts", import.meta.url) - ) - - // 设置消息监听 - workerRef.current.onmessage = (event: MessageEvent) => { - const { type } = event.data - - switch (type) { - case "READY": - setReady(true) - addLog("FFmpeg core loaded and ready.") - break - case "LOG": - if ("message" in event.data) addLog(event.data.message) - break - case "DONE": - if ("data" in event.data) { - const url = URL.createObjectURL(event.data.data) - setVideoUrl(url) - setProcessing(false) - addLog("Transcoding complete!") - } - break - case "ERROR": - if ("error" in event.data) { - addLog(`Error: ${event.data.error}`) - setProcessing(false) - } - break - } - } - - // 告诉 Worker 加载 libav - // 这里的路径必须指向 public 目录下的实际文件 - // 使用 default-cli 变体以确保包含 MP4/H.264 等常见格式的解封装器和解码器 - const LIBAV_PATH = "/libav/libav-6.8.8.0-default.js" - workerRef.current.postMessage({ - type: "LOAD", - config: { corePath: location.origin + LIBAV_PATH }, - } as WorkerMessage) - - return () => { - workerRef.current?.terminate() - } - }, []) - - const handleFileChange = (e: ChangeEvent) => { - const file = e.target.files?.[0] - if (!file || !workerRef.current || !ready) return - - setProcessing(true) - setVideoUrl(null) - setLogs([]) - addLog(`Starting upload: ${file.name}`) - - workerRef.current.postMessage({ - type: "TRANSCODE", - file: file, - } as WorkerMessage) - } - - return ( -
-

Next.js + Libav.js + Worker

- -
-

- Status:{" "} - {ready ? ( - Ready - ) : ( - Loading Core... - )} -

-

Processing: {processing ? "Yes" : "No"}

-
- - - - {videoUrl && ( -
-

Result:

-
- )} - -
- {logs.map((log, i) => ( -
- {log} -
- ))} -
-
- ) -} diff --git a/app/component/label/video/types/libav.d.ts b/app/component/label/video/types/libav.d.ts deleted file mode 100644 index f1a7e17..0000000 --- a/app/component/label/video/types/libav.d.ts +++ /dev/null @@ -1,74 +0,0 @@ -// src/types/libav.d.ts - -export type WorkerMessage = - | { type: "LOAD"; config: { corePath: string } } - | { type: "TRANSCODE"; file: File } - | { type: "EXTRACT_FRAME"; time: number } - -export type WorkerResponse = - | { type: "READY" } - | { type: "LOG"; message: string } - | { type: "DONE"; data: any } // data 类型根据实际返回调整 - | { type: "FRAME"; data: Blob; time: number } - | { type: "ERROR"; error: string } - -declare global { - interface Window { - LibAV: LibAVWrapper - } -} - -export interface LibAVOpts { - noworker?: boolean - base?: string - wasmurl?: string - variant?: string -} - -export interface LibAVWrapper { - LibAV(opts?: LibAVOpts): Promise -} - -// 核心实例接口：添加官方例子中用到的底层方法 -export interface LibAVInstance { - writeFile(name: string, content: Uint8Array): Promise - unlink(name: string): Promise - terminate(): void - - // 1. 初始化解封装器 (Demuxer) - // 返回 [Context, Streams[]] - ff_init_demuxer_file(filename: string): Promise<[number, any[]]> - - // 2. 初始化解码器 (Decoder) - // 返回 [Codec, CodecContext, Packet, Frame] - ff_init_decoder( - codec_id: number, - codecpar: number - ): Promise<[number, number, number, number]> - - // 3. 读取多个包 - // 返回 [ResultCode, PacketsMap] - ff_read_frame_multi( - fmt_ctx: number, - pkt: number, - opts?: { - limit?: number // OUTPUT limit, in bytes - unify?: boolean // If true, unify the packets into a single stream (called 0), so that the output is in the same order as the input - copyoutPacket?: "default" // Version of ff_copyout_packet to use - } - ): Promise<[number, Record]> - - // 4. 解码多个包 - // 返回 DecodedFrames[] - ff_decode_multi( - ctx: number, - pkt: number, - frame: number, - packets: any[], - fin?: boolean - ): Promise - - // 辅助：释放资源 - avformat_close_input_js(fmt_ctx: number): Promise - ff_free_decoder(c: number, pkt: number, frame: number): Promise -} diff --git a/app/component/label/video2image/processVideo.ts b/app/component/label/video2image/processVideo.ts new file mode 100644 index 0000000..4ab4a58 --- /dev/null +++ b/app/component/label/video2image/processVideo.ts @@ -0,0 +1,196 @@ +"use client" + +import type { WorkerMessage, WorkerResponse } from "../ffmpeg/ffmpeg.worker" + +export interface VideoFrameFile { + name: string + dataUrl: string +} + +interface ProcessVideoParams { + base64Data: string + fileName: string + namePrefix?: string + baseURL?: string + args?: string[] +} + +const defaultArgs = [ + "-vf", + "fps=1,scale='min(720,iw)':-1", + "-q:v", + "2", + "frame_%03d.jpg", +] + +const mimeByExt: Record = { + jpg: "image/jpeg", + jpeg: "image/jpeg", + png: "image/png", + webp: "image/webp", + mp4: "video/mp4", + mov: "video/quicktime", + webm: "video/webm", + mkv: "video/x-matroska", + avi: "video/x-msvideo", + h264: "video/h264", + "264": "video/h264", +} + +const getFileExt = (fileName: string) => { + const segments = fileName.split(".") + if (segments.length <= 1) return "" + return segments[segments.length - 1].toLowerCase() +} + +const stripFileExt = (fileName: string) => { + const index = fileName.lastIndexOf(".") + if (index <= 0) return fileName + return fileName.slice(0, index) +} + +const normalizeBase64 = (text: string) => { + if (!text) return "" + const raw = text.includes(",") ? text.split(",").slice(1).join(",") : text + return raw.replace(/\s+/g, "") +} + +const sanitizePrefix = (prefix: string) => { + const safe = prefix + .replace(/[^a-zA-Z0-9_-]+/g, "_") + .replace(/_+/g, "_") + .replace(/^_|_$/g, "") + return safe || "video" +} + +const uint8ArrayToBase64 = (arr: Uint8Array) => { + const chunkSize = 0x8000 + let binary = "" + for (let i = 0; i < arr.length; i += chunkSize) { + binary += String.fromCharCode(...arr.subarray(i, i + chunkSize)) + } + return btoa(binary) +} + +const base64ToFile = (base64Data: string, fileName: string) => { + const payload = normalizeBase64(base64Data) + if (!payload) throw new Error("empty video payload") + const binary = atob(payload) + const len = binary.length + const bytes = new Uint8Array(len) + for (let i = 0; i < len; i += 1) { + bytes[i] = binary.charCodeAt(i) + } + const fileExt = getFileExt(fileName) + return new File([bytes], fileName, { + type: mimeByExt[fileExt] || "video/mp4", + }) +} + +const getOutputExtFromArgs = (args: string[]) => { + for (let i = args.length - 1; i >= 0; i -= 1) { + const ext = getFileExt(args[i]) + if (ext) return ext + } + return "jpg" +} + +export const processVideo = async ( + params: ProcessVideoParams +): Promise => { + if (typeof window === "undefined") return [] + + const args = params.args && params.args.length ? params.args : defaultArgs + const outputExt = getOutputExtFromArgs(args) + const prefix = sanitizePrefix( + params.namePrefix || stripFileExt(params.fileName) + ) + const baseURL = params.baseURL || `${window.location.origin}/wasm` + const inputFile = base64ToFile(params.base64Data, params.fileName) + + const worker = new Worker( + new URL("../ffmpeg/ffmpeg.worker.ts", import.meta.url) + ) + let frameIndex = 0 + const frames: VideoFrameFile[] = [] + + try { + return await new Promise((resolve, reject) => { + let settled = false + + const finish = (next: () => void) => { + if (settled) return + settled = true + next() + } + + worker.onmessage = (event: MessageEvent) => { + const message = event.data + switch (message.type) { + case "READY": { + const runMessage: WorkerMessage = { + type: "EXEC", + file: inputFile, + args, + outputPattern: "frame_", + } + worker.postMessage(runMessage) + return + } + case "SEGMENT_DATA": { + const files = message.files || [] + files.forEach((file) => { + const ext = getFileExt(file.name) || outputExt || "jpg" + const frameName = `${prefix}_frame_${String(frameIndex).padStart( + 6, + "0" + )}.${ext}` + frameIndex += 1 + frames.push({ + name: frameName, + dataUrl: `data:${mimeByExt[ext] || "image/jpeg"};base64,${uint8ArrayToBase64(file.data)}`, + }) + }) + return + } + case "DONE": { + finish(() => { + if (!frames.length) { + reject(new Error("video has no extracted frames")) + } else { + resolve(frames) + } + }) + return + } + case "FATAL": { + finish(() => + reject(new Error(message.error || "video process fatal")) + ) + return + } + case "ERROR": { + finish(() => + reject(new Error(message.error || "video process error")) + ) + return + } + default: + return + } + } + + worker.onerror = (event) => { + finish(() => reject(new Error(event.message || "video worker error"))) + } + + const loadMessage: WorkerMessage = { + type: "LOAD", + baseURL, + } + worker.postMessage(loadMessage) + }) + } finally { + worker.terminate() + } +} diff --git a/components/label/LabelNossr.tsx b/components/label/LabelNossr.tsx index ed68002..f7926af 100644 --- a/components/label/LabelNossr.tsx +++ b/components/label/LabelNossr.tsx @@ -12,12 +12,13 @@ import ScaleComponent from "./components/ScaleComponent" import { Box, Flex, LoadingOverlay, Stack } from "@mantine/core" import { showNotification } from "@mantine/notifications" -import { getLabelResult } from "./api/label" +import { getLabelResult, getServerImage } from "./api/label" import { Comment, WorkLoad } from "./api/label/typing" import { getProjectDetail } from "./api/project" import { Project } from "./api/project/typing" import { getTaskList, getTaskWorkFlow } from "./api/task" import { Task } from "./api/task/typing" +import { processVideo } from "@/app/component/label/video2image/processVideo" import BottomTools from "./components/BottomTools" import ConfirmModal from "./components/ConfirmModal" import { splitWord } from "./components/EditorContainer" @@ -29,7 +30,13 @@ import RightQATools from "./components/RightQATools" import RightTaskTools from "./components/RightTaskTools" import ScaleToolContainer from "./components/ScaleToolContainer" import TopTools from "./components/TopTools" -import { useKeyEventStore, useLabelStore, useObjectStore } from "./store" +import { + useImagesStore, + useKeyEventStore, + useLabelStore, + useObjectStore, + useVideoFrameStore, +} from "./store" import { usePermissionStore } from "./store/auth" import { useBottomToolsStore } from "./useBottomToolsStore" import { useDescToolsStore } from "./useDescToolsStore" @@ -74,6 +81,16 @@ const getSegmentPoints = (arr: Array) => { return arr.map((item) => [item[0], item[1]]) } +const videoExtPattern = /\.(mp4|mov|avi|mkv|webm|h264|264|ts|m4v)$/i + +const normalizeNamePrefix = (name: string) => { + const value = name + .replace(/[^a-zA-Z0-9_-]+/g, "_") + .replace(/_+/g, "_") + .replace(/^_|_$/g, "") + return value || "video" +} + interface LabelProps { headerHeight: number leftWidth: number @@ -218,6 +235,91 @@ const LabelPage = ({ setQaOperations, ]) + const normalizeVideoTaskData = useCallback( + async (detail: Task.DataProps) => { + try { + const sourceNames = detail.data_name + .map((item) => item?.name?.[0] || "") + .filter((name) => !!name) + if (!sourceNames.length || !projectId) return detail + + const taskDataType = Number( + (detail as { [key: string]: any })?.data_type ?? + (detail as { [key: string]: any })?.task_data_type ?? + -1 + ) + const isVideoTask = + taskDataType === 2 || + sourceNames.some((name) => videoExtPattern.test(name)) + if (!isVideoTask) return detail + + const imagesMap = new Map(useImagesStore.getState().images) + let imageMapUpdated = false + const normalizedDataName: Task.DataProps["data_name"] = [] + + for (const videoName of sourceNames) { + const cacheKey = `${projectId}:${taskId}:${videoName}` + let frameNames = + useVideoFrameStore.getState().videoFrames.get(cacheKey) ?? [] + + const hasAllCachedFrames = + frameNames.length > 0 && + frameNames.every((frameName) => { + const cache = imagesMap.get(frameName) + return typeof cache === "string" && cache.length > 0 + }) + + if (!hasAllCachedFrames) { + const base64Video = await getServerImage({ + data_names: [videoName], + data_type: 2, + project_id: projectId, + }) + const frameData = await processVideo({ + base64Data: String(base64Video || ""), + fileName: videoName, + namePrefix: normalizeNamePrefix( + `${projectId}_${taskId}_${videoName.replace(/\.[^.]+$/, "")}` + ), + }) + frameNames = frameData.map((frame) => frame.name) + frameData.forEach((frame) => { + imagesMap.set(frame.name, frame.dataUrl) + }) + imageMapUpdated = true + useVideoFrameStore.getState().setVideoFrames(cacheKey, frameNames) + } + + frameNames.forEach((frameName) => { + normalizedDataName.push({ + name: [frameName] as [string], + related_images: [], + }) + }) + } + + if (imageMapUpdated) { + useImagesStore.getState().setImages(imagesMap) + } + if (!normalizedDataName.length) return detail + + return { + ...detail, + data_name: normalizedDataName, + } + } catch (error) { + console.log(error) + showNotification({ + color: "red", + title: "视频解码失败", + message: "视频帧生成失败，已回退到原始数据", + }) + return detail + } + }, + [projectId, taskId] + ) + const asyncGetTaskDetail = useCallback(async () => { if (taskId) { try { @@ -230,6 +332,7 @@ const LabelPage = ({ page_size: 1, }) let detail = res.task_list[0] + detail = await normalizeVideoTaskData(detail) setTaskDetail(detail) // 帧列表 useBottomToolsStore @@ -579,7 +682,14 @@ const LabelPage = ({ usePaperStore.getState().setLoadingData(false) } } - }, [setLabel, setLabelTime, setStateStack, taskId, user_id]) + }, [ + normalizeVideoTaskData, + setLabel, + setLabelTime, + setStateStack, + taskId, + user_id, + ]) // 保存时更新当前workload const updateOldWorkLoad = (data: WorkLoad) => { diff --git a/components/label/api/task/typing.ts b/components/label/api/task/typing.ts index 6129549..43cd90c 100644 --- a/components/label/api/task/typing.ts +++ b/components/label/api/task/typing.ts @@ -2,6 +2,8 @@ export namespace Task { export interface DataProps { id: number create_date: string + data_type?: number + task_data_type?: number data_name: { name: [string] related_images: [] @@ -62,6 +64,7 @@ export namespace Task { } export interface ListRequest { + data_type?: number current_uid?: number[] data_name?: string first_commit_label_time_end?: string diff --git a/components/label/store.ts b/components/label/store.ts index bdb729d..b2ee58f 100644 --- a/components/label/store.ts +++ b/components/label/store.ts @@ -498,6 +498,12 @@ interface ImageState { setImages: (value: any) => void } +interface VideoFrameState { + videoFrames: Map + setVideoFrames: (key: string, value: string[]) => void + removeVideoFrames: (key: string) => void +} + export const useImagesStore = create( persist( (storeSet) => ({ @@ -516,3 +522,33 @@ export const useImagesStore = create( } ) ) + +export const useVideoFrameStore = create( + persist( + (storeSet) => ({ + videoFrames: new Map(), + setVideoFrames: (key: string, value: string[]) => + storeSet((state) => { + const nextMap = new Map(state.videoFrames) + nextMap.set(key, value) + return { + ...state, + videoFrames: nextMap, + } + }), + removeVideoFrames: (key: string) => + storeSet((state) => { + const nextMap = new Map(state.videoFrames) + nextMap.delete(key) + return { + ...state, + videoFrames: nextMap, + } + }), + }), + { + name: "video-frame-storage", + storage: storage, + } + ) +)