paperclip-bot/labelmain-demo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(label): video process

Browse Source
This commit is contained in:
shay7sev
2026-02-27 13:57:42 +08:00
parent 36d5a0e5b0
commit bb70cee307
7 changed files with 348 additions and 614 deletions
Download Patch File Download Diff File Expand all files Collapse all files

410
app/component/label/video/libav.worker.ts
View File

@@ -1,410 +0,0 @@
// app/component/label/video/libav.worker.ts
import type {
LibAVWrapper,
WorkerMessage,
WorkerResponse,
} from "@/app/component/label/video/types/libav"
const ctx: Worker = self as any
let libav: any = null // Use any to access full API
const PROXY_FILE = "proxy.avi"
ctx.onmessage = async (event: MessageEvent<WorkerMessage>) => {
const { type } = event.data
try {
switch (type) {
case "LOAD":
await loadLibAV(event.data.config.corePath)
break
case "TRANSCODE":
if (!libav) throw new Error("LibAV not loaded")
if ("file" in event.data) {
await handleTranscode(event.data.file)
}
break
case "EXTRACT_FRAME":
if (!libav) throw new Error("LibAV not loaded")
if ("time" in event.data) {
await handleExtractFrame(event.data.time)
}
break
}
} catch (err: any) {
console.error("Worker Error:", err)
ctx.postMessage({
type: "ERROR",
error: err.message || "Unknown error",
} as WorkerResponse)
}
}
async function loadLibAV(corePath: string) {
if (libav) {
ctx.postMessage({ type: "READY" } as WorkerResponse)
return
}
try {
importScripts(corePath)
const LibAVWrapper = (self as any).LibAV as LibAVWrapper
if (!LibAVWrapper || typeof LibAVWrapper.LibAV !== "function") {
throw new Error("LibAV factory not found.")
}
const baseUrl = corePath.substring(0, corePath.lastIndexOf("/") + 1)
// Dynamic WASM URL based on the JS file name (standard convention)
const wasmUrl = corePath.replace(".js", ".wasm.wasm")
libav = await LibAVWrapper.LibAV({
noworker: true,
base: baseUrl,
wasmurl: wasmUrl,
})
ctx.postMessage({ type: "READY" } as WorkerResponse)
} catch (error) {
throw new Error(`Failed to load libav: ${error}`)
}
}
async function handleTranscode(file: File) {
const ext = file.name.match(/\.([^.]+)$/)?.[0] || ".mp4"
const inputFile = `input_video${ext}`
try {
// 1. Write Input File
const fileBuffer = await file.arrayBuffer()
const u8Arr = new Uint8Array(fileBuffer)
console.log(`[Worker] Received file: ${file.name}, size: ${u8Arr.length}`)
await libav.writeFile(inputFile, u8Arr)
// Note: libav.js (webcodecs-avf) does not support readdir/stat via API directly
// but readFile works. We trust writeFile worked.
// 2. Init Demuxer (Input)
// webcodecs-avf SHOULD have the demuxer for MP4.
let fmt_ctx, streams
try {
;[fmt_ctx, streams] = await libav.ff_init_demuxer_file(inputFile)
} catch (e) {
console.error("Demuxer init failed:", e)
throw new Error(
`Demuxer failed. Libav variant might lack MP4 support. Error: ${e}`
)
}
// Find Video Stream
let videoStreamIdx = -1
let videoStream = null
for (let i = 0; i < streams.length; i++) {
if (streams[i].codec_type === 0) {
// 0 is Video
videoStreamIdx = i
videoStream = streams[i]
break
}
}
if (videoStreamIdx === -1) throw new Error("No video stream found")
console.log(
`[Worker] Found video stream #${videoStreamIdx}, codec_id=${videoStream.codec_id}`
)
// 3. Init Decoder (Input)
// For obsolete variant, we might have software H.264 decoders.
// Try standard init first.
let c, pkt, frame
try {
// Try standard init first
;[, c, pkt, frame] = await libav.ff_init_decoder(
videoStream.codec_id,
videoStream.codecpar
)
} catch (e) {
console.warn(
"[Worker] Standard decoder init failed, trying explicit H264...",
e
)
try {
// Try explicit H264
;[, c, pkt, frame] = await libav.ff_init_decoder(
"h264",
videoStream.codecpar
)
} catch (e2) {
console.error("[Worker] All decoder init attempts failed", e2)
throw new Error("Codec not found (decoder init failed).")
}
}
// Get input properties
let width = await libav.AVCodecParameters_width(videoStream.codecpar)
let height = await libav.AVCodecParameters_height(videoStream.codecpar)
console.log(`[Worker] Video dimensions: ${width}x${height}`)
// 4. Init Encoder (Output: MJPEG)
const codec_name = "mjpeg"
const [, enc_c, enc_frame, enc_pkt] = await libav.ff_init_encoder(
codec_name,
{
ctx: {
width: width,
height: height,
pix_fmt: 0, // AV_PIX_FMT_YUV420P
time_base: [1, 25],
framerate: [25, 1],
},
}
)
// 5. Init Muxer (Output: AVI container)
const [oc, , pb, [stream_ctx]] = await libav.ff_init_muxer(
{ format_name: "avi", filename: PROXY_FILE, open: true },
[[enc_c, 1, 25]]
)
console.log(stream_ctx)
// Write Header
await libav.avformat_write_header(oc, 0)
ctx.postMessage({
type: "LOG",
message: "Transcoding started...",
} as WorkerResponse)
// 6. Loop: Read -> Decode -> Encode -> Write
let frameCount = 0
// We need to use "video" copyout mode for WebCodecs to work efficiently?
// Actually, ff_decode_multi returns Frame objects.
// If WebCodecs is used, the Frame might be backed by VideoFrame.
while (true) {
const [res, packets] = await libav.ff_read_frame_multi(fmt_ctx, pkt)
if (res === libav.AVERROR_EOF) break
if (res < 0 && res !== -11) break
const streamPackets = packets[videoStreamIdx]
if (!streamPackets || streamPackets.length === 0) continue
// Decode
const decodedFrames = await libav.ff_decode_multi(
c,
pkt,
frame,
streamPackets
)
for (const f of decodedFrames) {
console.log(f)
// Encode
const encodedPackets = await libav.ff_encode_multi(
enc_c,
enc_frame,
enc_pkt
)
for (const p of encodedPackets) {
p.stream_index = 0
}
await libav.ff_write_multi(oc, enc_pkt, encodedPackets)
frameCount++
}
if (frameCount % 30 === 0) {
ctx.postMessage({
type: "LOG",
message: `Transcoded ${frameCount} frames...`,
} as WorkerResponse)
}
}
// Flush Encoder
const flushedPackets = await libav.ff_encode_multi(
enc_c,
enc_frame,
enc_pkt,
[],
true
)
if (flushedPackets.length > 0) {
for (const p of flushedPackets) p.stream_index = 0
await libav.ff_write_multi(oc, enc_pkt, flushedPackets)
}
// Write Trailer
await libav.av_write_trailer(oc)
// Cleanup
await libav.ff_free_muxer(oc, pb)
await libav.ff_free_encoder(enc_c, enc_frame, enc_pkt)
await libav.ff_free_decoder(c, pkt, frame)
await libav.avformat_close_input_js(fmt_ctx)
await libav.unlink(inputFile)
ctx.postMessage({
type: "DONE",
data: `Transcoding complete. ${frameCount} frames ready.`,
} as WorkerResponse)
} catch (e: any) {
console.error(e)
throw e
}
}
async function handleExtractFrame(time: number) {
if (!libav) return
try {
// 1. Open Proxy File
// Note: For efficiency in a real app, keep this open.
// But for safety here, we open/close per request or keep it open in global?
// Let's open it fresh. AVI parsing is fast.
const [fmt_ctx, streams] = await libav.ff_init_demuxer_file(PROXY_FILE)
const streamIdx = 0 // We know we only have 1 video stream
const stream = streams[streamIdx]
console.log(stream)
// 2. Init Decoder (MJPEG)
const [, c, pkt, frame] = await libav.ff_init_decoder("mjpeg")
// 3. Seek
// Calculate timestamp in AV_TIME_BASE or stream time base
// av_seek_frame(ctx, stream_index, timestamp, flags)
// If stream_index is -1, timestamp is in AV_TIME_BASE (microseconds)
// We'll use stream index 0. We need to know its timebase.
// But easier: Use -1 and AV_TIME_BASE
const targetTS = Math.floor(time * 1000000) // seconds -> microseconds
// seek flags: 1 (BACKWARD) is usually good to find keyframe before time
await libav.av_seek_frame(fmt_ctx, -1, targetTS, 1)
// 4. Read & Decode until we hit the frame
// Since MJPEG is all-intra, the first frame we read after seek should be the one (or close)
let foundFrame = null
// Limit attempts
for (let i = 0; i < 10; i++) {
const [res, packets] = await libav.ff_read_frame_multi(fmt_ctx, pkt, {
limit: 1,
})
if (res === libav.AVERROR_EOF) break
const streamPackets = packets[streamIdx]
if (!streamPackets || streamPackets.length === 0) continue
// Decode
const frames = await libav.ff_decode_multi(c, pkt, frame, streamPackets)
if (frames && frames.length > 0) {
foundFrame = frames[0]
break // Just take the first one found after seek
}
}
if (foundFrame) {
// 5. Convert to Blob (BMP is simplest uncompressed format libav supports encoding to?)
// Or use libav to encode to JPEG?
// Actually, since we are in MJPEG, the packet *is* a JPEG.
// But we just decoded it.
// Let's re-encode the single frame to JPEG for the UI.
// Or just return the raw packet?
// Let's re-encode to be safe (ensure headers etc).
// Init single-frame encoder
// const [, jpeg_c, jpeg_frame, jpeg_pkt] = await libav.ff_init_encoder("mjpeg", {
// ctx: { width: foundFrame.width, height: foundFrame.height, ... }
// })
// This is heavy.
// Alternative: "copyoutFrame" as "ImageData" (only in main thread).
// In Worker, we can use "video_packed" to get RGB/RGBA buffer.
// libav.js Frame object has data.
// Let's stick to the prompt: "extract single images".
// I'll return a Blob of a JPEG.
// Since I have a valid frame, I can use a simple JS JPEG encoder or libav.
// Using libav is robust.
// Let's try to grab the PACKET directly from the read step?
// If the packet is a full JPEG, we can just dump it.
// MJPEG stream packets usually miss Huffman tables (DHT). Browsers often fail to render them directly.
// So safe bet: Decode -> Encode to JPEG (single frame).
// Reuse the decoder context? No, that's for decoding.
// Create a throwaway encoder for the snapshot.
const width = await libav.AVCodecContext_width(c)
const height = await libav.AVCodecContext_height(c)
// We need to know the pixel format of the decoded frame.
// usually YUVJ420P.
const [, enc_c, enc_frame, enc_pkt] = await libav.ff_init_encoder(
"mjpeg",
{
ctx: {
width,
height,
pix_fmt: 0, // YUV420P
time_base: [1, 1],
framerate: [1, 1],
},
options: {
// Force standard JPEG tables
qscale: "2",
},
}
)
const encodedPackets = await libav.ff_encode_multi(
enc_c,
enc_frame,
enc_pkt,
[foundFrame]
)
// Combine packets if multiple (usually 1 for JPEG)
// But actually, ff_encode_multi returns packets.
// We can concat their data.
const chunks = []
for (const p of encodedPackets) {
// p.data is the pointer. We need to copy it out.
// Wait, ff_encode_multi returns objects with .data usually if copyoutPacket is default?
// No, it returns Packet objects which refer to memory.
// We need to copy data out.
const data = await libav.ff_copyout_packet(p.ptr) // or just p.data if it's already copied?
// libav.js docs: "Most functions will instead copy out their data into libav.js Frame or Packet objects."
// So `encodedPackets` are JS objects with `data` (Uint8Array).
console.log(data)
chunks.push(p.data)
}
const blob = new Blob(chunks, { type: "image/jpeg" })
ctx.postMessage({
type: "FRAME",
data: blob,
time: time,
} as WorkerResponse)
await libav.ff_free_encoder(enc_c, enc_frame, enc_pkt)
} else {
console.warn("Frame not found after seek")
}
// Cleanup extraction
await libav.ff_free_decoder(c, pkt, frame)
await libav.avformat_close_input_js(fmt_ctx)
} catch (e) {
console.error("Extraction error", e)
}
}

127
app/component/label/video/page.tsx
View File

@@ -1,127 +0,0 @@
"use client"
import { ChangeEvent, useEffect, useRef, useState } from "react"
import type {
WorkerMessage,
WorkerResponse,
} from "@/app/component/label/video/types/libav"
export default function Home() {
const workerRef = useRef<Worker | null>(null)
const [ready, setReady] = useState(false)
const [logs, setLogs] = useState<string[]>([])
const [videoUrl, setVideoUrl] = useState<string | null>(null)
const [processing, setProcessing] = useState(false)
const addLog = (msg: string) => {
setLogs((prev) => [...prev.slice(-10), msg]) // 只保留最近10条
}
useEffect(() => {
// 初始化 Worker
// Next.js 会自动检测到这个语法并单独打包 Worker
workerRef.current = new Worker(
new URL("./libav.worker.ts", import.meta.url)
)
// 设置消息监听
workerRef.current.onmessage = (event: MessageEvent<WorkerResponse>) => {
const { type } = event.data
switch (type) {
case "READY":
setReady(true)
addLog("FFmpeg core loaded and ready.")
break
case "LOG":
if ("message" in event.data) addLog(event.data.message)
break
case "DONE":
if ("data" in event.data) {
const url = URL.createObjectURL(event.data.data)
setVideoUrl(url)
setProcessing(false)
addLog("Transcoding complete!")
}
break
case "ERROR":
if ("error" in event.data) {
addLog(`Error: ${event.data.error}`)
setProcessing(false)
}
break
}
}
// 告诉 Worker 加载 libav
// 这里的路径必须指向 public 目录下的实际文件
// 使用 default-cli 变体以确保包含 MP4/H.264 等常见格式的解封装器和解码器
const LIBAV_PATH = "/libav/libav-6.8.8.0-default.js"
workerRef.current.postMessage({
type: "LOAD",
config: { corePath: location.origin + LIBAV_PATH },
} as WorkerMessage)
return () => {
workerRef.current?.terminate()
}
}, [])
const handleFileChange = (e: ChangeEvent<HTMLInputElement>) => {
const file = e.target.files?.[0]
if (!file || !workerRef.current || !ready) return
setProcessing(true)
setVideoUrl(null)
setLogs([])
addLog(`Starting upload: ${file.name}`)
workerRef.current.postMessage({
type: "TRANSCODE",
file: file,
} as WorkerMessage)
}
return (
<main className="p-8 font-mono">
<h1 className="text-2xl font-bold mb-4">Next.js + Libav.js + Worker</h1>
<div className="mb-4 p-4 border rounded bg-gray-50">
<p>
Status:{" "}
{ready ? (
<span className="text-green-600">Ready</span>
) : (
<span className="text-yellow-600">Loading Core...</span>
)}
</p>
<p>Processing: {processing ? "Yes" : "No"}</p>
</div>
<input
type="file"
onChange={handleFileChange}
disabled={!ready || processing}
className="mb-4 block w-full text-sm text-slate-500
file:mr-4 file:py-2 file:px-4
file:rounded-full file:border-0
file:text-sm file:font-semibold
file:bg-violet-50 file:text-violet-700
hover:file:bg-violet-100"
/>
{videoUrl && (
<div className="mb-4">
<h2 className="font-bold">Result:</h2>
<video controls src={videoUrl} className="w-full max-w-md border" />
</div>
)}
<div className="bg-black text-white p-4 rounded text-xs h-64 overflow-y-auto">
{logs.map((log, i) => (
<div key={i} className="border-b border-gray-800 py-1">
{log}
</div>
))}
</div>
</main>
)
}

74
app/component/label/video/types/libav.d.ts vendored
View File

@@ -1,74 +0,0 @@
// src/types/libav.d.ts
export type WorkerMessage =
| { type: "LOAD"; config: { corePath: string } }
| { type: "TRANSCODE"; file: File }
| { type: "EXTRACT_FRAME"; time: number }
export type WorkerResponse =
| { type: "READY" }
| { type: "LOG"; message: string }
| { type: "DONE"; data: any } // data 类型根据实际返回调整
| { type: "FRAME"; data: Blob; time: number }
| { type: "ERROR"; error: string }
declare global {
interface Window {
LibAV: LibAVWrapper
}
}
export interface LibAVOpts {
noworker?: boolean
base?: string
wasmurl?: string
variant?: string
}
export interface LibAVWrapper {
LibAV(opts?: LibAVOpts): Promise<LibAVInstance>
}
// 核心实例接口：添加官方例子中用到的底层方法
export interface LibAVInstance {
writeFile(name: string, content: Uint8Array): Promise<void>
unlink(name: string): Promise<void>
terminate(): void
// 1. 初始化解封装器 (Demuxer)
// 返回 [Context, Streams[]]
ff_init_demuxer_file(filename: string): Promise<[number, any[]]>
// 2. 初始化解码器 (Decoder)
// 返回 [Codec, CodecContext, Packet, Frame]
ff_init_decoder(
codec_id: number,
codecpar: number
): Promise<[number, number, number, number]>
// 3. 读取多个包
// 返回 [ResultCode, PacketsMap]
ff_read_frame_multi(
fmt_ctx: number,
pkt: number,
opts?: {
limit?: number // OUTPUT limit, in bytes
unify?: boolean // If true, unify the packets into a single stream (called 0), so that the output is in the same order as the input
copyoutPacket?: "default" // Version of ff_copyout_packet to use
}
): Promise<[number, Record<number, Packet[]>]>
// 4. 解码多个包
// 返回 DecodedFrames[]
ff_decode_multi(
ctx: number,
pkt: number,
frame: number,
packets: any[],
fin?: boolean
): Promise<any[]>
// 辅助：释放资源
avformat_close_input_js(fmt_ctx: number): Promise<void>
ff_free_decoder(c: number, pkt: number, frame: number): Promise<void>
}

196
app/component/label/video2image/processVideo.ts Normal file
View File

@@ -0,0 +1,196 @@
"use client"
import type { WorkerMessage, WorkerResponse } from "../ffmpeg/ffmpeg.worker"
export interface VideoFrameFile {
name: string
dataUrl: string
}
interface ProcessVideoParams {
base64Data: string
fileName: string
namePrefix?: string
baseURL?: string
args?: string[]
}
const defaultArgs = [
"-vf",
"fps=1,scale='min(720,iw)':-1",
"-q:v",
"2",
"frame_%03d.jpg",
]
const mimeByExt: Record<string, string> = {
jpg: "image/jpeg",
jpeg: "image/jpeg",
png: "image/png",
webp: "image/webp",
mp4: "video/mp4",
mov: "video/quicktime",
webm: "video/webm",
mkv: "video/x-matroska",
avi: "video/x-msvideo",
h264: "video/h264",
"264": "video/h264",
}
const getFileExt = (fileName: string) => {
const segments = fileName.split(".")
if (segments.length <= 1) return ""
return segments[segments.length - 1].toLowerCase()
}
const stripFileExt = (fileName: string) => {
const index = fileName.lastIndexOf(".")
if (index <= 0) return fileName
return fileName.slice(0, index)
}
const normalizeBase64 = (text: string) => {
if (!text) return ""
const raw = text.includes(",") ? text.split(",").slice(1).join(",") : text
return raw.replace(/\s+/g, "")
}
const sanitizePrefix = (prefix: string) => {
const safe = prefix
.replace(/[^a-zA-Z0-9_-]+/g, "_")
.replace(/_+/g, "_")
.replace(/^_|_$/g, "")
return safe || "video"
}
const uint8ArrayToBase64 = (arr: Uint8Array) => {
const chunkSize = 0x8000
let binary = ""
for (let i = 0; i < arr.length; i += chunkSize) {
binary += String.fromCharCode(...arr.subarray(i, i + chunkSize))
}
return btoa(binary)
}
const base64ToFile = (base64Data: string, fileName: string) => {
const payload = normalizeBase64(base64Data)
if (!payload) throw new Error("empty video payload")
const binary = atob(payload)
const len = binary.length
const bytes = new Uint8Array(len)
for (let i = 0; i < len; i += 1) {
bytes[i] = binary.charCodeAt(i)
}
const fileExt = getFileExt(fileName)
return new File([bytes], fileName, {
type: mimeByExt[fileExt] || "video/mp4",
})
}
const getOutputExtFromArgs = (args: string[]) => {
for (let i = args.length - 1; i >= 0; i -= 1) {
const ext = getFileExt(args[i])
if (ext) return ext
}
return "jpg"
}
export const processVideo = async (
params: ProcessVideoParams
): Promise<VideoFrameFile[]> => {
if (typeof window === "undefined") return []
const args = params.args && params.args.length ? params.args : defaultArgs
const outputExt = getOutputExtFromArgs(args)
const prefix = sanitizePrefix(
params.namePrefix || stripFileExt(params.fileName)
)
const baseURL = params.baseURL || `${window.location.origin}/wasm`
const inputFile = base64ToFile(params.base64Data, params.fileName)
const worker = new Worker(
new URL("../ffmpeg/ffmpeg.worker.ts", import.meta.url)
)
let frameIndex = 0
const frames: VideoFrameFile[] = []
try {
return await new Promise<VideoFrameFile[]>((resolve, reject) => {
let settled = false
const finish = (next: () => void) => {
if (settled) return
settled = true
next()
}
worker.onmessage = (event: MessageEvent<WorkerResponse>) => {
const message = event.data
switch (message.type) {
case "READY": {
const runMessage: WorkerMessage = {
type: "EXEC",
file: inputFile,
args,
outputPattern: "frame_",
}
worker.postMessage(runMessage)
return
}
case "SEGMENT_DATA": {
const files = message.files || []
files.forEach((file) => {
const ext = getFileExt(file.name) || outputExt || "jpg"
const frameName = `${prefix}_frame_${String(frameIndex).padStart(
6,
"0"
)}.${ext}`
frameIndex += 1
frames.push({
name: frameName,
dataUrl: `data:${mimeByExt[ext] || "image/jpeg"};base64,${uint8ArrayToBase64(file.data)}`,
})
})
return
}
case "DONE": {
finish(() => {
if (!frames.length) {
reject(new Error("video has no extracted frames"))
} else {
resolve(frames)
}
})
return
}
case "FATAL": {
finish(() =>
reject(new Error(message.error || "video process fatal"))
)
return
}
case "ERROR": {
finish(() =>
reject(new Error(message.error || "video process error"))
)
return
}
default:
return
}
}
worker.onerror = (event) => {
finish(() => reject(new Error(event.message || "video worker error")))
}
const loadMessage: WorkerMessage = {
type: "LOAD",
baseURL,
}
worker.postMessage(loadMessage)
})
} finally {
worker.terminate()
}
}

116
components/label/LabelNossr.tsx
View File

@@ -12,12 +12,13 @@ import ScaleComponent from "./components/ScaleComponent"
import { Box, Flex, LoadingOverlay, Stack } from "@mantine/core" import { Box, Flex, LoadingOverlay, Stack } from "@mantine/core"
import { showNotification } from "@mantine/notifications" import { showNotification } from "@mantine/notifications"
import { getLabelResult } from "./api/label" import { getLabelResult, getServerImage } from "./api/label"
import { Comment, WorkLoad } from "./api/label/typing" import { Comment, WorkLoad } from "./api/label/typing"
import { getProjectDetail } from "./api/project" import { getProjectDetail } from "./api/project"
import { Project } from "./api/project/typing" import { Project } from "./api/project/typing"
import { getTaskList, getTaskWorkFlow } from "./api/task" import { getTaskList, getTaskWorkFlow } from "./api/task"
import { Task } from "./api/task/typing" import { Task } from "./api/task/typing"
import { processVideo } from "@/app/component/label/video2image/processVideo"
import BottomTools from "./components/BottomTools" import BottomTools from "./components/BottomTools"
import ConfirmModal from "./components/ConfirmModal" import ConfirmModal from "./components/ConfirmModal"
import { splitWord } from "./components/EditorContainer" import { splitWord } from "./components/EditorContainer"
@@ -29,7 +30,13 @@ import RightQATools from "./components/RightQATools"
import RightTaskTools from "./components/RightTaskTools" import RightTaskTools from "./components/RightTaskTools"
import ScaleToolContainer from "./components/ScaleToolContainer" import ScaleToolContainer from "./components/ScaleToolContainer"
import TopTools from "./components/TopTools" import TopTools from "./components/TopTools"
import { useKeyEventStore, useLabelStore, useObjectStore } from "./store" import {
useImagesStore,
useKeyEventStore,
useLabelStore,
useObjectStore,
useVideoFrameStore,
} from "./store"
import { usePermissionStore } from "./store/auth" import { usePermissionStore } from "./store/auth"
import { useBottomToolsStore } from "./useBottomToolsStore" import { useBottomToolsStore } from "./useBottomToolsStore"
import { useDescToolsStore } from "./useDescToolsStore" import { useDescToolsStore } from "./useDescToolsStore"
@@ -74,6 +81,16 @@ const getSegmentPoints = (arr: Array<number[]>) => {
return arr.map((item) => [item[0], item[1]]) return arr.map((item) => [item[0], item[1]])
} }
const videoExtPattern = /\.(mp4|mov|avi|mkv|webm|h264|264|ts|m4v)$/i
const normalizeNamePrefix = (name: string) => {
const value = name
.replace(/[^a-zA-Z0-9_-]+/g, "_")
.replace(/_+/g, "_")
.replace(/^_|_$/g, "")
return value || "video"
}
interface LabelProps { interface LabelProps {
headerHeight: number headerHeight: number
leftWidth: number leftWidth: number
@@ -218,6 +235,91 @@ const LabelPage = ({
setQaOperations, setQaOperations,
]) ])
const normalizeVideoTaskData = useCallback(
async (detail: Task.DataProps) => {
try {
const sourceNames = detail.data_name
.map((item) => item?.name?.[0] || "")
.filter((name) => !!name)
if (!sourceNames.length || !projectId) return detail
const taskDataType = Number(
(detail as { [key: string]: any })?.data_type ??
(detail as { [key: string]: any })?.task_data_type ??
-1
)
const isVideoTask =
taskDataType === 2 ||
sourceNames.some((name) => videoExtPattern.test(name))
if (!isVideoTask) return detail
const imagesMap = new Map(useImagesStore.getState().images)
let imageMapUpdated = false
const normalizedDataName: Task.DataProps["data_name"] = []
for (const videoName of sourceNames) {
const cacheKey = `${projectId}:${taskId}:${videoName}`
let frameNames =
useVideoFrameStore.getState().videoFrames.get(cacheKey) ?? []
const hasAllCachedFrames =
frameNames.length > 0 &&
frameNames.every((frameName) => {
const cache = imagesMap.get(frameName)
return typeof cache === "string" && cache.length > 0
})
if (!hasAllCachedFrames) {
const base64Video = await getServerImage({
data_names: [videoName],
data_type: 2,
project_id: projectId,
})
const frameData = await processVideo({
base64Data: String(base64Video || ""),
fileName: videoName,
namePrefix: normalizeNamePrefix(
`${projectId}_${taskId}_${videoName.replace(/\.[^.]+$/, "")}`
),
})
frameNames = frameData.map((frame) => frame.name)
frameData.forEach((frame) => {
imagesMap.set(frame.name, frame.dataUrl)
})
imageMapUpdated = true
useVideoFrameStore.getState().setVideoFrames(cacheKey, frameNames)
}
frameNames.forEach((frameName) => {
normalizedDataName.push({
name: [frameName] as [string],
related_images: [],
})
})
}
if (imageMapUpdated) {
useImagesStore.getState().setImages(imagesMap)
}
if (!normalizedDataName.length) return detail
return {
...detail,
data_name: normalizedDataName,
}
} catch (error) {
console.log(error)
showNotification({
color: "red",
title: "视频解码失败",
message: "视频帧生成失败，已回退到原始数据",
})
return detail
}
},
[projectId, taskId]
)
const asyncGetTaskDetail = useCallback(async () => { const asyncGetTaskDetail = useCallback(async () => {
if (taskId) { if (taskId) {
try { try {
@@ -230,6 +332,7 @@ const LabelPage = ({
page_size: 1, page_size: 1,
}) })
let detail = res.task_list[0] let detail = res.task_list[0]
detail = await normalizeVideoTaskData(detail)
setTaskDetail(detail) setTaskDetail(detail)
// 帧列表 // 帧列表
useBottomToolsStore useBottomToolsStore
@@ -579,7 +682,14 @@ const LabelPage = ({
usePaperStore.getState().setLoadingData(false) usePaperStore.getState().setLoadingData(false)
} }
} }
}, [setLabel, setLabelTime, setStateStack, taskId, user_id]) }, [
normalizeVideoTaskData,
setLabel,
setLabelTime,
setStateStack,
taskId,
user_id,
])
// 保存时更新当前workload // 保存时更新当前workload
const updateOldWorkLoad = (data: WorkLoad) => { const updateOldWorkLoad = (data: WorkLoad) => {

3
components/label/api/task/typing.ts
View File

@@ -2,6 +2,8 @@ export namespace Task {
export interface DataProps { export interface DataProps {
id: number id: number
create_date: string create_date: string
data_type?: number
task_data_type?: number
data_name: { data_name: {
name: [string] name: [string]
related_images: [] related_images: []
@@ -62,6 +64,7 @@ export namespace Task {
} }
export interface ListRequest { export interface ListRequest {
data_type?: number
current_uid?: number[] current_uid?: number[]
data_name?: string data_name?: string
first_commit_label_time_end?: string first_commit_label_time_end?: string

36
components/label/store.ts
View File

@@ -498,6 +498,12 @@ interface ImageState {
setImages: (value: any) => void setImages: (value: any) => void
} }
interface VideoFrameState {
videoFrames: Map<string, string[]>
setVideoFrames: (key: string, value: string[]) => void
removeVideoFrames: (key: string) => void
}
export const useImagesStore = create( export const useImagesStore = create(
persist<ImageState>( persist<ImageState>(
(storeSet) => ({ (storeSet) => ({
@@ -516,3 +522,33 @@ export const useImagesStore = create(
} }
) )
) )
export const useVideoFrameStore = create(
persist<VideoFrameState>(
(storeSet) => ({
videoFrames: new Map(),
setVideoFrames: (key: string, value: string[]) =>
storeSet((state) => {
const nextMap = new Map(state.videoFrames)
nextMap.set(key, value)
return {
...state,
videoFrames: nextMap,
}
}),
removeVideoFrames: (key: string) =>
storeSet((state) => {
const nextMap = new Map(state.videoFrames)
nextMap.delete(key)
return {
...state,
videoFrames: nextMap,
}
}),
}),
{
name: "video-frame-storage",
storage: storage,
}
)
)