feat(label): video process
This commit is contained in:
@@ -1,410 +0,0 @@
|
||||
// app/component/label/video/libav.worker.ts
|
||||
import type {
|
||||
LibAVWrapper,
|
||||
WorkerMessage,
|
||||
WorkerResponse,
|
||||
} from "@/app/component/label/video/types/libav"
|
||||
|
||||
const ctx: Worker = self as any
|
||||
let libav: any = null // Use any to access full API
|
||||
const PROXY_FILE = "proxy.avi"
|
||||
|
||||
ctx.onmessage = async (event: MessageEvent<WorkerMessage>) => {
|
||||
const { type } = event.data
|
||||
|
||||
try {
|
||||
switch (type) {
|
||||
case "LOAD":
|
||||
await loadLibAV(event.data.config.corePath)
|
||||
break
|
||||
case "TRANSCODE":
|
||||
if (!libav) throw new Error("LibAV not loaded")
|
||||
if ("file" in event.data) {
|
||||
await handleTranscode(event.data.file)
|
||||
}
|
||||
break
|
||||
case "EXTRACT_FRAME":
|
||||
if (!libav) throw new Error("LibAV not loaded")
|
||||
if ("time" in event.data) {
|
||||
await handleExtractFrame(event.data.time)
|
||||
}
|
||||
break
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.error("Worker Error:", err)
|
||||
ctx.postMessage({
|
||||
type: "ERROR",
|
||||
error: err.message || "Unknown error",
|
||||
} as WorkerResponse)
|
||||
}
|
||||
}
|
||||
|
||||
async function loadLibAV(corePath: string) {
|
||||
if (libav) {
|
||||
ctx.postMessage({ type: "READY" } as WorkerResponse)
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
importScripts(corePath)
|
||||
|
||||
const LibAVWrapper = (self as any).LibAV as LibAVWrapper
|
||||
if (!LibAVWrapper || typeof LibAVWrapper.LibAV !== "function") {
|
||||
throw new Error("LibAV factory not found.")
|
||||
}
|
||||
|
||||
const baseUrl = corePath.substring(0, corePath.lastIndexOf("/") + 1)
|
||||
// Dynamic WASM URL based on the JS file name (standard convention)
|
||||
const wasmUrl = corePath.replace(".js", ".wasm.wasm")
|
||||
|
||||
libav = await LibAVWrapper.LibAV({
|
||||
noworker: true,
|
||||
base: baseUrl,
|
||||
wasmurl: wasmUrl,
|
||||
})
|
||||
|
||||
ctx.postMessage({ type: "READY" } as WorkerResponse)
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to load libav: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
async function handleTranscode(file: File) {
|
||||
const ext = file.name.match(/\.([^.]+)$/)?.[0] || ".mp4"
|
||||
const inputFile = `input_video${ext}`
|
||||
|
||||
try {
|
||||
// 1. Write Input File
|
||||
const fileBuffer = await file.arrayBuffer()
|
||||
const u8Arr = new Uint8Array(fileBuffer)
|
||||
console.log(`[Worker] Received file: ${file.name}, size: ${u8Arr.length}`)
|
||||
|
||||
await libav.writeFile(inputFile, u8Arr)
|
||||
|
||||
// Note: libav.js (webcodecs-avf) does not support readdir/stat via API directly
|
||||
// but readFile works. We trust writeFile worked.
|
||||
|
||||
// 2. Init Demuxer (Input)
|
||||
// webcodecs-avf SHOULD have the demuxer for MP4.
|
||||
let fmt_ctx, streams
|
||||
try {
|
||||
;[fmt_ctx, streams] = await libav.ff_init_demuxer_file(inputFile)
|
||||
} catch (e) {
|
||||
console.error("Demuxer init failed:", e)
|
||||
throw new Error(
|
||||
`Demuxer failed. Libav variant might lack MP4 support. Error: ${e}`
|
||||
)
|
||||
}
|
||||
|
||||
// Find Video Stream
|
||||
let videoStreamIdx = -1
|
||||
let videoStream = null
|
||||
for (let i = 0; i < streams.length; i++) {
|
||||
if (streams[i].codec_type === 0) {
|
||||
// 0 is Video
|
||||
videoStreamIdx = i
|
||||
videoStream = streams[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if (videoStreamIdx === -1) throw new Error("No video stream found")
|
||||
|
||||
console.log(
|
||||
`[Worker] Found video stream #${videoStreamIdx}, codec_id=${videoStream.codec_id}`
|
||||
)
|
||||
|
||||
// 3. Init Decoder (Input)
|
||||
// For obsolete variant, we might have software H.264 decoders.
|
||||
// Try standard init first.
|
||||
|
||||
let c, pkt, frame
|
||||
try {
|
||||
// Try standard init first
|
||||
;[, c, pkt, frame] = await libav.ff_init_decoder(
|
||||
videoStream.codec_id,
|
||||
videoStream.codecpar
|
||||
)
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
"[Worker] Standard decoder init failed, trying explicit H264...",
|
||||
e
|
||||
)
|
||||
try {
|
||||
// Try explicit H264
|
||||
;[, c, pkt, frame] = await libav.ff_init_decoder(
|
||||
"h264",
|
||||
videoStream.codecpar
|
||||
)
|
||||
} catch (e2) {
|
||||
console.error("[Worker] All decoder init attempts failed", e2)
|
||||
throw new Error("Codec not found (decoder init failed).")
|
||||
}
|
||||
}
|
||||
|
||||
// Get input properties
|
||||
let width = await libav.AVCodecParameters_width(videoStream.codecpar)
|
||||
let height = await libav.AVCodecParameters_height(videoStream.codecpar)
|
||||
|
||||
console.log(`[Worker] Video dimensions: ${width}x${height}`)
|
||||
|
||||
// 4. Init Encoder (Output: MJPEG)
|
||||
const codec_name = "mjpeg"
|
||||
const [, enc_c, enc_frame, enc_pkt] = await libav.ff_init_encoder(
|
||||
codec_name,
|
||||
{
|
||||
ctx: {
|
||||
width: width,
|
||||
height: height,
|
||||
pix_fmt: 0, // AV_PIX_FMT_YUV420P
|
||||
time_base: [1, 25],
|
||||
framerate: [25, 1],
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
// 5. Init Muxer (Output: AVI container)
|
||||
const [oc, , pb, [stream_ctx]] = await libav.ff_init_muxer(
|
||||
{ format_name: "avi", filename: PROXY_FILE, open: true },
|
||||
[[enc_c, 1, 25]]
|
||||
)
|
||||
|
||||
console.log(stream_ctx)
|
||||
|
||||
// Write Header
|
||||
await libav.avformat_write_header(oc, 0)
|
||||
|
||||
ctx.postMessage({
|
||||
type: "LOG",
|
||||
message: "Transcoding started...",
|
||||
} as WorkerResponse)
|
||||
|
||||
// 6. Loop: Read -> Decode -> Encode -> Write
|
||||
let frameCount = 0
|
||||
|
||||
// We need to use "video" copyout mode for WebCodecs to work efficiently?
|
||||
// Actually, ff_decode_multi returns Frame objects.
|
||||
// If WebCodecs is used, the Frame might be backed by VideoFrame.
|
||||
|
||||
while (true) {
|
||||
const [res, packets] = await libav.ff_read_frame_multi(fmt_ctx, pkt)
|
||||
|
||||
if (res === libav.AVERROR_EOF) break
|
||||
if (res < 0 && res !== -11) break
|
||||
|
||||
const streamPackets = packets[videoStreamIdx]
|
||||
if (!streamPackets || streamPackets.length === 0) continue
|
||||
|
||||
// Decode
|
||||
const decodedFrames = await libav.ff_decode_multi(
|
||||
c,
|
||||
pkt,
|
||||
frame,
|
||||
streamPackets
|
||||
)
|
||||
|
||||
for (const f of decodedFrames) {
|
||||
console.log(f)
|
||||
|
||||
// Encode
|
||||
const encodedPackets = await libav.ff_encode_multi(
|
||||
enc_c,
|
||||
enc_frame,
|
||||
enc_pkt
|
||||
)
|
||||
|
||||
for (const p of encodedPackets) {
|
||||
p.stream_index = 0
|
||||
}
|
||||
|
||||
await libav.ff_write_multi(oc, enc_pkt, encodedPackets)
|
||||
frameCount++
|
||||
}
|
||||
|
||||
if (frameCount % 30 === 0) {
|
||||
ctx.postMessage({
|
||||
type: "LOG",
|
||||
message: `Transcoded ${frameCount} frames...`,
|
||||
} as WorkerResponse)
|
||||
}
|
||||
}
|
||||
|
||||
// Flush Encoder
|
||||
const flushedPackets = await libav.ff_encode_multi(
|
||||
enc_c,
|
||||
enc_frame,
|
||||
enc_pkt,
|
||||
[],
|
||||
true
|
||||
)
|
||||
if (flushedPackets.length > 0) {
|
||||
for (const p of flushedPackets) p.stream_index = 0
|
||||
await libav.ff_write_multi(oc, enc_pkt, flushedPackets)
|
||||
}
|
||||
|
||||
// Write Trailer
|
||||
await libav.av_write_trailer(oc)
|
||||
|
||||
// Cleanup
|
||||
await libav.ff_free_muxer(oc, pb)
|
||||
await libav.ff_free_encoder(enc_c, enc_frame, enc_pkt)
|
||||
await libav.ff_free_decoder(c, pkt, frame)
|
||||
await libav.avformat_close_input_js(fmt_ctx)
|
||||
await libav.unlink(inputFile)
|
||||
|
||||
ctx.postMessage({
|
||||
type: "DONE",
|
||||
data: `Transcoding complete. ${frameCount} frames ready.`,
|
||||
} as WorkerResponse)
|
||||
} catch (e: any) {
|
||||
console.error(e)
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
async function handleExtractFrame(time: number) {
|
||||
if (!libav) return
|
||||
|
||||
try {
|
||||
// 1. Open Proxy File
|
||||
// Note: For efficiency in a real app, keep this open.
|
||||
// But for safety here, we open/close per request or keep it open in global?
|
||||
// Let's open it fresh. AVI parsing is fast.
|
||||
const [fmt_ctx, streams] = await libav.ff_init_demuxer_file(PROXY_FILE)
|
||||
|
||||
const streamIdx = 0 // We know we only have 1 video stream
|
||||
const stream = streams[streamIdx]
|
||||
console.log(stream)
|
||||
|
||||
// 2. Init Decoder (MJPEG)
|
||||
const [, c, pkt, frame] = await libav.ff_init_decoder("mjpeg")
|
||||
|
||||
// 3. Seek
|
||||
// Calculate timestamp in AV_TIME_BASE or stream time base
|
||||
// av_seek_frame(ctx, stream_index, timestamp, flags)
|
||||
// If stream_index is -1, timestamp is in AV_TIME_BASE (microseconds)
|
||||
// We'll use stream index 0. We need to know its timebase.
|
||||
// But easier: Use -1 and AV_TIME_BASE
|
||||
const targetTS = Math.floor(time * 1000000) // seconds -> microseconds
|
||||
|
||||
// seek flags: 1 (BACKWARD) is usually good to find keyframe before time
|
||||
await libav.av_seek_frame(fmt_ctx, -1, targetTS, 1)
|
||||
|
||||
// 4. Read & Decode until we hit the frame
|
||||
// Since MJPEG is all-intra, the first frame we read after seek should be the one (or close)
|
||||
let foundFrame = null
|
||||
|
||||
// Limit attempts
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const [res, packets] = await libav.ff_read_frame_multi(fmt_ctx, pkt, {
|
||||
limit: 1,
|
||||
})
|
||||
if (res === libav.AVERROR_EOF) break
|
||||
|
||||
const streamPackets = packets[streamIdx]
|
||||
if (!streamPackets || streamPackets.length === 0) continue
|
||||
|
||||
// Decode
|
||||
const frames = await libav.ff_decode_multi(c, pkt, frame, streamPackets)
|
||||
if (frames && frames.length > 0) {
|
||||
foundFrame = frames[0]
|
||||
break // Just take the first one found after seek
|
||||
}
|
||||
}
|
||||
|
||||
if (foundFrame) {
|
||||
// 5. Convert to Blob (BMP is simplest uncompressed format libav supports encoding to?)
|
||||
// Or use libav to encode to JPEG?
|
||||
// Actually, since we are in MJPEG, the packet *is* a JPEG.
|
||||
// But we just decoded it.
|
||||
// Let's re-encode the single frame to JPEG for the UI.
|
||||
// Or just return the raw packet?
|
||||
// Let's re-encode to be safe (ensure headers etc).
|
||||
|
||||
// Init single-frame encoder
|
||||
// const [, jpeg_c, jpeg_frame, jpeg_pkt] = await libav.ff_init_encoder("mjpeg", {
|
||||
// ctx: { width: foundFrame.width, height: foundFrame.height, ... }
|
||||
// })
|
||||
// This is heavy.
|
||||
|
||||
// Alternative: "copyoutFrame" as "ImageData" (only in main thread).
|
||||
// In Worker, we can use "video_packed" to get RGB/RGBA buffer.
|
||||
// libav.js Frame object has data.
|
||||
// Let's stick to the prompt: "extract single images".
|
||||
// I'll return a Blob of a JPEG.
|
||||
// Since I have a valid frame, I can use a simple JS JPEG encoder or libav.
|
||||
// Using libav is robust.
|
||||
|
||||
// Let's try to grab the PACKET directly from the read step?
|
||||
// If the packet is a full JPEG, we can just dump it.
|
||||
// MJPEG stream packets usually miss Huffman tables (DHT). Browsers often fail to render them directly.
|
||||
// So safe bet: Decode -> Encode to JPEG (single frame).
|
||||
|
||||
// Reuse the decoder context? No, that's for decoding.
|
||||
// Create a throwaway encoder for the snapshot.
|
||||
const width = await libav.AVCodecContext_width(c)
|
||||
const height = await libav.AVCodecContext_height(c)
|
||||
|
||||
// We need to know the pixel format of the decoded frame.
|
||||
// usually YUVJ420P.
|
||||
|
||||
const [, enc_c, enc_frame, enc_pkt] = await libav.ff_init_encoder(
|
||||
"mjpeg",
|
||||
{
|
||||
ctx: {
|
||||
width,
|
||||
height,
|
||||
pix_fmt: 0, // YUV420P
|
||||
time_base: [1, 1],
|
||||
framerate: [1, 1],
|
||||
},
|
||||
options: {
|
||||
// Force standard JPEG tables
|
||||
qscale: "2",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
const encodedPackets = await libav.ff_encode_multi(
|
||||
enc_c,
|
||||
enc_frame,
|
||||
enc_pkt,
|
||||
[foundFrame]
|
||||
)
|
||||
|
||||
// Combine packets if multiple (usually 1 for JPEG)
|
||||
// But actually, ff_encode_multi returns packets.
|
||||
// We can concat their data.
|
||||
|
||||
const chunks = []
|
||||
for (const p of encodedPackets) {
|
||||
// p.data is the pointer. We need to copy it out.
|
||||
// Wait, ff_encode_multi returns objects with .data usually if copyoutPacket is default?
|
||||
// No, it returns Packet objects which refer to memory.
|
||||
// We need to copy data out.
|
||||
const data = await libav.ff_copyout_packet(p.ptr) // or just p.data if it's already copied?
|
||||
// libav.js docs: "Most functions will instead copy out their data into libav.js Frame or Packet objects."
|
||||
// So `encodedPackets` are JS objects with `data` (Uint8Array).
|
||||
console.log(data)
|
||||
chunks.push(p.data)
|
||||
}
|
||||
|
||||
const blob = new Blob(chunks, { type: "image/jpeg" })
|
||||
|
||||
ctx.postMessage({
|
||||
type: "FRAME",
|
||||
data: blob,
|
||||
time: time,
|
||||
} as WorkerResponse)
|
||||
|
||||
await libav.ff_free_encoder(enc_c, enc_frame, enc_pkt)
|
||||
} else {
|
||||
console.warn("Frame not found after seek")
|
||||
}
|
||||
|
||||
// Cleanup extraction
|
||||
await libav.ff_free_decoder(c, pkt, frame)
|
||||
await libav.avformat_close_input_js(fmt_ctx)
|
||||
} catch (e) {
|
||||
console.error("Extraction error", e)
|
||||
}
|
||||
}
|
||||
@@ -1,127 +0,0 @@
|
||||
"use client"
|
||||
|
||||
import { ChangeEvent, useEffect, useRef, useState } from "react"
|
||||
import type {
|
||||
WorkerMessage,
|
||||
WorkerResponse,
|
||||
} from "@/app/component/label/video/types/libav"
|
||||
|
||||
export default function Home() {
|
||||
const workerRef = useRef<Worker | null>(null)
|
||||
const [ready, setReady] = useState(false)
|
||||
const [logs, setLogs] = useState<string[]>([])
|
||||
const [videoUrl, setVideoUrl] = useState<string | null>(null)
|
||||
const [processing, setProcessing] = useState(false)
|
||||
const addLog = (msg: string) => {
|
||||
setLogs((prev) => [...prev.slice(-10), msg]) // 只保留最近10条
|
||||
}
|
||||
useEffect(() => {
|
||||
// 初始化 Worker
|
||||
// Next.js 会自动检测到这个语法并单独打包 Worker
|
||||
workerRef.current = new Worker(
|
||||
new URL("./libav.worker.ts", import.meta.url)
|
||||
)
|
||||
|
||||
// 设置消息监听
|
||||
workerRef.current.onmessage = (event: MessageEvent<WorkerResponse>) => {
|
||||
const { type } = event.data
|
||||
|
||||
switch (type) {
|
||||
case "READY":
|
||||
setReady(true)
|
||||
addLog("FFmpeg core loaded and ready.")
|
||||
break
|
||||
case "LOG":
|
||||
if ("message" in event.data) addLog(event.data.message)
|
||||
break
|
||||
case "DONE":
|
||||
if ("data" in event.data) {
|
||||
const url = URL.createObjectURL(event.data.data)
|
||||
setVideoUrl(url)
|
||||
setProcessing(false)
|
||||
addLog("Transcoding complete!")
|
||||
}
|
||||
break
|
||||
case "ERROR":
|
||||
if ("error" in event.data) {
|
||||
addLog(`Error: ${event.data.error}`)
|
||||
setProcessing(false)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// 告诉 Worker 加载 libav
|
||||
// 这里的路径必须指向 public 目录下的实际文件
|
||||
// 使用 default-cli 变体以确保包含 MP4/H.264 等常见格式的解封装器和解码器
|
||||
const LIBAV_PATH = "/libav/libav-6.8.8.0-default.js"
|
||||
workerRef.current.postMessage({
|
||||
type: "LOAD",
|
||||
config: { corePath: location.origin + LIBAV_PATH },
|
||||
} as WorkerMessage)
|
||||
|
||||
return () => {
|
||||
workerRef.current?.terminate()
|
||||
}
|
||||
}, [])
|
||||
|
||||
const handleFileChange = (e: ChangeEvent<HTMLInputElement>) => {
|
||||
const file = e.target.files?.[0]
|
||||
if (!file || !workerRef.current || !ready) return
|
||||
|
||||
setProcessing(true)
|
||||
setVideoUrl(null)
|
||||
setLogs([])
|
||||
addLog(`Starting upload: ${file.name}`)
|
||||
|
||||
workerRef.current.postMessage({
|
||||
type: "TRANSCODE",
|
||||
file: file,
|
||||
} as WorkerMessage)
|
||||
}
|
||||
|
||||
return (
|
||||
<main className="p-8 font-mono">
|
||||
<h1 className="text-2xl font-bold mb-4">Next.js + Libav.js + Worker</h1>
|
||||
|
||||
<div className="mb-4 p-4 border rounded bg-gray-50">
|
||||
<p>
|
||||
Status:{" "}
|
||||
{ready ? (
|
||||
<span className="text-green-600">Ready</span>
|
||||
) : (
|
||||
<span className="text-yellow-600">Loading Core...</span>
|
||||
)}
|
||||
</p>
|
||||
<p>Processing: {processing ? "Yes" : "No"}</p>
|
||||
</div>
|
||||
|
||||
<input
|
||||
type="file"
|
||||
onChange={handleFileChange}
|
||||
disabled={!ready || processing}
|
||||
className="mb-4 block w-full text-sm text-slate-500
|
||||
file:mr-4 file:py-2 file:px-4
|
||||
file:rounded-full file:border-0
|
||||
file:text-sm file:font-semibold
|
||||
file:bg-violet-50 file:text-violet-700
|
||||
hover:file:bg-violet-100"
|
||||
/>
|
||||
|
||||
{videoUrl && (
|
||||
<div className="mb-4">
|
||||
<h2 className="font-bold">Result:</h2>
|
||||
<video controls src={videoUrl} className="w-full max-w-md border" />
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="bg-black text-white p-4 rounded text-xs h-64 overflow-y-auto">
|
||||
{logs.map((log, i) => (
|
||||
<div key={i} className="border-b border-gray-800 py-1">
|
||||
{log}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</main>
|
||||
)
|
||||
}
|
||||
74
app/component/label/video/types/libav.d.ts
vendored
74
app/component/label/video/types/libav.d.ts
vendored
@@ -1,74 +0,0 @@
|
||||
// src/types/libav.d.ts
|
||||
|
||||
export type WorkerMessage =
|
||||
| { type: "LOAD"; config: { corePath: string } }
|
||||
| { type: "TRANSCODE"; file: File }
|
||||
| { type: "EXTRACT_FRAME"; time: number }
|
||||
|
||||
export type WorkerResponse =
|
||||
| { type: "READY" }
|
||||
| { type: "LOG"; message: string }
|
||||
| { type: "DONE"; data: any } // data 类型根据实际返回调整
|
||||
| { type: "FRAME"; data: Blob; time: number }
|
||||
| { type: "ERROR"; error: string }
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
LibAV: LibAVWrapper
|
||||
}
|
||||
}
|
||||
|
||||
export interface LibAVOpts {
|
||||
noworker?: boolean
|
||||
base?: string
|
||||
wasmurl?: string
|
||||
variant?: string
|
||||
}
|
||||
|
||||
export interface LibAVWrapper {
|
||||
LibAV(opts?: LibAVOpts): Promise<LibAVInstance>
|
||||
}
|
||||
|
||||
// 核心实例接口：添加官方例子中用到的底层方法
|
||||
export interface LibAVInstance {
|
||||
writeFile(name: string, content: Uint8Array): Promise<void>
|
||||
unlink(name: string): Promise<void>
|
||||
terminate(): void
|
||||
|
||||
// 1. 初始化解封装器 (Demuxer)
|
||||
// 返回 [Context, Streams[]]
|
||||
ff_init_demuxer_file(filename: string): Promise<[number, any[]]>
|
||||
|
||||
// 2. 初始化解码器 (Decoder)
|
||||
// 返回 [Codec, CodecContext, Packet, Frame]
|
||||
ff_init_decoder(
|
||||
codec_id: number,
|
||||
codecpar: number
|
||||
): Promise<[number, number, number, number]>
|
||||
|
||||
// 3. 读取多个包
|
||||
// 返回 [ResultCode, PacketsMap]
|
||||
ff_read_frame_multi(
|
||||
fmt_ctx: number,
|
||||
pkt: number,
|
||||
opts?: {
|
||||
limit?: number // OUTPUT limit, in bytes
|
||||
unify?: boolean // If true, unify the packets into a single stream (called 0), so that the output is in the same order as the input
|
||||
copyoutPacket?: "default" // Version of ff_copyout_packet to use
|
||||
}
|
||||
): Promise<[number, Record<number, Packet[]>]>
|
||||
|
||||
// 4. 解码多个包
|
||||
// 返回 DecodedFrames[]
|
||||
ff_decode_multi(
|
||||
ctx: number,
|
||||
pkt: number,
|
||||
frame: number,
|
||||
packets: any[],
|
||||
fin?: boolean
|
||||
): Promise<any[]>
|
||||
|
||||
// 辅助：释放资源
|
||||
avformat_close_input_js(fmt_ctx: number): Promise<void>
|
||||
ff_free_decoder(c: number, pkt: number, frame: number): Promise<void>
|
||||
}
|
||||
196
app/component/label/video2image/processVideo.ts
Normal file
196
app/component/label/video2image/processVideo.ts
Normal file
@@ -0,0 +1,196 @@
|
||||
"use client"
|
||||
|
||||
import type { WorkerMessage, WorkerResponse } from "../ffmpeg/ffmpeg.worker"
|
||||
|
||||
export interface VideoFrameFile {
|
||||
name: string
|
||||
dataUrl: string
|
||||
}
|
||||
|
||||
interface ProcessVideoParams {
|
||||
base64Data: string
|
||||
fileName: string
|
||||
namePrefix?: string
|
||||
baseURL?: string
|
||||
args?: string[]
|
||||
}
|
||||
|
||||
const defaultArgs = [
|
||||
"-vf",
|
||||
"fps=1,scale='min(720,iw)':-1",
|
||||
"-q:v",
|
||||
"2",
|
||||
"frame_%03d.jpg",
|
||||
]
|
||||
|
||||
const mimeByExt: Record<string, string> = {
|
||||
jpg: "image/jpeg",
|
||||
jpeg: "image/jpeg",
|
||||
png: "image/png",
|
||||
webp: "image/webp",
|
||||
mp4: "video/mp4",
|
||||
mov: "video/quicktime",
|
||||
webm: "video/webm",
|
||||
mkv: "video/x-matroska",
|
||||
avi: "video/x-msvideo",
|
||||
h264: "video/h264",
|
||||
"264": "video/h264",
|
||||
}
|
||||
|
||||
const getFileExt = (fileName: string) => {
|
||||
const segments = fileName.split(".")
|
||||
if (segments.length <= 1) return ""
|
||||
return segments[segments.length - 1].toLowerCase()
|
||||
}
|
||||
|
||||
const stripFileExt = (fileName: string) => {
|
||||
const index = fileName.lastIndexOf(".")
|
||||
if (index <= 0) return fileName
|
||||
return fileName.slice(0, index)
|
||||
}
|
||||
|
||||
const normalizeBase64 = (text: string) => {
|
||||
if (!text) return ""
|
||||
const raw = text.includes(",") ? text.split(",").slice(1).join(",") : text
|
||||
return raw.replace(/\s+/g, "")
|
||||
}
|
||||
|
||||
const sanitizePrefix = (prefix: string) => {
|
||||
const safe = prefix
|
||||
.replace(/[^a-zA-Z0-9_-]+/g, "_")
|
||||
.replace(/_+/g, "_")
|
||||
.replace(/^_|_$/g, "")
|
||||
return safe || "video"
|
||||
}
|
||||
|
||||
const uint8ArrayToBase64 = (arr: Uint8Array) => {
|
||||
const chunkSize = 0x8000
|
||||
let binary = ""
|
||||
for (let i = 0; i < arr.length; i += chunkSize) {
|
||||
binary += String.fromCharCode(...arr.subarray(i, i + chunkSize))
|
||||
}
|
||||
return btoa(binary)
|
||||
}
|
||||
|
||||
const base64ToFile = (base64Data: string, fileName: string) => {
|
||||
const payload = normalizeBase64(base64Data)
|
||||
if (!payload) throw new Error("empty video payload")
|
||||
const binary = atob(payload)
|
||||
const len = binary.length
|
||||
const bytes = new Uint8Array(len)
|
||||
for (let i = 0; i < len; i += 1) {
|
||||
bytes[i] = binary.charCodeAt(i)
|
||||
}
|
||||
const fileExt = getFileExt(fileName)
|
||||
return new File([bytes], fileName, {
|
||||
type: mimeByExt[fileExt] || "video/mp4",
|
||||
})
|
||||
}
|
||||
|
||||
const getOutputExtFromArgs = (args: string[]) => {
|
||||
for (let i = args.length - 1; i >= 0; i -= 1) {
|
||||
const ext = getFileExt(args[i])
|
||||
if (ext) return ext
|
||||
}
|
||||
return "jpg"
|
||||
}
|
||||
|
||||
export const processVideo = async (
|
||||
params: ProcessVideoParams
|
||||
): Promise<VideoFrameFile[]> => {
|
||||
if (typeof window === "undefined") return []
|
||||
|
||||
const args = params.args && params.args.length ? params.args : defaultArgs
|
||||
const outputExt = getOutputExtFromArgs(args)
|
||||
const prefix = sanitizePrefix(
|
||||
params.namePrefix || stripFileExt(params.fileName)
|
||||
)
|
||||
const baseURL = params.baseURL || `${window.location.origin}/wasm`
|
||||
const inputFile = base64ToFile(params.base64Data, params.fileName)
|
||||
|
||||
const worker = new Worker(
|
||||
new URL("../ffmpeg/ffmpeg.worker.ts", import.meta.url)
|
||||
)
|
||||
let frameIndex = 0
|
||||
const frames: VideoFrameFile[] = []
|
||||
|
||||
try {
|
||||
return await new Promise<VideoFrameFile[]>((resolve, reject) => {
|
||||
let settled = false
|
||||
|
||||
const finish = (next: () => void) => {
|
||||
if (settled) return
|
||||
settled = true
|
||||
next()
|
||||
}
|
||||
|
||||
worker.onmessage = (event: MessageEvent<WorkerResponse>) => {
|
||||
const message = event.data
|
||||
switch (message.type) {
|
||||
case "READY": {
|
||||
const runMessage: WorkerMessage = {
|
||||
type: "EXEC",
|
||||
file: inputFile,
|
||||
args,
|
||||
outputPattern: "frame_",
|
||||
}
|
||||
worker.postMessage(runMessage)
|
||||
return
|
||||
}
|
||||
case "SEGMENT_DATA": {
|
||||
const files = message.files || []
|
||||
files.forEach((file) => {
|
||||
const ext = getFileExt(file.name) || outputExt || "jpg"
|
||||
const frameName = `${prefix}_frame_${String(frameIndex).padStart(
|
||||
6,
|
||||
"0"
|
||||
)}.${ext}`
|
||||
frameIndex += 1
|
||||
frames.push({
|
||||
name: frameName,
|
||||
dataUrl: `data:${mimeByExt[ext] || "image/jpeg"};base64,${uint8ArrayToBase64(file.data)}`,
|
||||
})
|
||||
})
|
||||
return
|
||||
}
|
||||
case "DONE": {
|
||||
finish(() => {
|
||||
if (!frames.length) {
|
||||
reject(new Error("video has no extracted frames"))
|
||||
} else {
|
||||
resolve(frames)
|
||||
}
|
||||
})
|
||||
return
|
||||
}
|
||||
case "FATAL": {
|
||||
finish(() =>
|
||||
reject(new Error(message.error || "video process fatal"))
|
||||
)
|
||||
return
|
||||
}
|
||||
case "ERROR": {
|
||||
finish(() =>
|
||||
reject(new Error(message.error || "video process error"))
|
||||
)
|
||||
return
|
||||
}
|
||||
default:
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
worker.onerror = (event) => {
|
||||
finish(() => reject(new Error(event.message || "video worker error")))
|
||||
}
|
||||
|
||||
const loadMessage: WorkerMessage = {
|
||||
type: "LOAD",
|
||||
baseURL,
|
||||
}
|
||||
worker.postMessage(loadMessage)
|
||||
})
|
||||
} finally {
|
||||
worker.terminate()
|
||||
}
|
||||
}
|
||||
@@ -12,12 +12,13 @@ import ScaleComponent from "./components/ScaleComponent"
|
||||
|
||||
import { Box, Flex, LoadingOverlay, Stack } from "@mantine/core"
|
||||
import { showNotification } from "@mantine/notifications"
|
||||
import { getLabelResult } from "./api/label"
|
||||
import { getLabelResult, getServerImage } from "./api/label"
|
||||
import { Comment, WorkLoad } from "./api/label/typing"
|
||||
import { getProjectDetail } from "./api/project"
|
||||
import { Project } from "./api/project/typing"
|
||||
import { getTaskList, getTaskWorkFlow } from "./api/task"
|
||||
import { Task } from "./api/task/typing"
|
||||
import { processVideo } from "@/app/component/label/video2image/processVideo"
|
||||
import BottomTools from "./components/BottomTools"
|
||||
import ConfirmModal from "./components/ConfirmModal"
|
||||
import { splitWord } from "./components/EditorContainer"
|
||||
@@ -29,7 +30,13 @@ import RightQATools from "./components/RightQATools"
|
||||
import RightTaskTools from "./components/RightTaskTools"
|
||||
import ScaleToolContainer from "./components/ScaleToolContainer"
|
||||
import TopTools from "./components/TopTools"
|
||||
import { useKeyEventStore, useLabelStore, useObjectStore } from "./store"
|
||||
import {
|
||||
useImagesStore,
|
||||
useKeyEventStore,
|
||||
useLabelStore,
|
||||
useObjectStore,
|
||||
useVideoFrameStore,
|
||||
} from "./store"
|
||||
import { usePermissionStore } from "./store/auth"
|
||||
import { useBottomToolsStore } from "./useBottomToolsStore"
|
||||
import { useDescToolsStore } from "./useDescToolsStore"
|
||||
@@ -74,6 +81,16 @@ const getSegmentPoints = (arr: Array<number[]>) => {
|
||||
return arr.map((item) => [item[0], item[1]])
|
||||
}
|
||||
|
||||
const videoExtPattern = /\.(mp4|mov|avi|mkv|webm|h264|264|ts|m4v)$/i
|
||||
|
||||
const normalizeNamePrefix = (name: string) => {
|
||||
const value = name
|
||||
.replace(/[^a-zA-Z0-9_-]+/g, "_")
|
||||
.replace(/_+/g, "_")
|
||||
.replace(/^_|_$/g, "")
|
||||
return value || "video"
|
||||
}
|
||||
|
||||
interface LabelProps {
|
||||
headerHeight: number
|
||||
leftWidth: number
|
||||
@@ -218,6 +235,91 @@ const LabelPage = ({
|
||||
setQaOperations,
|
||||
])
|
||||
|
||||
const normalizeVideoTaskData = useCallback(
|
||||
async (detail: Task.DataProps) => {
|
||||
try {
|
||||
const sourceNames = detail.data_name
|
||||
.map((item) => item?.name?.[0] || "")
|
||||
.filter((name) => !!name)
|
||||
if (!sourceNames.length || !projectId) return detail
|
||||
|
||||
const taskDataType = Number(
|
||||
(detail as { [key: string]: any })?.data_type ??
|
||||
(detail as { [key: string]: any })?.task_data_type ??
|
||||
-1
|
||||
)
|
||||
const isVideoTask =
|
||||
taskDataType === 2 ||
|
||||
sourceNames.some((name) => videoExtPattern.test(name))
|
||||
if (!isVideoTask) return detail
|
||||
|
||||
const imagesMap = new Map(useImagesStore.getState().images)
|
||||
let imageMapUpdated = false
|
||||
const normalizedDataName: Task.DataProps["data_name"] = []
|
||||
|
||||
for (const videoName of sourceNames) {
|
||||
const cacheKey = `${projectId}:${taskId}:${videoName}`
|
||||
let frameNames =
|
||||
useVideoFrameStore.getState().videoFrames.get(cacheKey) ?? []
|
||||
|
||||
const hasAllCachedFrames =
|
||||
frameNames.length > 0 &&
|
||||
frameNames.every((frameName) => {
|
||||
const cache = imagesMap.get(frameName)
|
||||
return typeof cache === "string" && cache.length > 0
|
||||
})
|
||||
|
||||
if (!hasAllCachedFrames) {
|
||||
const base64Video = await getServerImage({
|
||||
data_names: [videoName],
|
||||
data_type: 2,
|
||||
project_id: projectId,
|
||||
})
|
||||
const frameData = await processVideo({
|
||||
base64Data: String(base64Video || ""),
|
||||
fileName: videoName,
|
||||
namePrefix: normalizeNamePrefix(
|
||||
`${projectId}_${taskId}_${videoName.replace(/\.[^.]+$/, "")}`
|
||||
),
|
||||
})
|
||||
frameNames = frameData.map((frame) => frame.name)
|
||||
frameData.forEach((frame) => {
|
||||
imagesMap.set(frame.name, frame.dataUrl)
|
||||
})
|
||||
imageMapUpdated = true
|
||||
useVideoFrameStore.getState().setVideoFrames(cacheKey, frameNames)
|
||||
}
|
||||
|
||||
frameNames.forEach((frameName) => {
|
||||
normalizedDataName.push({
|
||||
name: [frameName] as [string],
|
||||
related_images: [],
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
if (imageMapUpdated) {
|
||||
useImagesStore.getState().setImages(imagesMap)
|
||||
}
|
||||
if (!normalizedDataName.length) return detail
|
||||
|
||||
return {
|
||||
...detail,
|
||||
data_name: normalizedDataName,
|
||||
}
|
||||
} catch (error) {
|
||||
console.log(error)
|
||||
showNotification({
|
||||
color: "red",
|
||||
title: "视频解码失败",
|
||||
message: "视频帧生成失败，已回退到原始数据",
|
||||
})
|
||||
return detail
|
||||
}
|
||||
},
|
||||
[projectId, taskId]
|
||||
)
|
||||
|
||||
const asyncGetTaskDetail = useCallback(async () => {
|
||||
if (taskId) {
|
||||
try {
|
||||
@@ -230,6 +332,7 @@ const LabelPage = ({
|
||||
page_size: 1,
|
||||
})
|
||||
let detail = res.task_list[0]
|
||||
detail = await normalizeVideoTaskData(detail)
|
||||
setTaskDetail(detail)
|
||||
// 帧列表
|
||||
useBottomToolsStore
|
||||
@@ -579,7 +682,14 @@ const LabelPage = ({
|
||||
usePaperStore.getState().setLoadingData(false)
|
||||
}
|
||||
}
|
||||
}, [setLabel, setLabelTime, setStateStack, taskId, user_id])
|
||||
}, [
|
||||
normalizeVideoTaskData,
|
||||
setLabel,
|
||||
setLabelTime,
|
||||
setStateStack,
|
||||
taskId,
|
||||
user_id,
|
||||
])
|
||||
|
||||
// 保存时更新当前workload
|
||||
const updateOldWorkLoad = (data: WorkLoad) => {
|
||||
|
||||
@@ -2,6 +2,8 @@ export namespace Task {
|
||||
export interface DataProps {
|
||||
id: number
|
||||
create_date: string
|
||||
data_type?: number
|
||||
task_data_type?: number
|
||||
data_name: {
|
||||
name: [string]
|
||||
related_images: []
|
||||
@@ -62,6 +64,7 @@ export namespace Task {
|
||||
}
|
||||
|
||||
export interface ListRequest {
|
||||
data_type?: number
|
||||
current_uid?: number[]
|
||||
data_name?: string
|
||||
first_commit_label_time_end?: string
|
||||
|
||||
@@ -498,6 +498,12 @@ interface ImageState {
|
||||
setImages: (value: any) => void
|
||||
}
|
||||
|
||||
interface VideoFrameState {
|
||||
videoFrames: Map<string, string[]>
|
||||
setVideoFrames: (key: string, value: string[]) => void
|
||||
removeVideoFrames: (key: string) => void
|
||||
}
|
||||
|
||||
export const useImagesStore = create(
|
||||
persist<ImageState>(
|
||||
(storeSet) => ({
|
||||
@@ -516,3 +522,33 @@ export const useImagesStore = create(
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
export const useVideoFrameStore = create(
|
||||
persist<VideoFrameState>(
|
||||
(storeSet) => ({
|
||||
videoFrames: new Map(),
|
||||
setVideoFrames: (key: string, value: string[]) =>
|
||||
storeSet((state) => {
|
||||
const nextMap = new Map(state.videoFrames)
|
||||
nextMap.set(key, value)
|
||||
return {
|
||||
...state,
|
||||
videoFrames: nextMap,
|
||||
}
|
||||
}),
|
||||
removeVideoFrames: (key: string) =>
|
||||
storeSet((state) => {
|
||||
const nextMap = new Map(state.videoFrames)
|
||||
nextMap.delete(key)
|
||||
return {
|
||||
...state,
|
||||
videoFrames: nextMap,
|
||||
}
|
||||
}),
|
||||
}),
|
||||
{
|
||||
name: "video-frame-storage",
|
||||
storage: storage,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user