feat(project): init
This commit is contained in:
406
app/component/label/video/libav.worker.ts
Normal file
406
app/component/label/video/libav.worker.ts
Normal file
@@ -0,0 +1,406 @@
|
||||
// src/workers/ffmpeg.worker.ts
|
||||
import type { LibAVWrapper, WorkerMessage, WorkerResponse } from "./types/libav"
|
||||
|
||||
const ctx: Worker = self as any
|
||||
let libav: any = null // Use any to access full API
|
||||
const PROXY_FILE = "proxy.avi"
|
||||
|
||||
ctx.onmessage = async (event: MessageEvent<WorkerMessage>) => {
|
||||
const { type } = event.data
|
||||
|
||||
try {
|
||||
switch (type) {
|
||||
case "LOAD":
|
||||
await loadLibAV(event.data.config.corePath)
|
||||
break
|
||||
case "TRANSCODE":
|
||||
if (!libav) throw new Error("LibAV not loaded")
|
||||
if ("file" in event.data) {
|
||||
await handleTranscode(event.data.file)
|
||||
}
|
||||
break
|
||||
case "EXTRACT_FRAME":
|
||||
if (!libav) throw new Error("LibAV not loaded")
|
||||
if ("time" in event.data) {
|
||||
await handleExtractFrame(event.data.time)
|
||||
}
|
||||
break
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.error("Worker Error:", err)
|
||||
ctx.postMessage({
|
||||
type: "ERROR",
|
||||
error: err.message || "Unknown error",
|
||||
} as WorkerResponse)
|
||||
}
|
||||
}
|
||||
|
||||
async function loadLibAV(corePath: string) {
|
||||
if (libav) {
|
||||
ctx.postMessage({ type: "READY" } as WorkerResponse)
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
importScripts(corePath)
|
||||
|
||||
const LibAVWrapper = (self as any).LibAV as LibAVWrapper
|
||||
if (!LibAVWrapper || typeof LibAVWrapper.LibAV !== "function") {
|
||||
throw new Error("LibAV factory not found.")
|
||||
}
|
||||
|
||||
const baseUrl = corePath.substring(0, corePath.lastIndexOf("/") + 1)
|
||||
// Dynamic WASM URL based on the JS file name (standard convention)
|
||||
const wasmUrl = corePath.replace(".js", ".wasm.wasm")
|
||||
|
||||
libav = await LibAVWrapper.LibAV({
|
||||
noworker: true,
|
||||
base: baseUrl,
|
||||
wasmurl: wasmUrl,
|
||||
})
|
||||
|
||||
ctx.postMessage({ type: "READY" } as WorkerResponse)
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to load libav: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
async function handleTranscode(file: File) {
|
||||
const ext = file.name.match(/\.([^.]+)$/)?.[0] || ".mp4"
|
||||
const inputFile = `input_video${ext}`
|
||||
|
||||
try {
|
||||
// 1. Write Input File
|
||||
const fileBuffer = await file.arrayBuffer()
|
||||
const u8Arr = new Uint8Array(fileBuffer)
|
||||
console.log(`[Worker] Received file: ${file.name}, size: ${u8Arr.length}`)
|
||||
|
||||
await libav.writeFile(inputFile, u8Arr)
|
||||
|
||||
// Note: libav.js (webcodecs-avf) does not support readdir/stat via API directly
|
||||
// but readFile works. We trust writeFile worked.
|
||||
|
||||
// 2. Init Demuxer (Input)
|
||||
// webcodecs-avf SHOULD have the demuxer for MP4.
|
||||
let fmt_ctx, streams
|
||||
try {
|
||||
;[fmt_ctx, streams] = await libav.ff_init_demuxer_file(inputFile)
|
||||
} catch (e) {
|
||||
console.error("Demuxer init failed:", e)
|
||||
throw new Error(
|
||||
`Demuxer failed. Libav variant might lack MP4 support. Error: ${e}`
|
||||
)
|
||||
}
|
||||
|
||||
// Find Video Stream
|
||||
let videoStreamIdx = -1
|
||||
let videoStream = null
|
||||
for (let i = 0; i < streams.length; i++) {
|
||||
if (streams[i].codec_type === 0) {
|
||||
// 0 is Video
|
||||
videoStreamIdx = i
|
||||
videoStream = streams[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if (videoStreamIdx === -1) throw new Error("No video stream found")
|
||||
|
||||
console.log(
|
||||
`[Worker] Found video stream #${videoStreamIdx}, codec_id=${videoStream.codec_id}`
|
||||
)
|
||||
|
||||
// 3. Init Decoder (Input)
|
||||
// For obsolete variant, we might have software H.264 decoders.
|
||||
// Try standard init first.
|
||||
|
||||
let c, pkt, frame
|
||||
try {
|
||||
// Try standard init first
|
||||
;[, c, pkt, frame] = await libav.ff_init_decoder(
|
||||
videoStream.codec_id,
|
||||
videoStream.codecpar
|
||||
)
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
"[Worker] Standard decoder init failed, trying explicit H264...",
|
||||
e
|
||||
)
|
||||
try {
|
||||
// Try explicit H264
|
||||
;[, c, pkt, frame] = await libav.ff_init_decoder(
|
||||
"h264",
|
||||
videoStream.codecpar
|
||||
)
|
||||
} catch (e2) {
|
||||
console.error("[Worker] All decoder init attempts failed", e2)
|
||||
throw new Error("Codec not found (decoder init failed).")
|
||||
}
|
||||
}
|
||||
|
||||
// Get input properties
|
||||
let width = await libav.AVCodecParameters_width(videoStream.codecpar)
|
||||
let height = await libav.AVCodecParameters_height(videoStream.codecpar)
|
||||
|
||||
console.log(`[Worker] Video dimensions: ${width}x${height}`)
|
||||
|
||||
// 4. Init Encoder (Output: MJPEG)
|
||||
const codec_name = "mjpeg"
|
||||
const [, enc_c, enc_frame, enc_pkt] = await libav.ff_init_encoder(
|
||||
codec_name,
|
||||
{
|
||||
ctx: {
|
||||
width: width,
|
||||
height: height,
|
||||
pix_fmt: 0, // AV_PIX_FMT_YUV420P
|
||||
time_base: [1, 25],
|
||||
framerate: [25, 1],
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
// 5. Init Muxer (Output: AVI container)
|
||||
const [oc, , pb, [stream_ctx]] = await libav.ff_init_muxer(
|
||||
{ format_name: "avi", filename: PROXY_FILE, open: true },
|
||||
[[enc_c, 1, 25]]
|
||||
)
|
||||
|
||||
console.log(stream_ctx)
|
||||
|
||||
// Write Header
|
||||
await libav.avformat_write_header(oc, 0)
|
||||
|
||||
ctx.postMessage({
|
||||
type: "LOG",
|
||||
message: "Transcoding started...",
|
||||
} as WorkerResponse)
|
||||
|
||||
// 6. Loop: Read -> Decode -> Encode -> Write
|
||||
let frameCount = 0
|
||||
|
||||
// We need to use "video" copyout mode for WebCodecs to work efficiently?
|
||||
// Actually, ff_decode_multi returns Frame objects.
|
||||
// If WebCodecs is used, the Frame might be backed by VideoFrame.
|
||||
|
||||
while (true) {
|
||||
const [res, packets] = await libav.ff_read_frame_multi(fmt_ctx, pkt)
|
||||
|
||||
if (res === libav.AVERROR_EOF) break
|
||||
if (res < 0 && res !== -11) break
|
||||
|
||||
const streamPackets = packets[videoStreamIdx]
|
||||
if (!streamPackets || streamPackets.length === 0) continue
|
||||
|
||||
// Decode
|
||||
const decodedFrames = await libav.ff_decode_multi(
|
||||
c,
|
||||
pkt,
|
||||
frame,
|
||||
streamPackets
|
||||
)
|
||||
|
||||
for (const f of decodedFrames) {
|
||||
console.log(f)
|
||||
|
||||
// Encode
|
||||
const encodedPackets = await libav.ff_encode_multi(
|
||||
enc_c,
|
||||
enc_frame,
|
||||
enc_pkt
|
||||
)
|
||||
|
||||
for (const p of encodedPackets) {
|
||||
p.stream_index = 0
|
||||
}
|
||||
|
||||
await libav.ff_write_multi(oc, enc_pkt, encodedPackets)
|
||||
frameCount++
|
||||
}
|
||||
|
||||
if (frameCount % 30 === 0) {
|
||||
ctx.postMessage({
|
||||
type: "LOG",
|
||||
message: `Transcoded ${frameCount} frames...`,
|
||||
} as WorkerResponse)
|
||||
}
|
||||
}
|
||||
|
||||
// Flush Encoder
|
||||
const flushedPackets = await libav.ff_encode_multi(
|
||||
enc_c,
|
||||
enc_frame,
|
||||
enc_pkt,
|
||||
[],
|
||||
true
|
||||
)
|
||||
if (flushedPackets.length > 0) {
|
||||
for (const p of flushedPackets) p.stream_index = 0
|
||||
await libav.ff_write_multi(oc, enc_pkt, flushedPackets)
|
||||
}
|
||||
|
||||
// Write Trailer
|
||||
await libav.av_write_trailer(oc)
|
||||
|
||||
// Cleanup
|
||||
await libav.ff_free_muxer(oc, pb)
|
||||
await libav.ff_free_encoder(enc_c, enc_frame, enc_pkt)
|
||||
await libav.ff_free_decoder(c, pkt, frame)
|
||||
await libav.avformat_close_input_js(fmt_ctx)
|
||||
await libav.unlink(inputFile)
|
||||
|
||||
ctx.postMessage({
|
||||
type: "DONE",
|
||||
data: `Transcoding complete. ${frameCount} frames ready.`,
|
||||
} as WorkerResponse)
|
||||
} catch (e: any) {
|
||||
console.error(e)
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
async function handleExtractFrame(time: number) {
|
||||
if (!libav) return
|
||||
|
||||
try {
|
||||
// 1. Open Proxy File
|
||||
// Note: For efficiency in a real app, keep this open.
|
||||
// But for safety here, we open/close per request or keep it open in global?
|
||||
// Let's open it fresh. AVI parsing is fast.
|
||||
const [fmt_ctx, streams] = await libav.ff_init_demuxer_file(PROXY_FILE)
|
||||
|
||||
const streamIdx = 0 // We know we only have 1 video stream
|
||||
const stream = streams[streamIdx]
|
||||
console.log(stream)
|
||||
|
||||
// 2. Init Decoder (MJPEG)
|
||||
const [, c, pkt, frame] = await libav.ff_init_decoder("mjpeg")
|
||||
|
||||
// 3. Seek
|
||||
// Calculate timestamp in AV_TIME_BASE or stream time base
|
||||
// av_seek_frame(ctx, stream_index, timestamp, flags)
|
||||
// If stream_index is -1, timestamp is in AV_TIME_BASE (microseconds)
|
||||
// We'll use stream index 0. We need to know its timebase.
|
||||
// But easier: Use -1 and AV_TIME_BASE
|
||||
const targetTS = Math.floor(time * 1000000) // seconds -> microseconds
|
||||
|
||||
// seek flags: 1 (BACKWARD) is usually good to find keyframe before time
|
||||
await libav.av_seek_frame(fmt_ctx, -1, targetTS, 1)
|
||||
|
||||
// 4. Read & Decode until we hit the frame
|
||||
// Since MJPEG is all-intra, the first frame we read after seek should be the one (or close)
|
||||
let foundFrame = null
|
||||
|
||||
// Limit attempts
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const [res, packets] = await libav.ff_read_frame_multi(fmt_ctx, pkt, {
|
||||
limit: 1,
|
||||
})
|
||||
if (res === libav.AVERROR_EOF) break
|
||||
|
||||
const streamPackets = packets[streamIdx]
|
||||
if (!streamPackets || streamPackets.length === 0) continue
|
||||
|
||||
// Decode
|
||||
const frames = await libav.ff_decode_multi(c, pkt, frame, streamPackets)
|
||||
if (frames && frames.length > 0) {
|
||||
foundFrame = frames[0]
|
||||
break // Just take the first one found after seek
|
||||
}
|
||||
}
|
||||
|
||||
if (foundFrame) {
|
||||
// 5. Convert to Blob (BMP is simplest uncompressed format libav supports encoding to?)
|
||||
// Or use libav to encode to JPEG?
|
||||
// Actually, since we are in MJPEG, the packet *is* a JPEG.
|
||||
// But we just decoded it.
|
||||
// Let's re-encode the single frame to JPEG for the UI.
|
||||
// Or just return the raw packet?
|
||||
// Let's re-encode to be safe (ensure headers etc).
|
||||
|
||||
// Init single-frame encoder
|
||||
// const [, jpeg_c, jpeg_frame, jpeg_pkt] = await libav.ff_init_encoder("mjpeg", {
|
||||
// ctx: { width: foundFrame.width, height: foundFrame.height, ... }
|
||||
// })
|
||||
// This is heavy.
|
||||
|
||||
// Alternative: "copyoutFrame" as "ImageData" (only in main thread).
|
||||
// In Worker, we can use "video_packed" to get RGB/RGBA buffer.
|
||||
// libav.js Frame object has data.
|
||||
// Let's stick to the prompt: "extract single images".
|
||||
// I'll return a Blob of a JPEG.
|
||||
// Since I have a valid frame, I can use a simple JS JPEG encoder or libav.
|
||||
// Using libav is robust.
|
||||
|
||||
// Let's try to grab the PACKET directly from the read step?
|
||||
// If the packet is a full JPEG, we can just dump it.
|
||||
// MJPEG stream packets usually miss Huffman tables (DHT). Browsers often fail to render them directly.
|
||||
// So safe bet: Decode -> Encode to JPEG (single frame).
|
||||
|
||||
// Reuse the decoder context? No, that's for decoding.
|
||||
// Create a throwaway encoder for the snapshot.
|
||||
const width = await libav.AVCodecContext_width(c)
|
||||
const height = await libav.AVCodecContext_height(c)
|
||||
|
||||
// We need to know the pixel format of the decoded frame.
|
||||
// usually YUVJ420P.
|
||||
|
||||
const [, enc_c, enc_frame, enc_pkt] = await libav.ff_init_encoder(
|
||||
"mjpeg",
|
||||
{
|
||||
ctx: {
|
||||
width,
|
||||
height,
|
||||
pix_fmt: 0, // YUV420P
|
||||
time_base: [1, 1],
|
||||
framerate: [1, 1],
|
||||
},
|
||||
options: {
|
||||
// Force standard JPEG tables
|
||||
qscale: "2",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
const encodedPackets = await libav.ff_encode_multi(
|
||||
enc_c,
|
||||
enc_frame,
|
||||
enc_pkt,
|
||||
[foundFrame]
|
||||
)
|
||||
|
||||
// Combine packets if multiple (usually 1 for JPEG)
|
||||
// But actually, ff_encode_multi returns packets.
|
||||
// We can concat their data.
|
||||
|
||||
const chunks = []
|
||||
for (const p of encodedPackets) {
|
||||
// p.data is the pointer. We need to copy it out.
|
||||
// Wait, ff_encode_multi returns objects with .data usually if copyoutPacket is default?
|
||||
// No, it returns Packet objects which refer to memory.
|
||||
// We need to copy data out.
|
||||
const data = await libav.ff_copyout_packet(p.ptr) // or just p.data if it's already copied?
|
||||
// libav.js docs: "Most functions will instead copy out their data into libav.js Frame or Packet objects."
|
||||
// So `encodedPackets` are JS objects with `data` (Uint8Array).
|
||||
console.log(data)
|
||||
chunks.push(p.data)
|
||||
}
|
||||
|
||||
const blob = new Blob(chunks, { type: "image/jpeg" })
|
||||
|
||||
ctx.postMessage({
|
||||
type: "FRAME",
|
||||
data: blob,
|
||||
time: time,
|
||||
} as WorkerResponse)
|
||||
|
||||
await libav.ff_free_encoder(enc_c, enc_frame, enc_pkt)
|
||||
} else {
|
||||
console.warn("Frame not found after seek")
|
||||
}
|
||||
|
||||
// Cleanup extraction
|
||||
await libav.ff_free_decoder(c, pkt, frame)
|
||||
await libav.avformat_close_input_js(fmt_ctx)
|
||||
} catch (e) {
|
||||
console.error("Extraction error", e)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user