// app/component/label/video/libav.worker.ts import type { LibAVWrapper, WorkerMessage, WorkerResponse, } from "@/app/component/label/video/types/libav" const ctx: Worker = self as any let libav: any = null // Use any to access full API const PROXY_FILE = "proxy.avi" ctx.onmessage = async (event: MessageEvent) => { const { type } = event.data try { switch (type) { case "LOAD": await loadLibAV(event.data.config.corePath) break case "TRANSCODE": if (!libav) throw new Error("LibAV not loaded") if ("file" in event.data) { await handleTranscode(event.data.file) } break case "EXTRACT_FRAME": if (!libav) throw new Error("LibAV not loaded") if ("time" in event.data) { await handleExtractFrame(event.data.time) } break } } catch (err: any) { console.error("Worker Error:", err) ctx.postMessage({ type: "ERROR", error: err.message || "Unknown error", } as WorkerResponse) } } async function loadLibAV(corePath: string) { if (libav) { ctx.postMessage({ type: "READY" } as WorkerResponse) return } try { importScripts(corePath) const LibAVWrapper = (self as any).LibAV as LibAVWrapper if (!LibAVWrapper || typeof LibAVWrapper.LibAV !== "function") { throw new Error("LibAV factory not found.") } const baseUrl = corePath.substring(0, corePath.lastIndexOf("/") + 1) // Dynamic WASM URL based on the JS file name (standard convention) const wasmUrl = corePath.replace(".js", ".wasm.wasm") libav = await LibAVWrapper.LibAV({ noworker: true, base: baseUrl, wasmurl: wasmUrl, }) ctx.postMessage({ type: "READY" } as WorkerResponse) } catch (error) { throw new Error(`Failed to load libav: ${error}`) } } async function handleTranscode(file: File) { const ext = file.name.match(/\.([^.]+)$/)?.[0] || ".mp4" const inputFile = `input_video${ext}` try { // 1. Write Input File const fileBuffer = await file.arrayBuffer() const u8Arr = new Uint8Array(fileBuffer) console.log(`[Worker] Received file: ${file.name}, size: ${u8Arr.length}`) await libav.writeFile(inputFile, u8Arr) // Note: libav.js (webcodecs-avf) does not support readdir/stat via API directly // but readFile works. We trust writeFile worked. // 2. Init Demuxer (Input) // webcodecs-avf SHOULD have the demuxer for MP4. let fmt_ctx, streams try { ;[fmt_ctx, streams] = await libav.ff_init_demuxer_file(inputFile) } catch (e) { console.error("Demuxer init failed:", e) throw new Error( `Demuxer failed. Libav variant might lack MP4 support. Error: ${e}` ) } // Find Video Stream let videoStreamIdx = -1 let videoStream = null for (let i = 0; i < streams.length; i++) { if (streams[i].codec_type === 0) { // 0 is Video videoStreamIdx = i videoStream = streams[i] break } } if (videoStreamIdx === -1) throw new Error("No video stream found") console.log( `[Worker] Found video stream #${videoStreamIdx}, codec_id=${videoStream.codec_id}` ) // 3. Init Decoder (Input) // For obsolete variant, we might have software H.264 decoders. // Try standard init first. let c, pkt, frame try { // Try standard init first ;[, c, pkt, frame] = await libav.ff_init_decoder( videoStream.codec_id, videoStream.codecpar ) } catch (e) { console.warn( "[Worker] Standard decoder init failed, trying explicit H264...", e ) try { // Try explicit H264 ;[, c, pkt, frame] = await libav.ff_init_decoder( "h264", videoStream.codecpar ) } catch (e2) { console.error("[Worker] All decoder init attempts failed", e2) throw new Error("Codec not found (decoder init failed).") } } // Get input properties let width = await libav.AVCodecParameters_width(videoStream.codecpar) let height = await libav.AVCodecParameters_height(videoStream.codecpar) console.log(`[Worker] Video dimensions: ${width}x${height}`) // 4. Init Encoder (Output: MJPEG) const codec_name = "mjpeg" const [, enc_c, enc_frame, enc_pkt] = await libav.ff_init_encoder( codec_name, { ctx: { width: width, height: height, pix_fmt: 0, // AV_PIX_FMT_YUV420P time_base: [1, 25], framerate: [25, 1], }, } ) // 5. Init Muxer (Output: AVI container) const [oc, , pb, [stream_ctx]] = await libav.ff_init_muxer( { format_name: "avi", filename: PROXY_FILE, open: true }, [[enc_c, 1, 25]] ) console.log(stream_ctx) // Write Header await libav.avformat_write_header(oc, 0) ctx.postMessage({ type: "LOG", message: "Transcoding started...", } as WorkerResponse) // 6. Loop: Read -> Decode -> Encode -> Write let frameCount = 0 // We need to use "video" copyout mode for WebCodecs to work efficiently? // Actually, ff_decode_multi returns Frame objects. // If WebCodecs is used, the Frame might be backed by VideoFrame. while (true) { const [res, packets] = await libav.ff_read_frame_multi(fmt_ctx, pkt) if (res === libav.AVERROR_EOF) break if (res < 0 && res !== -11) break const streamPackets = packets[videoStreamIdx] if (!streamPackets || streamPackets.length === 0) continue // Decode const decodedFrames = await libav.ff_decode_multi( c, pkt, frame, streamPackets ) for (const f of decodedFrames) { console.log(f) // Encode const encodedPackets = await libav.ff_encode_multi( enc_c, enc_frame, enc_pkt ) for (const p of encodedPackets) { p.stream_index = 0 } await libav.ff_write_multi(oc, enc_pkt, encodedPackets) frameCount++ } if (frameCount % 30 === 0) { ctx.postMessage({ type: "LOG", message: `Transcoded ${frameCount} frames...`, } as WorkerResponse) } } // Flush Encoder const flushedPackets = await libav.ff_encode_multi( enc_c, enc_frame, enc_pkt, [], true ) if (flushedPackets.length > 0) { for (const p of flushedPackets) p.stream_index = 0 await libav.ff_write_multi(oc, enc_pkt, flushedPackets) } // Write Trailer await libav.av_write_trailer(oc) // Cleanup await libav.ff_free_muxer(oc, pb) await libav.ff_free_encoder(enc_c, enc_frame, enc_pkt) await libav.ff_free_decoder(c, pkt, frame) await libav.avformat_close_input_js(fmt_ctx) await libav.unlink(inputFile) ctx.postMessage({ type: "DONE", data: `Transcoding complete. ${frameCount} frames ready.`, } as WorkerResponse) } catch (e: any) { console.error(e) throw e } } async function handleExtractFrame(time: number) { if (!libav) return try { // 1. Open Proxy File // Note: For efficiency in a real app, keep this open. // But for safety here, we open/close per request or keep it open in global? // Let's open it fresh. AVI parsing is fast. const [fmt_ctx, streams] = await libav.ff_init_demuxer_file(PROXY_FILE) const streamIdx = 0 // We know we only have 1 video stream const stream = streams[streamIdx] console.log(stream) // 2. Init Decoder (MJPEG) const [, c, pkt, frame] = await libav.ff_init_decoder("mjpeg") // 3. Seek // Calculate timestamp in AV_TIME_BASE or stream time base // av_seek_frame(ctx, stream_index, timestamp, flags) // If stream_index is -1, timestamp is in AV_TIME_BASE (microseconds) // We'll use stream index 0. We need to know its timebase. // But easier: Use -1 and AV_TIME_BASE const targetTS = Math.floor(time * 1000000) // seconds -> microseconds // seek flags: 1 (BACKWARD) is usually good to find keyframe before time await libav.av_seek_frame(fmt_ctx, -1, targetTS, 1) // 4. Read & Decode until we hit the frame // Since MJPEG is all-intra, the first frame we read after seek should be the one (or close) let foundFrame = null // Limit attempts for (let i = 0; i < 10; i++) { const [res, packets] = await libav.ff_read_frame_multi(fmt_ctx, pkt, { limit: 1, }) if (res === libav.AVERROR_EOF) break const streamPackets = packets[streamIdx] if (!streamPackets || streamPackets.length === 0) continue // Decode const frames = await libav.ff_decode_multi(c, pkt, frame, streamPackets) if (frames && frames.length > 0) { foundFrame = frames[0] break // Just take the first one found after seek } } if (foundFrame) { // 5. Convert to Blob (BMP is simplest uncompressed format libav supports encoding to?) // Or use libav to encode to JPEG? // Actually, since we are in MJPEG, the packet *is* a JPEG. // But we just decoded it. // Let's re-encode the single frame to JPEG for the UI. // Or just return the raw packet? // Let's re-encode to be safe (ensure headers etc). // Init single-frame encoder // const [, jpeg_c, jpeg_frame, jpeg_pkt] = await libav.ff_init_encoder("mjpeg", { // ctx: { width: foundFrame.width, height: foundFrame.height, ... } // }) // This is heavy. // Alternative: "copyoutFrame" as "ImageData" (only in main thread). // In Worker, we can use "video_packed" to get RGB/RGBA buffer. // libav.js Frame object has data. // Let's stick to the prompt: "extract single images". // I'll return a Blob of a JPEG. // Since I have a valid frame, I can use a simple JS JPEG encoder or libav. // Using libav is robust. // Let's try to grab the PACKET directly from the read step? // If the packet is a full JPEG, we can just dump it. // MJPEG stream packets usually miss Huffman tables (DHT). Browsers often fail to render them directly. // So safe bet: Decode -> Encode to JPEG (single frame). // Reuse the decoder context? No, that's for decoding. // Create a throwaway encoder for the snapshot. const width = await libav.AVCodecContext_width(c) const height = await libav.AVCodecContext_height(c) // We need to know the pixel format of the decoded frame. // usually YUVJ420P. const [, enc_c, enc_frame, enc_pkt] = await libav.ff_init_encoder( "mjpeg", { ctx: { width, height, pix_fmt: 0, // YUV420P time_base: [1, 1], framerate: [1, 1], }, options: { // Force standard JPEG tables qscale: "2", }, } ) const encodedPackets = await libav.ff_encode_multi( enc_c, enc_frame, enc_pkt, [foundFrame] ) // Combine packets if multiple (usually 1 for JPEG) // But actually, ff_encode_multi returns packets. // We can concat their data. const chunks = [] for (const p of encodedPackets) { // p.data is the pointer. We need to copy it out. // Wait, ff_encode_multi returns objects with .data usually if copyoutPacket is default? // No, it returns Packet objects which refer to memory. // We need to copy data out. const data = await libav.ff_copyout_packet(p.ptr) // or just p.data if it's already copied? // libav.js docs: "Most functions will instead copy out their data into libav.js Frame or Packet objects." // So `encodedPackets` are JS objects with `data` (Uint8Array). console.log(data) chunks.push(p.data) } const blob = new Blob(chunks, { type: "image/jpeg" }) ctx.postMessage({ type: "FRAME", data: blob, time: time, } as WorkerResponse) await libav.ff_free_encoder(enc_c, enc_frame, enc_pkt) } else { console.warn("Frame not found after seek") } // Cleanup extraction await libav.ff_free_decoder(c, pkt, frame) await libav.avformat_close_input_js(fmt_ctx) } catch (e) { console.error("Extraction error", e) } }