feat(project): init
This commit is contained in:
14
app/component/label/ffmpeg/README.md
Normal file
14
app/component/label/ffmpeg/README.md
Normal file
@@ -0,0 +1,14 @@
|
||||
# Video Frame Extractor (FFmpeg.wasm)
|
||||
|
||||
## 运行位置
|
||||
- 纯前端:所有转码/抽帧在浏览器 Web Worker + WASM 内完成,不依赖后端服务。
|
||||
|
||||
## 大文件最佳实践
|
||||
- 优先使用 WORKERFS 挂载输入文件,避免 `writeFile(arrayBuffer)` 将整段视频复制进 WASM 内存。
|
||||
- 抽帧建议走“单帧模式”:每次 `-ss <time> -frames:v 1` 只产出一张,读取后立即删除,避免在 WASM 内累积。
|
||||
- 输出建议使用 image2 序列 pattern(例如 `frame_%06d.jpg`)并配合 `-start_number`,避免 image2 的“必须是序列 pattern”报错路径。
|
||||
- UI 端必须对日志与图片列表做上限与释放(`URL.revokeObjectURL`),否则长视频会导致 JS 堆持续增长。
|
||||
|
||||
## 故障恢复
|
||||
- 一旦出现 `Aborted()` / `memory access out of bounds`,当前 FFmpeg.wasm 实例可能进入不可恢复状态,需要 terminate 并重建实例/worker。
|
||||
|
||||
326
app/component/label/ffmpeg/VideoFrameExtractor.tsx
Normal file
326
app/component/label/ffmpeg/VideoFrameExtractor.tsx
Normal file
@@ -0,0 +1,326 @@
|
||||
import {
|
||||
ActionIcon,
|
||||
Alert,
|
||||
Box,
|
||||
Button,
|
||||
Card,
|
||||
Container,
|
||||
FileInput,
|
||||
Group,
|
||||
Image,
|
||||
Loader,
|
||||
Progress,
|
||||
ScrollArea,
|
||||
SimpleGrid,
|
||||
Stack,
|
||||
Text,
|
||||
Title,
|
||||
} from "@mantine/core"
|
||||
import {
|
||||
IconAlertCircle,
|
||||
IconDownload,
|
||||
IconMovie,
|
||||
IconPhoto,
|
||||
} from "@tabler/icons-react"
|
||||
import { useEffect, useRef, useState } from "react"
|
||||
import { appendCapped } from "./cap"
|
||||
import type { WorkerResponse } from "./ffmpeg.worker"
|
||||
|
||||
export default function VideoFrameExtractor() {
|
||||
const [loaded, setLoaded] = useState(false)
|
||||
const [isLoading, setIsLoading] = useState(false)
|
||||
const [file, setFile] = useState<File | null>(null)
|
||||
const [processing, setProcessing] = useState(false)
|
||||
const [progress, setProgress] = useState(0)
|
||||
const [message, setMessage] = useState("Wait for loading...")
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [images, setImages] = useState<{ url: string; name: string }[]>([])
|
||||
const [logs, setLogs] = useState<string[]>([])
|
||||
|
||||
const workerRef = useRef<Worker | null>(null)
|
||||
const messageRef = useRef<HTMLDivElement>(null)
|
||||
const extractedCountRef = useRef(0)
|
||||
|
||||
const MAX_LOG_LINES = 300
|
||||
const MAX_IMAGES = 1500
|
||||
|
||||
// Initialize Worker once
|
||||
useEffect(() => {
|
||||
queueMicrotask(() => {
|
||||
if (!workerRef.current) {
|
||||
setIsLoading(true)
|
||||
|
||||
const baseURL = window.location.origin + "/wasm"
|
||||
|
||||
const createWorker = () => {
|
||||
const worker = new Worker(
|
||||
new URL("./ffmpeg.worker.ts", import.meta.url)
|
||||
)
|
||||
workerRef.current = worker
|
||||
return worker
|
||||
}
|
||||
|
||||
const handleMessage = (event: MessageEvent<WorkerResponse>) => {
|
||||
const { type } = event.data
|
||||
|
||||
switch (type) {
|
||||
case "READY":
|
||||
setLoaded(true)
|
||||
setIsLoading(false)
|
||||
setMessage("FFmpeg loaded successfully")
|
||||
break
|
||||
case "LOG":
|
||||
if ("message" in event.data) {
|
||||
const msg = event.data.message
|
||||
setLogs((prev) => appendCapped(prev, [msg], MAX_LOG_LINES))
|
||||
if (messageRef.current) {
|
||||
messageRef.current.scrollTop = messageRef.current.scrollHeight
|
||||
}
|
||||
}
|
||||
break
|
||||
case "PROGRESS":
|
||||
if ("progress" in event.data) {
|
||||
const p = Math.round(event.data.progress * 100)
|
||||
setProgress(p)
|
||||
setMessage(`Processing... ${p}%`)
|
||||
}
|
||||
break
|
||||
case "SEGMENT_DATA":
|
||||
if ("files" in event.data) {
|
||||
const newImages = event.data.files.map((f: any) => {
|
||||
const blob = new Blob([f.data], { type: "image/jpeg" })
|
||||
return { url: URL.createObjectURL(blob), name: f.name }
|
||||
})
|
||||
extractedCountRef.current += newImages.length
|
||||
setImages((prev) =>
|
||||
appendCapped(prev, newImages, MAX_IMAGES, (img) =>
|
||||
URL.revokeObjectURL((img as any).url)
|
||||
)
|
||||
)
|
||||
setMessage(
|
||||
`Processing... Extracted ${extractedCountRef.current} frames.`
|
||||
)
|
||||
}
|
||||
break
|
||||
case "DONE":
|
||||
setMessage(
|
||||
`Completed! Extracted ${extractedCountRef.current} frames.`
|
||||
)
|
||||
setProcessing(false)
|
||||
break
|
||||
case "FATAL":
|
||||
if ("error" in event.data) {
|
||||
setError(event.data.error)
|
||||
setProcessing(false)
|
||||
setLoaded(false)
|
||||
setIsLoading(true)
|
||||
}
|
||||
if (workerRef.current) {
|
||||
workerRef.current.terminate()
|
||||
workerRef.current = null
|
||||
}
|
||||
{
|
||||
const worker = createWorker()
|
||||
worker.onmessage = handleMessage
|
||||
worker.postMessage({ type: "LOAD", baseURL })
|
||||
}
|
||||
break
|
||||
case "ERROR":
|
||||
if ("error" in event.data) {
|
||||
setError(event.data.error)
|
||||
setProcessing(false)
|
||||
setIsLoading(false)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
const worker = createWorker()
|
||||
worker.onmessage = handleMessage
|
||||
|
||||
worker.postMessage({ type: "LOAD", baseURL })
|
||||
}
|
||||
})
|
||||
// Only create worker if not exists
|
||||
|
||||
// Cleanup worker on unmount?
|
||||
// Usually good practice, but if we want to keep it alive for navigation back/forth,
|
||||
// we might want to move it to Context.
|
||||
// For this task, local cleanup is fine.
|
||||
return () => {
|
||||
if (workerRef.current) {
|
||||
workerRef.current.terminate()
|
||||
workerRef.current = null
|
||||
}
|
||||
}
|
||||
}, [])
|
||||
|
||||
const processVideo = () => {
|
||||
if (!file || !loaded || !workerRef.current) return
|
||||
|
||||
setProcessing(true)
|
||||
setProgress(0)
|
||||
extractedCountRef.current = 0
|
||||
for (const img of images) URL.revokeObjectURL(img.url)
|
||||
setImages([])
|
||||
setLogs([])
|
||||
setError(null)
|
||||
setMessage("Starting processing...")
|
||||
|
||||
// Extract 1 frame per second: -vf fps=1
|
||||
// Optimize: Scale down to 720p max width to save memory
|
||||
workerRef.current.postMessage({
|
||||
type: "EXEC",
|
||||
file,
|
||||
args: [
|
||||
"-vf",
|
||||
"fps=1,scale='min(720,iw)':-1",
|
||||
"-q:v",
|
||||
"2",
|
||||
"frame_%03d.jpg",
|
||||
],
|
||||
outputPattern: "frame_",
|
||||
})
|
||||
}
|
||||
|
||||
const downloadAll = () => {
|
||||
images.forEach((img, index) => {
|
||||
setTimeout(() => {
|
||||
const a = document.createElement("a")
|
||||
a.href = img.url
|
||||
a.download = img.name
|
||||
a.click()
|
||||
}, index * 200)
|
||||
})
|
||||
}
|
||||
|
||||
return (
|
||||
<Container size="lg" py="xl">
|
||||
<Stack gap="lg">
|
||||
<Title order={2}>Video Frame Extractor (Worker + Local WASM)</Title>
|
||||
<Text c="dimmed">
|
||||
Extract frames from your video files purely in the browser using
|
||||
FFmpeg.wasm (Async Worker).
|
||||
</Text>
|
||||
|
||||
{error && (
|
||||
<Alert
|
||||
icon={<IconAlertCircle size={16} />}
|
||||
title="Error"
|
||||
color="red"
|
||||
withCloseButton
|
||||
onClose={() => setError(null)}>
|
||||
{error}
|
||||
</Alert>
|
||||
)}
|
||||
|
||||
<Card withBorder shadow="sm" p="lg" radius="md">
|
||||
<Stack gap="md">
|
||||
<FileInput
|
||||
label="Select Video File"
|
||||
placeholder="Click to select MP4, WebM..."
|
||||
accept="video/*"
|
||||
leftSection={<IconMovie size={16} />}
|
||||
value={file}
|
||||
onChange={setFile}
|
||||
disabled={!loaded || processing}
|
||||
/>
|
||||
|
||||
{!loaded && isLoading && (
|
||||
<Group>
|
||||
<Loader size="sm" />
|
||||
<Text size="sm">Loading FFmpeg core...</Text>
|
||||
</Group>
|
||||
)}
|
||||
|
||||
{loaded && (
|
||||
<Group justify="flex-end">
|
||||
<Button
|
||||
onClick={processVideo}
|
||||
disabled={!file || processing}
|
||||
loading={processing}
|
||||
leftSection={<IconPhoto size={16} />}>
|
||||
{processing ? "Processing..." : "Start Extraction"}
|
||||
</Button>
|
||||
</Group>
|
||||
)}
|
||||
|
||||
{processing && (
|
||||
<Stack gap="xs">
|
||||
<Text size="sm">{message}</Text>
|
||||
<Progress value={progress} animated />
|
||||
</Stack>
|
||||
)}
|
||||
</Stack>
|
||||
</Card>
|
||||
|
||||
{logs.length > 0 && (
|
||||
<Card withBorder shadow="sm" p="xs" radius="md" bg="gray.0">
|
||||
<Text size="xs" fw={500} mb="xs">
|
||||
Logs
|
||||
</Text>
|
||||
<ScrollArea h={100} type="always" viewportRef={messageRef}>
|
||||
<Box
|
||||
style={{
|
||||
whiteSpace: "pre-wrap",
|
||||
fontFamily: "monospace",
|
||||
fontSize: "11px",
|
||||
}}>
|
||||
{logs.map((log, i) => (
|
||||
<div key={i}>{log}</div>
|
||||
))}
|
||||
</Box>
|
||||
</ScrollArea>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
{images.length > 0 && (
|
||||
<Stack gap="md">
|
||||
<Group justify="space-between" align="center">
|
||||
<Title order={3}>Extracted Frames ({images.length})</Title>
|
||||
<Button
|
||||
variant="outline"
|
||||
leftSection={<IconDownload size={16} />}
|
||||
onClick={downloadAll}>
|
||||
Download All
|
||||
</Button>
|
||||
</Group>
|
||||
|
||||
<SimpleGrid cols={{ base: 2, sm: 3, md: 4 }} spacing="md">
|
||||
{images.map((img) => (
|
||||
<Card
|
||||
key={img.name}
|
||||
shadow="sm"
|
||||
padding="xs"
|
||||
radius="md"
|
||||
withBorder>
|
||||
<Card.Section>
|
||||
<Image
|
||||
src={img.url}
|
||||
height={160}
|
||||
alt={img.name}
|
||||
fit="cover"
|
||||
/>
|
||||
</Card.Section>
|
||||
<Group justify="space-between" mt="xs" mb="xs">
|
||||
<Text fw={500} size="xs" truncate>
|
||||
{img.name}
|
||||
</Text>
|
||||
<ActionIcon
|
||||
variant="light"
|
||||
color="blue"
|
||||
component="a"
|
||||
href={img.url}
|
||||
download={img.name}>
|
||||
<IconDownload size={14} />
|
||||
</ActionIcon>
|
||||
</Group>
|
||||
</Card>
|
||||
))}
|
||||
</SimpleGrid>
|
||||
</Stack>
|
||||
)}
|
||||
</Stack>
|
||||
</Container>
|
||||
)
|
||||
}
|
||||
18
app/component/label/ffmpeg/cap.test.ts
Normal file
18
app/component/label/ffmpeg/cap.test.ts
Normal file
@@ -0,0 +1,18 @@
|
||||
import { appendCapped } from "./cap"
|
||||
|
||||
describe("appendCapped", () => {
|
||||
test("keeps all items when under max", () => {
|
||||
expect(appendCapped([1, 2], [3], 10)).toEqual([1, 2, 3])
|
||||
})
|
||||
|
||||
test("drops from head when exceeding max", () => {
|
||||
expect(appendCapped([1, 2], [3, 4, 5], 3)).toEqual([3, 4, 5])
|
||||
})
|
||||
|
||||
test("calls onDrop for dropped items", () => {
|
||||
const dropped: number[] = []
|
||||
const result = appendCapped([1, 2], [3, 4, 5], 3, (x) => dropped.push(x))
|
||||
expect(result).toEqual([3, 4, 5])
|
||||
expect(dropped).toEqual([1, 2])
|
||||
})
|
||||
})
|
||||
14
app/component/label/ffmpeg/cap.ts
Normal file
14
app/component/label/ffmpeg/cap.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
export const appendCapped = <T>(
|
||||
prev: T[],
|
||||
next: T[],
|
||||
max: number,
|
||||
onDrop?: (item: T) => void
|
||||
) => {
|
||||
const merged = [...prev, ...next]
|
||||
if (merged.length <= max) return merged
|
||||
const dropCount = merged.length - max
|
||||
if (onDrop) {
|
||||
for (let i = 0; i < dropCount; i += 1) onDrop(merged[i])
|
||||
}
|
||||
return merged.slice(dropCount)
|
||||
}
|
||||
520
app/component/label/ffmpeg/ffmpeg.worker.ts
Normal file
520
app/component/label/ffmpeg/ffmpeg.worker.ts
Normal file
@@ -0,0 +1,520 @@
|
||||
// Define types for worker messages
|
||||
export type WorkerMessage =
|
||||
| { type: "LOAD"; baseURL: string }
|
||||
| { type: "EXEC"; file: File; args: string[]; outputPattern: string }
|
||||
|
||||
export type WorkerResponse =
|
||||
| { type: "READY" }
|
||||
| { type: "LOG"; message: string }
|
||||
| { type: "PROGRESS"; progress: number }
|
||||
| { type: "DONE" }
|
||||
| { type: "SEGMENT_DATA"; files: { name: string; data: Uint8Array }[] }
|
||||
| { type: "FATAL"; error: string }
|
||||
| { type: "ERROR"; error: string }
|
||||
|
||||
import { FFmpeg } from "@ffmpeg/ffmpeg"
|
||||
|
||||
const ctx: Worker = self as any
|
||||
const ffmpeg = new FFmpeg()
|
||||
|
||||
let loaded = false
|
||||
let lastBaseURL: string | null = null
|
||||
|
||||
// Custom Logger
|
||||
ffmpeg.on("log", ({ message }) => {
|
||||
ctx.postMessage({ type: "LOG", message })
|
||||
})
|
||||
|
||||
ffmpeg.on("progress", ({ progress }) => {
|
||||
ctx.postMessage({ type: "PROGRESS", progress })
|
||||
})
|
||||
|
||||
ctx.onmessage = async (event: MessageEvent<WorkerMessage>) => {
|
||||
const { type } = event.data
|
||||
|
||||
try {
|
||||
switch (type) {
|
||||
case "LOAD":
|
||||
if (loaded) {
|
||||
ctx.postMessage({ type: "READY" })
|
||||
return
|
||||
}
|
||||
const { baseURL } = event.data
|
||||
lastBaseURL = baseURL
|
||||
|
||||
await ffmpeg.load({
|
||||
coreURL: `${baseURL}/ffmpeg-core.js`,
|
||||
wasmURL: `${baseURL}/ffmpeg-core.wasm`,
|
||||
})
|
||||
|
||||
loaded = true
|
||||
ctx.postMessage({ type: "READY" })
|
||||
break
|
||||
|
||||
case "EXEC":
|
||||
if (!loaded) throw new Error("FFmpeg not loaded")
|
||||
const { file, args, outputPattern } = event.data
|
||||
|
||||
ctx.postMessage({
|
||||
type: "LOG",
|
||||
message: `[start] outputPattern=${outputPattern}`,
|
||||
})
|
||||
|
||||
const mountDir = "/input_mount"
|
||||
const inputPath = `${mountDir}/${file.name}`
|
||||
|
||||
const mountInput = async () => {
|
||||
await ffmpeg.createDir(mountDir)
|
||||
await ffmpeg.mount("WORKERFS" as any, { files: [file] }, mountDir)
|
||||
}
|
||||
|
||||
const unmountInput = async () => {
|
||||
await ffmpeg.unmount(mountDir).catch(() => {})
|
||||
await ffmpeg.deleteDir(mountDir).catch(() => {})
|
||||
}
|
||||
|
||||
await mountInput()
|
||||
|
||||
try {
|
||||
const normalizeError = (e: unknown) => {
|
||||
if (typeof e === "string") return e
|
||||
if (e && typeof e === "object" && "message" in e)
|
||||
return String((e as any).message)
|
||||
return String(e)
|
||||
}
|
||||
|
||||
const isOom = (msg: string) =>
|
||||
msg.includes("memory access out of bounds") ||
|
||||
msg.includes("Cannot enlarge memory") ||
|
||||
msg.includes("Aborted")
|
||||
|
||||
const nowMs = () =>
|
||||
typeof performance !== "undefined" ? performance.now() : Date.now()
|
||||
|
||||
const makeThrottledLogger = (intervalMs: number) => {
|
||||
let last = 0
|
||||
return (message: string) => {
|
||||
const t = nowMs()
|
||||
if (t - last < intervalMs) return
|
||||
last = t
|
||||
ctx.postMessage({ type: "LOG", message })
|
||||
}
|
||||
}
|
||||
|
||||
const logThrottled = makeThrottledLogger(1000)
|
||||
|
||||
const findArgValue = (flag: string) => {
|
||||
const idx = args.indexOf(flag)
|
||||
if (idx < 0) return null
|
||||
const value = args[idx + 1]
|
||||
return typeof value === "string" ? value : null
|
||||
}
|
||||
|
||||
const parseFps = () => {
|
||||
const vf = findArgValue("-vf")
|
||||
if (!vf) return null
|
||||
const match = vf.match(/(?:^|,)fps=(\d+(?:\.\d+)?)(?:,|$)/)
|
||||
if (!match) return null
|
||||
const fps = Number(match[1])
|
||||
return Number.isFinite(fps) && fps > 0 ? fps : null
|
||||
}
|
||||
|
||||
let durationSec = 0
|
||||
const durationProbePath = "__duration.txt"
|
||||
try {
|
||||
await ffmpeg.ffprobe([
|
||||
"-v",
|
||||
"error",
|
||||
"-show_entries",
|
||||
"format=duration",
|
||||
"-of",
|
||||
"default=noprint_wrappers=1:nokey=1",
|
||||
inputPath,
|
||||
"-o",
|
||||
durationProbePath,
|
||||
])
|
||||
const txt = (await ffmpeg.readFile(
|
||||
durationProbePath,
|
||||
"utf8"
|
||||
)) as string
|
||||
const parsed = Number.parseFloat(String(txt).trim())
|
||||
if (Number.isFinite(parsed) && parsed > 0) durationSec = parsed
|
||||
} catch (e) {
|
||||
ctx.postMessage({
|
||||
type: "LOG",
|
||||
message: `[probe] ffprobe failed: ${normalizeError(e)}`,
|
||||
})
|
||||
} finally {
|
||||
await ffmpeg.deleteFile(durationProbePath).catch(() => {})
|
||||
}
|
||||
|
||||
if (!durationSec) {
|
||||
const logHandler = ({ message }: { message: string }) => {
|
||||
const match = message.match(
|
||||
/Duration: (\d+):(\d+):(\d+(?:\.\d+)?)/
|
||||
)
|
||||
if (match) {
|
||||
const [, h, m, s] = match
|
||||
durationSec =
|
||||
parseFloat(h) * 3600 + parseFloat(m) * 60 + parseFloat(s)
|
||||
}
|
||||
}
|
||||
|
||||
ffmpeg.on("log", logHandler)
|
||||
try {
|
||||
await ffmpeg.exec(["-i", inputPath])
|
||||
} catch (e) {
|
||||
void e
|
||||
} finally {
|
||||
ffmpeg.off("log", logHandler)
|
||||
}
|
||||
}
|
||||
|
||||
ctx.postMessage({
|
||||
type: "LOG",
|
||||
message: `[probe] durationSec=${durationSec || "unknown"}`,
|
||||
})
|
||||
if (!durationSec) durationSec = 600
|
||||
|
||||
const fps = parseFps() ?? 1
|
||||
const stepSec = 1 / fps
|
||||
const preferSingleFrame = file.size >= 20 * 1024 * 1024
|
||||
const maxFrames = preferSingleFrame ? 30 : 0
|
||||
const reloadEvery = preferSingleFrame ? 10 : 0
|
||||
let sentBytes = 0
|
||||
|
||||
const maxWidth =
|
||||
file.size >= 40 * 1024 * 1024 ? 360 : preferSingleFrame ? 480 : 720
|
||||
|
||||
ctx.postMessage({
|
||||
type: "LOG",
|
||||
message: `[config] fileMB=${Math.round(file.size / 1024 / 1024)} fps=${fps} maxWidth=${maxWidth} preferSingleFrame=${preferSingleFrame}`,
|
||||
})
|
||||
|
||||
const outputExt = (() => {
|
||||
for (let i = args.length - 1; i >= 0; i -= 1) {
|
||||
const v = args[i]
|
||||
if (typeof v !== "string") continue
|
||||
const m = v.match(/\.([a-zA-Z0-9]+)$/)
|
||||
if (m) return m[1].toLowerCase()
|
||||
}
|
||||
return "jpg"
|
||||
})()
|
||||
|
||||
const buildSingleFrameArgs = (
|
||||
outputPattern: string,
|
||||
startNumber: number
|
||||
) => {
|
||||
const vf = findArgValue("-vf")
|
||||
const qv = findArgValue("-q:v")
|
||||
const baseFilter =
|
||||
vf
|
||||
?.replace(/(?:^|,)fps=\d+(?:\.\d+)?(?=,|$)/g, "")
|
||||
.replace(/^,|,$/g, "")
|
||||
.replace(/,,+/g, ",") ?? null
|
||||
|
||||
const patchedFilter = (() => {
|
||||
if (!baseFilter || !baseFilter.trim()) return null
|
||||
let f = baseFilter
|
||||
f = f.replace(/min\(\s*720\s*,\s*iw\s*\)/g, `min(${maxWidth},iw)`)
|
||||
f = f.replace(/min\(\s*480\s*,\s*iw\s*\)/g, `min(${maxWidth},iw)`)
|
||||
f = f.replace(/min\(\s*360\s*,\s*iw\s*\)/g, `min(${maxWidth},iw)`)
|
||||
if (!/(?:^|,)format=/.test(f)) f = `${f},format=yuv420p`
|
||||
return f
|
||||
})()
|
||||
|
||||
const out: string[] = []
|
||||
if (patchedFilter && patchedFilter.trim())
|
||||
out.push("-vf", patchedFilter)
|
||||
if (qv) {
|
||||
const qNum = Number(qv)
|
||||
if (Number.isFinite(qNum) && preferSingleFrame) {
|
||||
out.push("-q:v", String(Math.max(qNum, 6)))
|
||||
} else {
|
||||
out.push("-q:v", qv)
|
||||
}
|
||||
}
|
||||
out.push("-pix_fmt", "yuv420p")
|
||||
out.push("-start_number", String(startNumber))
|
||||
out.push("-frames:v", "1", outputPattern)
|
||||
return out
|
||||
}
|
||||
|
||||
const segmentCandidates = preferSingleFrame
|
||||
? []
|
||||
: [2, 1.2, 1].filter((v) => v >= stepSec && v > 0)
|
||||
|
||||
const trySegmentDuration = async (segmentDurationSec: number) => {
|
||||
let t = 0
|
||||
let emptyStreak = 0
|
||||
let producedAny = false
|
||||
|
||||
while (t < durationSec) {
|
||||
logThrottled(
|
||||
`[exec] mode=segment t=${t.toFixed(3)} dur=${segmentDurationSec}`
|
||||
)
|
||||
|
||||
const segmentArgs = [
|
||||
"-ss",
|
||||
t.toString(),
|
||||
"-t",
|
||||
segmentDurationSec.toString(),
|
||||
"-i",
|
||||
inputPath,
|
||||
"-threads",
|
||||
"1",
|
||||
"-an",
|
||||
"-sn",
|
||||
"-dn",
|
||||
...args,
|
||||
]
|
||||
|
||||
const ret = await ffmpeg.exec(segmentArgs)
|
||||
if (ret !== 0) {
|
||||
logThrottled(
|
||||
`[exec] mode=segment ret=${ret} t=${t.toFixed(3)} dur=${segmentDurationSec}`
|
||||
)
|
||||
}
|
||||
|
||||
const files = await ffmpeg.listDir(".")
|
||||
const imageFiles = files.filter(
|
||||
(f) =>
|
||||
!f.isDir &&
|
||||
f.name.startsWith("frame_") &&
|
||||
f.name.endsWith(`.${outputExt}`)
|
||||
)
|
||||
|
||||
if (imageFiles.length === 0) {
|
||||
emptyStreak += 1
|
||||
if (emptyStreak >= Math.ceil(5 / segmentDurationSec)) break
|
||||
} else {
|
||||
emptyStreak = 0
|
||||
producedAny = true
|
||||
}
|
||||
|
||||
const segmentFiles: { name: string; data: Uint8Array }[] = []
|
||||
const transfer: ArrayBuffer[] = []
|
||||
|
||||
for (const f of imageFiles) {
|
||||
const raw = (await ffmpeg.readFile(f.name)) as Uint8Array
|
||||
const copy = raw.slice()
|
||||
segmentFiles.push({
|
||||
name: `t_${t.toFixed(3)}_${f.name}`,
|
||||
data: copy,
|
||||
})
|
||||
transfer.push(copy.buffer)
|
||||
await ffmpeg.deleteFile(f.name)
|
||||
}
|
||||
|
||||
if (segmentFiles.length > 0) {
|
||||
ctx.postMessage(
|
||||
{ type: "SEGMENT_DATA", files: segmentFiles },
|
||||
transfer
|
||||
)
|
||||
}
|
||||
|
||||
ctx.postMessage({
|
||||
type: "PROGRESS",
|
||||
progress: Math.min((t + segmentDurationSec) / durationSec, 1),
|
||||
})
|
||||
|
||||
t += segmentDurationSec
|
||||
}
|
||||
|
||||
if (!producedAny) {
|
||||
throw new Error("NO_FRAMES")
|
||||
}
|
||||
}
|
||||
|
||||
const runSingleFrameMode = async () => {
|
||||
let index = 0
|
||||
let emptyStreak = 0
|
||||
const pattern = `frame_%06d.${outputExt}`
|
||||
|
||||
const reloadCore = async (reason: string) => {
|
||||
if (!lastBaseURL) return
|
||||
ctx.postMessage({ type: "LOG", message: `[reload] ${reason}` })
|
||||
await unmountInput()
|
||||
ffmpeg.terminate()
|
||||
loaded = false
|
||||
await ffmpeg.load({
|
||||
coreURL: `${lastBaseURL}/ffmpeg-core.js`,
|
||||
wasmURL: `${lastBaseURL}/ffmpeg-core.wasm`,
|
||||
})
|
||||
loaded = true
|
||||
await mountInput()
|
||||
}
|
||||
|
||||
for (let t = 0; t < durationSec; t += stepSec) {
|
||||
if (maxFrames > 0 && index >= maxFrames) break
|
||||
if (reloadEvery > 0 && index > 0 && index % reloadEvery === 0) {
|
||||
await reloadCore(`periodic frame=${index}`)
|
||||
}
|
||||
|
||||
const outputName = `frame_${String(index).padStart(6, "0")}.${outputExt}`
|
||||
const singleFrameArgs = [
|
||||
"-ss",
|
||||
t.toString(),
|
||||
"-i",
|
||||
inputPath,
|
||||
"-threads",
|
||||
"1",
|
||||
"-an",
|
||||
"-sn",
|
||||
"-dn",
|
||||
...(buildSingleFrameArgs(pattern, index) || []),
|
||||
]
|
||||
|
||||
try {
|
||||
const ret = await ffmpeg.exec(singleFrameArgs)
|
||||
ctx.postMessage({
|
||||
type: "LOG",
|
||||
message: `[exec] mode=single ret=${ret} t=${t.toFixed(3)} out=${outputName}`,
|
||||
})
|
||||
} catch (e) {
|
||||
const msg = normalizeError(e)
|
||||
if (isOom(msg)) throw e
|
||||
ctx.postMessage({
|
||||
type: "LOG",
|
||||
message: `[exec] mode=single error t=${t.toFixed(3)}: ${msg}`,
|
||||
})
|
||||
if (
|
||||
msg.includes("At least one output file must be specified")
|
||||
) {
|
||||
ctx.postMessage({
|
||||
type: "LOG",
|
||||
message: `[exec] mode=single argsTail=${JSON.stringify(
|
||||
singleFrameArgs.slice(-12)
|
||||
)}`,
|
||||
})
|
||||
}
|
||||
await ffmpeg.deleteFile(outputName).catch(() => {})
|
||||
index += 1
|
||||
continue
|
||||
}
|
||||
|
||||
try {
|
||||
const raw = (await ffmpeg.readFile(outputName)) as Uint8Array
|
||||
const copy = raw.slice()
|
||||
sentBytes += copy.byteLength
|
||||
await ffmpeg.deleteFile(outputName).catch(() => {})
|
||||
|
||||
ctx.postMessage(
|
||||
{
|
||||
type: "SEGMENT_DATA",
|
||||
files: [{ name: outputName, data: copy }],
|
||||
},
|
||||
[copy.buffer]
|
||||
)
|
||||
|
||||
logThrottled(
|
||||
`[frame] idx=${index} t=${t.toFixed(3)} size=${copy.byteLength} sentMB=${Math.round((sentBytes / 1024 / 1024) * 10) / 10}`
|
||||
)
|
||||
} catch (e) {
|
||||
void e
|
||||
emptyStreak += 1
|
||||
if (emptyStreak >= 3) break
|
||||
index += 1
|
||||
continue
|
||||
}
|
||||
emptyStreak = 0
|
||||
|
||||
ctx.postMessage({
|
||||
type: "PROGRESS",
|
||||
progress: Math.min((t + stepSec) / durationSec, 1),
|
||||
})
|
||||
|
||||
index += 1
|
||||
}
|
||||
}
|
||||
|
||||
let segmentOk = false
|
||||
for (const seg of segmentCandidates) {
|
||||
try {
|
||||
ctx.postMessage({
|
||||
type: "LOG",
|
||||
message: `[mode] trying segmentDuration=${seg}s stepSec=${stepSec}s`,
|
||||
})
|
||||
await trySegmentDuration(seg)
|
||||
segmentOk = true
|
||||
break
|
||||
} catch (e) {
|
||||
const msg = normalizeError(e)
|
||||
if (msg.includes("NO_FRAMES")) break
|
||||
if (
|
||||
msg.includes("Output file is empty") ||
|
||||
msg.includes("nothing was encoded")
|
||||
) {
|
||||
break
|
||||
}
|
||||
if (!isOom(msg)) throw e
|
||||
}
|
||||
}
|
||||
if (!segmentOk) {
|
||||
ctx.postMessage({
|
||||
type: "LOG",
|
||||
message: `[mode] segment mode unavailable; fallback to single-frame sampling`,
|
||||
})
|
||||
await runSingleFrameMode()
|
||||
}
|
||||
|
||||
// Send DONE without data
|
||||
ctx.postMessage({ type: "DONE" })
|
||||
} catch (e: any) {
|
||||
const errorMessage =
|
||||
typeof e === "string" ? e : e?.message || String(e)
|
||||
if (errorMessage.includes("memory access out of bounds")) {
|
||||
try {
|
||||
ffmpeg.terminate()
|
||||
loaded = false
|
||||
ctx.postMessage({
|
||||
type: "LOG",
|
||||
message: `[fatal] ffmpeg terminated due to: ${errorMessage}`,
|
||||
})
|
||||
|
||||
if (lastBaseURL) {
|
||||
await ffmpeg.load({
|
||||
coreURL: `${lastBaseURL}/ffmpeg-core.js`,
|
||||
wasmURL: `${lastBaseURL}/ffmpeg-core.wasm`,
|
||||
})
|
||||
loaded = true
|
||||
ctx.postMessage({
|
||||
type: "LOG",
|
||||
message: `[fatal] ffmpeg reloaded`,
|
||||
})
|
||||
}
|
||||
} catch (e2) {
|
||||
ctx.postMessage({
|
||||
type: "LOG",
|
||||
message: `[fatal] reload failed: ${
|
||||
typeof e2 === "string"
|
||||
? e2
|
||||
: (e2 as any)?.message || String(e2)
|
||||
}`,
|
||||
})
|
||||
}
|
||||
|
||||
ctx.postMessage({ type: "FATAL", error: errorMessage })
|
||||
return
|
||||
}
|
||||
|
||||
throw e
|
||||
} finally {
|
||||
// Cleanup input
|
||||
try {
|
||||
await unmountInput()
|
||||
} catch (e) {
|
||||
ctx.postMessage({
|
||||
type: "LOG",
|
||||
message: `[cleanup] ${typeof e === "string" ? e : (e as any)?.message || String(e)}`,
|
||||
})
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
} catch (error: any) {
|
||||
const msg =
|
||||
typeof error === "string" ? error : error?.message || String(error)
|
||||
ctx.postMessage({ type: "ERROR", error: msg })
|
||||
}
|
||||
}
|
||||
7
app/component/label/ffmpeg/page.tsx
Normal file
7
app/component/label/ffmpeg/page.tsx
Normal file
@@ -0,0 +1,7 @@
|
||||
"use client"
|
||||
|
||||
import VideoFrameExtractor from "./VideoFrameExtractor"
|
||||
|
||||
export default function ComponentLabelFfmpegPage() {
|
||||
return <VideoFrameExtractor />
|
||||
}
|
||||
39
app/component/label/login/page.tsx
Normal file
39
app/component/label/login/page.tsx
Normal file
@@ -0,0 +1,39 @@
|
||||
"use client"
|
||||
|
||||
import { userLogin } from "@/components/label/api/label"
|
||||
import { usePermissionStore } from "@/components/label/store/auth"
|
||||
import { Button } from "@mantine/core"
|
||||
|
||||
export default function ComponentLabelLoginPage() {
|
||||
const { setUserInfo, setUserPassword } = usePermissionStore()
|
||||
const user_id = usePermissionStore.getState().user_id
|
||||
const token = usePermissionStore.getState().token
|
||||
|
||||
// 用户登录
|
||||
const handleUserLoginBtnClick = async () => {
|
||||
try {
|
||||
const res = await userLogin({
|
||||
name: "admin",
|
||||
password: "123456",
|
||||
})
|
||||
const { uid, name, token, refresh_token } = res
|
||||
setUserInfo({
|
||||
user_id: uid,
|
||||
user_name: name,
|
||||
token,
|
||||
refresh_token,
|
||||
detailInfo: res,
|
||||
})
|
||||
setUserPassword("123456")
|
||||
} catch (error) {
|
||||
console.log(error)
|
||||
}
|
||||
}
|
||||
return (
|
||||
<>
|
||||
<Button onClick={handleUserLoginBtnClick}>Login</Button>
|
||||
user_id:{user_id}
|
||||
token:{token}
|
||||
</>
|
||||
)
|
||||
}
|
||||
41
app/component/label/picture/page.tsx
Normal file
41
app/component/label/picture/page.tsx
Normal file
@@ -0,0 +1,41 @@
|
||||
"use client"
|
||||
|
||||
import LabelContent from "@/components/label"
|
||||
import { useAppLayoutStore } from "@/components/layout/store"
|
||||
import { Breadcrumbs, Flex, Paper, Stack, UnstyledButton } from "@mantine/core"
|
||||
import { useMemo } from "react"
|
||||
|
||||
export default function ComponentLabelPicturePage() {
|
||||
const items = [{ title: "地图展示" }].map((item) => (
|
||||
<UnstyledButton key={item.title}>{item.title}</UnstyledButton>
|
||||
))
|
||||
const { isOpen } = useAppLayoutStore()
|
||||
let leftWidth = useMemo(() => {
|
||||
let width = 69
|
||||
return !isOpen ? width : width + 160
|
||||
}, [isOpen])
|
||||
return (
|
||||
<>
|
||||
<Stack w="100%" h="calc(100vh - 56px)" p={"md"}>
|
||||
<Flex justify={"space-between"} align={"center"} h={"16px"}>
|
||||
<Breadcrumbs>{items}</Breadcrumbs>
|
||||
</Flex>
|
||||
<Paper
|
||||
shadow="xs"
|
||||
p="0"
|
||||
flex={1}
|
||||
display="flex"
|
||||
style={{
|
||||
overflow: "hidden",
|
||||
}}>
|
||||
<LabelContent
|
||||
headerHeight={128}
|
||||
leftWidth={leftWidth}
|
||||
project_id={1}
|
||||
task_id={3}
|
||||
/>
|
||||
</Paper>
|
||||
</Stack>
|
||||
</>
|
||||
)
|
||||
}
|
||||
11
app/component/label/picture/standalone/page.tsx
Normal file
11
app/component/label/picture/standalone/page.tsx
Normal file
@@ -0,0 +1,11 @@
|
||||
"use client"
|
||||
|
||||
import LabelContent from "@/components/label"
|
||||
|
||||
export default function ComponentLabelPictureStandalonePage() {
|
||||
return (
|
||||
<>
|
||||
<LabelContent headerHeight={0} leftWidth={0} project_id={1} task_id={3} />
|
||||
</>
|
||||
)
|
||||
}
|
||||
406
app/component/label/video/libav.worker.ts
Normal file
406
app/component/label/video/libav.worker.ts
Normal file
@@ -0,0 +1,406 @@
|
||||
// src/workers/ffmpeg.worker.ts
|
||||
import type { LibAVWrapper, WorkerMessage, WorkerResponse } from "./types/libav"
|
||||
|
||||
const ctx: Worker = self as any
|
||||
let libav: any = null // Use any to access full API
|
||||
const PROXY_FILE = "proxy.avi"
|
||||
|
||||
ctx.onmessage = async (event: MessageEvent<WorkerMessage>) => {
|
||||
const { type } = event.data
|
||||
|
||||
try {
|
||||
switch (type) {
|
||||
case "LOAD":
|
||||
await loadLibAV(event.data.config.corePath)
|
||||
break
|
||||
case "TRANSCODE":
|
||||
if (!libav) throw new Error("LibAV not loaded")
|
||||
if ("file" in event.data) {
|
||||
await handleTranscode(event.data.file)
|
||||
}
|
||||
break
|
||||
case "EXTRACT_FRAME":
|
||||
if (!libav) throw new Error("LibAV not loaded")
|
||||
if ("time" in event.data) {
|
||||
await handleExtractFrame(event.data.time)
|
||||
}
|
||||
break
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.error("Worker Error:", err)
|
||||
ctx.postMessage({
|
||||
type: "ERROR",
|
||||
error: err.message || "Unknown error",
|
||||
} as WorkerResponse)
|
||||
}
|
||||
}
|
||||
|
||||
async function loadLibAV(corePath: string) {
|
||||
if (libav) {
|
||||
ctx.postMessage({ type: "READY" } as WorkerResponse)
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
importScripts(corePath)
|
||||
|
||||
const LibAVWrapper = (self as any).LibAV as LibAVWrapper
|
||||
if (!LibAVWrapper || typeof LibAVWrapper.LibAV !== "function") {
|
||||
throw new Error("LibAV factory not found.")
|
||||
}
|
||||
|
||||
const baseUrl = corePath.substring(0, corePath.lastIndexOf("/") + 1)
|
||||
// Dynamic WASM URL based on the JS file name (standard convention)
|
||||
const wasmUrl = corePath.replace(".js", ".wasm.wasm")
|
||||
|
||||
libav = await LibAVWrapper.LibAV({
|
||||
noworker: true,
|
||||
base: baseUrl,
|
||||
wasmurl: wasmUrl,
|
||||
})
|
||||
|
||||
ctx.postMessage({ type: "READY" } as WorkerResponse)
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to load libav: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
async function handleTranscode(file: File) {
|
||||
const ext = file.name.match(/\.([^.]+)$/)?.[0] || ".mp4"
|
||||
const inputFile = `input_video${ext}`
|
||||
|
||||
try {
|
||||
// 1. Write Input File
|
||||
const fileBuffer = await file.arrayBuffer()
|
||||
const u8Arr = new Uint8Array(fileBuffer)
|
||||
console.log(`[Worker] Received file: ${file.name}, size: ${u8Arr.length}`)
|
||||
|
||||
await libav.writeFile(inputFile, u8Arr)
|
||||
|
||||
// Note: libav.js (webcodecs-avf) does not support readdir/stat via API directly
|
||||
// but readFile works. We trust writeFile worked.
|
||||
|
||||
// 2. Init Demuxer (Input)
|
||||
// webcodecs-avf SHOULD have the demuxer for MP4.
|
||||
let fmt_ctx, streams
|
||||
try {
|
||||
;[fmt_ctx, streams] = await libav.ff_init_demuxer_file(inputFile)
|
||||
} catch (e) {
|
||||
console.error("Demuxer init failed:", e)
|
||||
throw new Error(
|
||||
`Demuxer failed. Libav variant might lack MP4 support. Error: ${e}`
|
||||
)
|
||||
}
|
||||
|
||||
// Find Video Stream
|
||||
let videoStreamIdx = -1
|
||||
let videoStream = null
|
||||
for (let i = 0; i < streams.length; i++) {
|
||||
if (streams[i].codec_type === 0) {
|
||||
// 0 is Video
|
||||
videoStreamIdx = i
|
||||
videoStream = streams[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if (videoStreamIdx === -1) throw new Error("No video stream found")
|
||||
|
||||
console.log(
|
||||
`[Worker] Found video stream #${videoStreamIdx}, codec_id=${videoStream.codec_id}`
|
||||
)
|
||||
|
||||
// 3. Init Decoder (Input)
|
||||
// For obsolete variant, we might have software H.264 decoders.
|
||||
// Try standard init first.
|
||||
|
||||
let c, pkt, frame
|
||||
try {
|
||||
// Try standard init first
|
||||
;[, c, pkt, frame] = await libav.ff_init_decoder(
|
||||
videoStream.codec_id,
|
||||
videoStream.codecpar
|
||||
)
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
"[Worker] Standard decoder init failed, trying explicit H264...",
|
||||
e
|
||||
)
|
||||
try {
|
||||
// Try explicit H264
|
||||
;[, c, pkt, frame] = await libav.ff_init_decoder(
|
||||
"h264",
|
||||
videoStream.codecpar
|
||||
)
|
||||
} catch (e2) {
|
||||
console.error("[Worker] All decoder init attempts failed", e2)
|
||||
throw new Error("Codec not found (decoder init failed).")
|
||||
}
|
||||
}
|
||||
|
||||
// Get input properties
|
||||
let width = await libav.AVCodecParameters_width(videoStream.codecpar)
|
||||
let height = await libav.AVCodecParameters_height(videoStream.codecpar)
|
||||
|
||||
console.log(`[Worker] Video dimensions: ${width}x${height}`)
|
||||
|
||||
// 4. Init Encoder (Output: MJPEG)
|
||||
const codec_name = "mjpeg"
|
||||
const [, enc_c, enc_frame, enc_pkt] = await libav.ff_init_encoder(
|
||||
codec_name,
|
||||
{
|
||||
ctx: {
|
||||
width: width,
|
||||
height: height,
|
||||
pix_fmt: 0, // AV_PIX_FMT_YUV420P
|
||||
time_base: [1, 25],
|
||||
framerate: [25, 1],
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
// 5. Init Muxer (Output: AVI container)
|
||||
const [oc, , pb, [stream_ctx]] = await libav.ff_init_muxer(
|
||||
{ format_name: "avi", filename: PROXY_FILE, open: true },
|
||||
[[enc_c, 1, 25]]
|
||||
)
|
||||
|
||||
console.log(stream_ctx)
|
||||
|
||||
// Write Header
|
||||
await libav.avformat_write_header(oc, 0)
|
||||
|
||||
ctx.postMessage({
|
||||
type: "LOG",
|
||||
message: "Transcoding started...",
|
||||
} as WorkerResponse)
|
||||
|
||||
// 6. Loop: Read -> Decode -> Encode -> Write
|
||||
let frameCount = 0
|
||||
|
||||
// We need to use "video" copyout mode for WebCodecs to work efficiently?
|
||||
// Actually, ff_decode_multi returns Frame objects.
|
||||
// If WebCodecs is used, the Frame might be backed by VideoFrame.
|
||||
|
||||
while (true) {
|
||||
const [res, packets] = await libav.ff_read_frame_multi(fmt_ctx, pkt)
|
||||
|
||||
if (res === libav.AVERROR_EOF) break
|
||||
if (res < 0 && res !== -11) break
|
||||
|
||||
const streamPackets = packets[videoStreamIdx]
|
||||
if (!streamPackets || streamPackets.length === 0) continue
|
||||
|
||||
// Decode
|
||||
const decodedFrames = await libav.ff_decode_multi(
|
||||
c,
|
||||
pkt,
|
||||
frame,
|
||||
streamPackets
|
||||
)
|
||||
|
||||
for (const f of decodedFrames) {
|
||||
console.log(f)
|
||||
|
||||
// Encode
|
||||
const encodedPackets = await libav.ff_encode_multi(
|
||||
enc_c,
|
||||
enc_frame,
|
||||
enc_pkt
|
||||
)
|
||||
|
||||
for (const p of encodedPackets) {
|
||||
p.stream_index = 0
|
||||
}
|
||||
|
||||
await libav.ff_write_multi(oc, enc_pkt, encodedPackets)
|
||||
frameCount++
|
||||
}
|
||||
|
||||
if (frameCount % 30 === 0) {
|
||||
ctx.postMessage({
|
||||
type: "LOG",
|
||||
message: `Transcoded ${frameCount} frames...`,
|
||||
} as WorkerResponse)
|
||||
}
|
||||
}
|
||||
|
||||
// Flush Encoder
|
||||
const flushedPackets = await libav.ff_encode_multi(
|
||||
enc_c,
|
||||
enc_frame,
|
||||
enc_pkt,
|
||||
[],
|
||||
true
|
||||
)
|
||||
if (flushedPackets.length > 0) {
|
||||
for (const p of flushedPackets) p.stream_index = 0
|
||||
await libav.ff_write_multi(oc, enc_pkt, flushedPackets)
|
||||
}
|
||||
|
||||
// Write Trailer
|
||||
await libav.av_write_trailer(oc)
|
||||
|
||||
// Cleanup
|
||||
await libav.ff_free_muxer(oc, pb)
|
||||
await libav.ff_free_encoder(enc_c, enc_frame, enc_pkt)
|
||||
await libav.ff_free_decoder(c, pkt, frame)
|
||||
await libav.avformat_close_input_js(fmt_ctx)
|
||||
await libav.unlink(inputFile)
|
||||
|
||||
ctx.postMessage({
|
||||
type: "DONE",
|
||||
data: `Transcoding complete. ${frameCount} frames ready.`,
|
||||
} as WorkerResponse)
|
||||
} catch (e: any) {
|
||||
console.error(e)
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
async function handleExtractFrame(time: number) {
|
||||
if (!libav) return
|
||||
|
||||
try {
|
||||
// 1. Open Proxy File
|
||||
// Note: For efficiency in a real app, keep this open.
|
||||
// But for safety here, we open/close per request or keep it open in global?
|
||||
// Let's open it fresh. AVI parsing is fast.
|
||||
const [fmt_ctx, streams] = await libav.ff_init_demuxer_file(PROXY_FILE)
|
||||
|
||||
const streamIdx = 0 // We know we only have 1 video stream
|
||||
const stream = streams[streamIdx]
|
||||
console.log(stream)
|
||||
|
||||
// 2. Init Decoder (MJPEG)
|
||||
const [, c, pkt, frame] = await libav.ff_init_decoder("mjpeg")
|
||||
|
||||
// 3. Seek
|
||||
// Calculate timestamp in AV_TIME_BASE or stream time base
|
||||
// av_seek_frame(ctx, stream_index, timestamp, flags)
|
||||
// If stream_index is -1, timestamp is in AV_TIME_BASE (microseconds)
|
||||
// We'll use stream index 0. We need to know its timebase.
|
||||
// But easier: Use -1 and AV_TIME_BASE
|
||||
const targetTS = Math.floor(time * 1000000) // seconds -> microseconds
|
||||
|
||||
// seek flags: 1 (BACKWARD) is usually good to find keyframe before time
|
||||
await libav.av_seek_frame(fmt_ctx, -1, targetTS, 1)
|
||||
|
||||
// 4. Read & Decode until we hit the frame
|
||||
// Since MJPEG is all-intra, the first frame we read after seek should be the one (or close)
|
||||
let foundFrame = null
|
||||
|
||||
// Limit attempts
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const [res, packets] = await libav.ff_read_frame_multi(fmt_ctx, pkt, {
|
||||
limit: 1,
|
||||
})
|
||||
if (res === libav.AVERROR_EOF) break
|
||||
|
||||
const streamPackets = packets[streamIdx]
|
||||
if (!streamPackets || streamPackets.length === 0) continue
|
||||
|
||||
// Decode
|
||||
const frames = await libav.ff_decode_multi(c, pkt, frame, streamPackets)
|
||||
if (frames && frames.length > 0) {
|
||||
foundFrame = frames[0]
|
||||
break // Just take the first one found after seek
|
||||
}
|
||||
}
|
||||
|
||||
if (foundFrame) {
|
||||
// 5. Convert to Blob (BMP is simplest uncompressed format libav supports encoding to?)
|
||||
// Or use libav to encode to JPEG?
|
||||
// Actually, since we are in MJPEG, the packet *is* a JPEG.
|
||||
// But we just decoded it.
|
||||
// Let's re-encode the single frame to JPEG for the UI.
|
||||
// Or just return the raw packet?
|
||||
// Let's re-encode to be safe (ensure headers etc).
|
||||
|
||||
// Init single-frame encoder
|
||||
// const [, jpeg_c, jpeg_frame, jpeg_pkt] = await libav.ff_init_encoder("mjpeg", {
|
||||
// ctx: { width: foundFrame.width, height: foundFrame.height, ... }
|
||||
// })
|
||||
// This is heavy.
|
||||
|
||||
// Alternative: "copyoutFrame" as "ImageData" (only in main thread).
|
||||
// In Worker, we can use "video_packed" to get RGB/RGBA buffer.
|
||||
// libav.js Frame object has data.
|
||||
// Let's stick to the prompt: "extract single images".
|
||||
// I'll return a Blob of a JPEG.
|
||||
// Since I have a valid frame, I can use a simple JS JPEG encoder or libav.
|
||||
// Using libav is robust.
|
||||
|
||||
// Let's try to grab the PACKET directly from the read step?
|
||||
// If the packet is a full JPEG, we can just dump it.
|
||||
// MJPEG stream packets usually miss Huffman tables (DHT). Browsers often fail to render them directly.
|
||||
// So safe bet: Decode -> Encode to JPEG (single frame).
|
||||
|
||||
// Reuse the decoder context? No, that's for decoding.
|
||||
// Create a throwaway encoder for the snapshot.
|
||||
const width = await libav.AVCodecContext_width(c)
|
||||
const height = await libav.AVCodecContext_height(c)
|
||||
|
||||
// We need to know the pixel format of the decoded frame.
|
||||
// usually YUVJ420P.
|
||||
|
||||
const [, enc_c, enc_frame, enc_pkt] = await libav.ff_init_encoder(
|
||||
"mjpeg",
|
||||
{
|
||||
ctx: {
|
||||
width,
|
||||
height,
|
||||
pix_fmt: 0, // YUV420P
|
||||
time_base: [1, 1],
|
||||
framerate: [1, 1],
|
||||
},
|
||||
options: {
|
||||
// Force standard JPEG tables
|
||||
qscale: "2",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
const encodedPackets = await libav.ff_encode_multi(
|
||||
enc_c,
|
||||
enc_frame,
|
||||
enc_pkt,
|
||||
[foundFrame]
|
||||
)
|
||||
|
||||
// Combine packets if multiple (usually 1 for JPEG)
|
||||
// But actually, ff_encode_multi returns packets.
|
||||
// We can concat their data.
|
||||
|
||||
const chunks = []
|
||||
for (const p of encodedPackets) {
|
||||
// p.data is the pointer. We need to copy it out.
|
||||
// Wait, ff_encode_multi returns objects with .data usually if copyoutPacket is default?
|
||||
// No, it returns Packet objects which refer to memory.
|
||||
// We need to copy data out.
|
||||
const data = await libav.ff_copyout_packet(p.ptr) // or just p.data if it's already copied?
|
||||
// libav.js docs: "Most functions will instead copy out their data into libav.js Frame or Packet objects."
|
||||
// So `encodedPackets` are JS objects with `data` (Uint8Array).
|
||||
console.log(data)
|
||||
chunks.push(p.data)
|
||||
}
|
||||
|
||||
const blob = new Blob(chunks, { type: "image/jpeg" })
|
||||
|
||||
ctx.postMessage({
|
||||
type: "FRAME",
|
||||
data: blob,
|
||||
time: time,
|
||||
} as WorkerResponse)
|
||||
|
||||
await libav.ff_free_encoder(enc_c, enc_frame, enc_pkt)
|
||||
} else {
|
||||
console.warn("Frame not found after seek")
|
||||
}
|
||||
|
||||
// Cleanup extraction
|
||||
await libav.ff_free_decoder(c, pkt, frame)
|
||||
await libav.avformat_close_input_js(fmt_ctx)
|
||||
} catch (e) {
|
||||
console.error("Extraction error", e)
|
||||
}
|
||||
}
|
||||
124
app/component/label/video/page.tsx
Normal file
124
app/component/label/video/page.tsx
Normal file
@@ -0,0 +1,124 @@
|
||||
"use client"
|
||||
|
||||
import { ChangeEvent, useEffect, useRef, useState } from "react"
|
||||
import type { WorkerMessage, WorkerResponse } from "./types/libav"
|
||||
|
||||
export default function Home() {
|
||||
const workerRef = useRef<Worker | null>(null)
|
||||
const [ready, setReady] = useState(false)
|
||||
const [logs, setLogs] = useState<string[]>([])
|
||||
const [videoUrl, setVideoUrl] = useState<string | null>(null)
|
||||
const [processing, setProcessing] = useState(false)
|
||||
const addLog = (msg: string) => {
|
||||
setLogs((prev) => [...prev.slice(-10), msg]) // 只保留最近10条
|
||||
}
|
||||
useEffect(() => {
|
||||
// 初始化 Worker
|
||||
// Next.js 会自动检测到这个语法并单独打包 Worker
|
||||
workerRef.current = new Worker(
|
||||
new URL("./libav.worker.ts", import.meta.url)
|
||||
)
|
||||
|
||||
// 设置消息监听
|
||||
workerRef.current.onmessage = (event: MessageEvent<WorkerResponse>) => {
|
||||
const { type } = event.data
|
||||
|
||||
switch (type) {
|
||||
case "READY":
|
||||
setReady(true)
|
||||
addLog("FFmpeg core loaded and ready.")
|
||||
break
|
||||
case "LOG":
|
||||
if ("message" in event.data) addLog(event.data.message)
|
||||
break
|
||||
case "DONE":
|
||||
if ("data" in event.data) {
|
||||
const url = URL.createObjectURL(event.data.data)
|
||||
setVideoUrl(url)
|
||||
setProcessing(false)
|
||||
addLog("Transcoding complete!")
|
||||
}
|
||||
break
|
||||
case "ERROR":
|
||||
if ("error" in event.data) {
|
||||
addLog(`Error: ${event.data.error}`)
|
||||
setProcessing(false)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// 告诉 Worker 加载 libav
|
||||
// 这里的路径必须指向 public 目录下的实际文件
|
||||
// 使用 default-cli 变体以确保包含 MP4/H.264 等常见格式的解封装器和解码器
|
||||
const LIBAV_PATH = "/libav/libav-6.8.8.0-default.js"
|
||||
workerRef.current.postMessage({
|
||||
type: "LOAD",
|
||||
config: { corePath: location.origin + LIBAV_PATH },
|
||||
} as WorkerMessage)
|
||||
|
||||
return () => {
|
||||
workerRef.current?.terminate()
|
||||
}
|
||||
}, [])
|
||||
|
||||
const handleFileChange = (e: ChangeEvent<HTMLInputElement>) => {
|
||||
const file = e.target.files?.[0]
|
||||
if (!file || !workerRef.current || !ready) return
|
||||
|
||||
setProcessing(true)
|
||||
setVideoUrl(null)
|
||||
setLogs([])
|
||||
addLog(`Starting upload: ${file.name}`)
|
||||
|
||||
workerRef.current.postMessage({
|
||||
type: "TRANSCODE",
|
||||
file: file,
|
||||
} as WorkerMessage)
|
||||
}
|
||||
|
||||
return (
|
||||
<main className="p-8 font-mono">
|
||||
<h1 className="text-2xl font-bold mb-4">Next.js + Libav.js + Worker</h1>
|
||||
|
||||
<div className="mb-4 p-4 border rounded bg-gray-50">
|
||||
<p>
|
||||
Status:{" "}
|
||||
{ready ? (
|
||||
<span className="text-green-600">Ready</span>
|
||||
) : (
|
||||
<span className="text-yellow-600">Loading Core...</span>
|
||||
)}
|
||||
</p>
|
||||
<p>Processing: {processing ? "Yes" : "No"}</p>
|
||||
</div>
|
||||
|
||||
<input
|
||||
type="file"
|
||||
onChange={handleFileChange}
|
||||
disabled={!ready || processing}
|
||||
className="mb-4 block w-full text-sm text-slate-500
|
||||
file:mr-4 file:py-2 file:px-4
|
||||
file:rounded-full file:border-0
|
||||
file:text-sm file:font-semibold
|
||||
file:bg-violet-50 file:text-violet-700
|
||||
hover:file:bg-violet-100"
|
||||
/>
|
||||
|
||||
{videoUrl && (
|
||||
<div className="mb-4">
|
||||
<h2 className="font-bold">Result:</h2>
|
||||
<video controls src={videoUrl} className="w-full max-w-md border" />
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="bg-black text-white p-4 rounded text-xs h-64 overflow-y-auto">
|
||||
{logs.map((log, i) => (
|
||||
<div key={i} className="border-b border-gray-800 py-1">
|
||||
{log}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</main>
|
||||
)
|
||||
}
|
||||
74
app/component/label/video/types/libav.d.ts
vendored
Normal file
74
app/component/label/video/types/libav.d.ts
vendored
Normal file
@@ -0,0 +1,74 @@
|
||||
// src/types/libav.d.ts
|
||||
|
||||
export type WorkerMessage =
|
||||
| { type: "LOAD"; config: { corePath: string } }
|
||||
| { type: "TRANSCODE"; file: File }
|
||||
| { type: "EXTRACT_FRAME"; time: number }
|
||||
|
||||
export type WorkerResponse =
|
||||
| { type: "READY" }
|
||||
| { type: "LOG"; message: string }
|
||||
| { type: "DONE"; data: any } // data 类型根据实际返回调整
|
||||
| { type: "FRAME"; data: Blob; time: number }
|
||||
| { type: "ERROR"; error: string }
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
LibAV: LibAVWrapper
|
||||
}
|
||||
}
|
||||
|
||||
export interface LibAVOpts {
|
||||
noworker?: boolean
|
||||
base?: string
|
||||
wasmurl?: string
|
||||
variant?: string
|
||||
}
|
||||
|
||||
export interface LibAVWrapper {
|
||||
LibAV(opts?: LibAVOpts): Promise<LibAVInstance>
|
||||
}
|
||||
|
||||
// 核心实例接口:添加官方例子中用到的底层方法
|
||||
export interface LibAVInstance {
|
||||
writeFile(name: string, content: Uint8Array): Promise<void>
|
||||
unlink(name: string): Promise<void>
|
||||
terminate(): void
|
||||
|
||||
// 1. 初始化解封装器 (Demuxer)
|
||||
// 返回 [Context, Streams[]]
|
||||
ff_init_demuxer_file(filename: string): Promise<[number, any[]]>
|
||||
|
||||
// 2. 初始化解码器 (Decoder)
|
||||
// 返回 [Codec, CodecContext, Packet, Frame]
|
||||
ff_init_decoder(
|
||||
codec_id: number,
|
||||
codecpar: number
|
||||
): Promise<[number, number, number, number]>
|
||||
|
||||
// 3. 读取多个包
|
||||
// 返回 [ResultCode, PacketsMap]
|
||||
ff_read_frame_multi(
|
||||
fmt_ctx: number,
|
||||
pkt: number,
|
||||
opts?: {
|
||||
limit?: number // OUTPUT limit, in bytes
|
||||
unify?: boolean // If true, unify the packets into a single stream (called 0), so that the output is in the same order as the input
|
||||
copyoutPacket?: "default" // Version of ff_copyout_packet to use
|
||||
}
|
||||
): Promise<[number, Record<number, Packet[]>]>
|
||||
|
||||
// 4. 解码多个包
|
||||
// 返回 DecodedFrames[]
|
||||
ff_decode_multi(
|
||||
ctx: number,
|
||||
pkt: number,
|
||||
frame: number,
|
||||
packets: any[],
|
||||
fin?: boolean
|
||||
): Promise<any[]>
|
||||
|
||||
// 辅助:释放资源
|
||||
avformat_close_input_js(fmt_ctx: number): Promise<void>
|
||||
ff_free_decoder(c: number, pkt: number, frame: number): Promise<void>
|
||||
}
|
||||
Reference in New Issue
Block a user