feat(project): init

This commit is contained in:
2026-02-03 18:05:47 +08:00
commit f37a119eff
188 changed files with 29246 additions and 0 deletions

View File

@@ -0,0 +1,14 @@
# Video Frame Extractor (FFmpeg.wasm)
## 运行位置
- 纯前端:所有转码/抽帧在浏览器 Web Worker + WASM 内完成,不依赖后端服务。
## 大文件最佳实践
- 优先使用 WORKERFS 挂载输入文件,避免 `writeFile(arrayBuffer)` 将整段视频复制进 WASM 内存。
- 抽帧建议走“单帧模式”:每次 `-ss <time> -frames:v 1` 只产出一张,读取后立即删除,避免在 WASM 内累积。
- 输出建议使用 image2 序列 pattern例如 `frame_%06d.jpg`)并配合 `-start_number`,避免 image2 的“必须是序列 pattern”报错路径。
- UI 端必须对日志与图片列表做上限与释放(`URL.revokeObjectURL`),否则长视频会导致 JS 堆持续增长。
## 故障恢复
- 一旦出现 `Aborted()` / `memory access out of bounds`,当前 FFmpeg.wasm 实例可能进入不可恢复状态,需要 terminate 并重建实例/worker。

View File

@@ -0,0 +1,326 @@
import {
ActionIcon,
Alert,
Box,
Button,
Card,
Container,
FileInput,
Group,
Image,
Loader,
Progress,
ScrollArea,
SimpleGrid,
Stack,
Text,
Title,
} from "@mantine/core"
import {
IconAlertCircle,
IconDownload,
IconMovie,
IconPhoto,
} from "@tabler/icons-react"
import { useEffect, useRef, useState } from "react"
import { appendCapped } from "./cap"
import type { WorkerResponse } from "./ffmpeg.worker"
export default function VideoFrameExtractor() {
const [loaded, setLoaded] = useState(false)
const [isLoading, setIsLoading] = useState(false)
const [file, setFile] = useState<File | null>(null)
const [processing, setProcessing] = useState(false)
const [progress, setProgress] = useState(0)
const [message, setMessage] = useState("Wait for loading...")
const [error, setError] = useState<string | null>(null)
const [images, setImages] = useState<{ url: string; name: string }[]>([])
const [logs, setLogs] = useState<string[]>([])
const workerRef = useRef<Worker | null>(null)
const messageRef = useRef<HTMLDivElement>(null)
const extractedCountRef = useRef(0)
const MAX_LOG_LINES = 300
const MAX_IMAGES = 1500
// Initialize Worker once
useEffect(() => {
queueMicrotask(() => {
if (!workerRef.current) {
setIsLoading(true)
const baseURL = window.location.origin + "/wasm"
const createWorker = () => {
const worker = new Worker(
new URL("./ffmpeg.worker.ts", import.meta.url)
)
workerRef.current = worker
return worker
}
const handleMessage = (event: MessageEvent<WorkerResponse>) => {
const { type } = event.data
switch (type) {
case "READY":
setLoaded(true)
setIsLoading(false)
setMessage("FFmpeg loaded successfully")
break
case "LOG":
if ("message" in event.data) {
const msg = event.data.message
setLogs((prev) => appendCapped(prev, [msg], MAX_LOG_LINES))
if (messageRef.current) {
messageRef.current.scrollTop = messageRef.current.scrollHeight
}
}
break
case "PROGRESS":
if ("progress" in event.data) {
const p = Math.round(event.data.progress * 100)
setProgress(p)
setMessage(`Processing... ${p}%`)
}
break
case "SEGMENT_DATA":
if ("files" in event.data) {
const newImages = event.data.files.map((f: any) => {
const blob = new Blob([f.data], { type: "image/jpeg" })
return { url: URL.createObjectURL(blob), name: f.name }
})
extractedCountRef.current += newImages.length
setImages((prev) =>
appendCapped(prev, newImages, MAX_IMAGES, (img) =>
URL.revokeObjectURL((img as any).url)
)
)
setMessage(
`Processing... Extracted ${extractedCountRef.current} frames.`
)
}
break
case "DONE":
setMessage(
`Completed! Extracted ${extractedCountRef.current} frames.`
)
setProcessing(false)
break
case "FATAL":
if ("error" in event.data) {
setError(event.data.error)
setProcessing(false)
setLoaded(false)
setIsLoading(true)
}
if (workerRef.current) {
workerRef.current.terminate()
workerRef.current = null
}
{
const worker = createWorker()
worker.onmessage = handleMessage
worker.postMessage({ type: "LOAD", baseURL })
}
break
case "ERROR":
if ("error" in event.data) {
setError(event.data.error)
setProcessing(false)
setIsLoading(false)
}
break
}
}
const worker = createWorker()
worker.onmessage = handleMessage
worker.postMessage({ type: "LOAD", baseURL })
}
})
// Only create worker if not exists
// Cleanup worker on unmount?
// Usually good practice, but if we want to keep it alive for navigation back/forth,
// we might want to move it to Context.
// For this task, local cleanup is fine.
return () => {
if (workerRef.current) {
workerRef.current.terminate()
workerRef.current = null
}
}
}, [])
const processVideo = () => {
if (!file || !loaded || !workerRef.current) return
setProcessing(true)
setProgress(0)
extractedCountRef.current = 0
for (const img of images) URL.revokeObjectURL(img.url)
setImages([])
setLogs([])
setError(null)
setMessage("Starting processing...")
// Extract 1 frame per second: -vf fps=1
// Optimize: Scale down to 720p max width to save memory
workerRef.current.postMessage({
type: "EXEC",
file,
args: [
"-vf",
"fps=1,scale='min(720,iw)':-1",
"-q:v",
"2",
"frame_%03d.jpg",
],
outputPattern: "frame_",
})
}
const downloadAll = () => {
images.forEach((img, index) => {
setTimeout(() => {
const a = document.createElement("a")
a.href = img.url
a.download = img.name
a.click()
}, index * 200)
})
}
return (
<Container size="lg" py="xl">
<Stack gap="lg">
<Title order={2}>Video Frame Extractor (Worker + Local WASM)</Title>
<Text c="dimmed">
Extract frames from your video files purely in the browser using
FFmpeg.wasm (Async Worker).
</Text>
{error && (
<Alert
icon={<IconAlertCircle size={16} />}
title="Error"
color="red"
withCloseButton
onClose={() => setError(null)}>
{error}
</Alert>
)}
<Card withBorder shadow="sm" p="lg" radius="md">
<Stack gap="md">
<FileInput
label="Select Video File"
placeholder="Click to select MP4, WebM..."
accept="video/*"
leftSection={<IconMovie size={16} />}
value={file}
onChange={setFile}
disabled={!loaded || processing}
/>
{!loaded && isLoading && (
<Group>
<Loader size="sm" />
<Text size="sm">Loading FFmpeg core...</Text>
</Group>
)}
{loaded && (
<Group justify="flex-end">
<Button
onClick={processVideo}
disabled={!file || processing}
loading={processing}
leftSection={<IconPhoto size={16} />}>
{processing ? "Processing..." : "Start Extraction"}
</Button>
</Group>
)}
{processing && (
<Stack gap="xs">
<Text size="sm">{message}</Text>
<Progress value={progress} animated />
</Stack>
)}
</Stack>
</Card>
{logs.length > 0 && (
<Card withBorder shadow="sm" p="xs" radius="md" bg="gray.0">
<Text size="xs" fw={500} mb="xs">
Logs
</Text>
<ScrollArea h={100} type="always" viewportRef={messageRef}>
<Box
style={{
whiteSpace: "pre-wrap",
fontFamily: "monospace",
fontSize: "11px",
}}>
{logs.map((log, i) => (
<div key={i}>{log}</div>
))}
</Box>
</ScrollArea>
</Card>
)}
{images.length > 0 && (
<Stack gap="md">
<Group justify="space-between" align="center">
<Title order={3}>Extracted Frames ({images.length})</Title>
<Button
variant="outline"
leftSection={<IconDownload size={16} />}
onClick={downloadAll}>
Download All
</Button>
</Group>
<SimpleGrid cols={{ base: 2, sm: 3, md: 4 }} spacing="md">
{images.map((img) => (
<Card
key={img.name}
shadow="sm"
padding="xs"
radius="md"
withBorder>
<Card.Section>
<Image
src={img.url}
height={160}
alt={img.name}
fit="cover"
/>
</Card.Section>
<Group justify="space-between" mt="xs" mb="xs">
<Text fw={500} size="xs" truncate>
{img.name}
</Text>
<ActionIcon
variant="light"
color="blue"
component="a"
href={img.url}
download={img.name}>
<IconDownload size={14} />
</ActionIcon>
</Group>
</Card>
))}
</SimpleGrid>
</Stack>
)}
</Stack>
</Container>
)
}

View File

@@ -0,0 +1,18 @@
import { appendCapped } from "./cap"
describe("appendCapped", () => {
test("keeps all items when under max", () => {
expect(appendCapped([1, 2], [3], 10)).toEqual([1, 2, 3])
})
test("drops from head when exceeding max", () => {
expect(appendCapped([1, 2], [3, 4, 5], 3)).toEqual([3, 4, 5])
})
test("calls onDrop for dropped items", () => {
const dropped: number[] = []
const result = appendCapped([1, 2], [3, 4, 5], 3, (x) => dropped.push(x))
expect(result).toEqual([3, 4, 5])
expect(dropped).toEqual([1, 2])
})
})

View File

@@ -0,0 +1,14 @@
export const appendCapped = <T>(
prev: T[],
next: T[],
max: number,
onDrop?: (item: T) => void
) => {
const merged = [...prev, ...next]
if (merged.length <= max) return merged
const dropCount = merged.length - max
if (onDrop) {
for (let i = 0; i < dropCount; i += 1) onDrop(merged[i])
}
return merged.slice(dropCount)
}

View File

@@ -0,0 +1,520 @@
// Define types for worker messages
export type WorkerMessage =
| { type: "LOAD"; baseURL: string }
| { type: "EXEC"; file: File; args: string[]; outputPattern: string }
export type WorkerResponse =
| { type: "READY" }
| { type: "LOG"; message: string }
| { type: "PROGRESS"; progress: number }
| { type: "DONE" }
| { type: "SEGMENT_DATA"; files: { name: string; data: Uint8Array }[] }
| { type: "FATAL"; error: string }
| { type: "ERROR"; error: string }
import { FFmpeg } from "@ffmpeg/ffmpeg"
const ctx: Worker = self as any
const ffmpeg = new FFmpeg()
let loaded = false
let lastBaseURL: string | null = null
// Custom Logger
ffmpeg.on("log", ({ message }) => {
ctx.postMessage({ type: "LOG", message })
})
ffmpeg.on("progress", ({ progress }) => {
ctx.postMessage({ type: "PROGRESS", progress })
})
ctx.onmessage = async (event: MessageEvent<WorkerMessage>) => {
const { type } = event.data
try {
switch (type) {
case "LOAD":
if (loaded) {
ctx.postMessage({ type: "READY" })
return
}
const { baseURL } = event.data
lastBaseURL = baseURL
await ffmpeg.load({
coreURL: `${baseURL}/ffmpeg-core.js`,
wasmURL: `${baseURL}/ffmpeg-core.wasm`,
})
loaded = true
ctx.postMessage({ type: "READY" })
break
case "EXEC":
if (!loaded) throw new Error("FFmpeg not loaded")
const { file, args, outputPattern } = event.data
ctx.postMessage({
type: "LOG",
message: `[start] outputPattern=${outputPattern}`,
})
const mountDir = "/input_mount"
const inputPath = `${mountDir}/${file.name}`
const mountInput = async () => {
await ffmpeg.createDir(mountDir)
await ffmpeg.mount("WORKERFS" as any, { files: [file] }, mountDir)
}
const unmountInput = async () => {
await ffmpeg.unmount(mountDir).catch(() => {})
await ffmpeg.deleteDir(mountDir).catch(() => {})
}
await mountInput()
try {
const normalizeError = (e: unknown) => {
if (typeof e === "string") return e
if (e && typeof e === "object" && "message" in e)
return String((e as any).message)
return String(e)
}
const isOom = (msg: string) =>
msg.includes("memory access out of bounds") ||
msg.includes("Cannot enlarge memory") ||
msg.includes("Aborted")
const nowMs = () =>
typeof performance !== "undefined" ? performance.now() : Date.now()
const makeThrottledLogger = (intervalMs: number) => {
let last = 0
return (message: string) => {
const t = nowMs()
if (t - last < intervalMs) return
last = t
ctx.postMessage({ type: "LOG", message })
}
}
const logThrottled = makeThrottledLogger(1000)
const findArgValue = (flag: string) => {
const idx = args.indexOf(flag)
if (idx < 0) return null
const value = args[idx + 1]
return typeof value === "string" ? value : null
}
const parseFps = () => {
const vf = findArgValue("-vf")
if (!vf) return null
const match = vf.match(/(?:^|,)fps=(\d+(?:\.\d+)?)(?:,|$)/)
if (!match) return null
const fps = Number(match[1])
return Number.isFinite(fps) && fps > 0 ? fps : null
}
let durationSec = 0
const durationProbePath = "__duration.txt"
try {
await ffmpeg.ffprobe([
"-v",
"error",
"-show_entries",
"format=duration",
"-of",
"default=noprint_wrappers=1:nokey=1",
inputPath,
"-o",
durationProbePath,
])
const txt = (await ffmpeg.readFile(
durationProbePath,
"utf8"
)) as string
const parsed = Number.parseFloat(String(txt).trim())
if (Number.isFinite(parsed) && parsed > 0) durationSec = parsed
} catch (e) {
ctx.postMessage({
type: "LOG",
message: `[probe] ffprobe failed: ${normalizeError(e)}`,
})
} finally {
await ffmpeg.deleteFile(durationProbePath).catch(() => {})
}
if (!durationSec) {
const logHandler = ({ message }: { message: string }) => {
const match = message.match(
/Duration: (\d+):(\d+):(\d+(?:\.\d+)?)/
)
if (match) {
const [, h, m, s] = match
durationSec =
parseFloat(h) * 3600 + parseFloat(m) * 60 + parseFloat(s)
}
}
ffmpeg.on("log", logHandler)
try {
await ffmpeg.exec(["-i", inputPath])
} catch (e) {
void e
} finally {
ffmpeg.off("log", logHandler)
}
}
ctx.postMessage({
type: "LOG",
message: `[probe] durationSec=${durationSec || "unknown"}`,
})
if (!durationSec) durationSec = 600
const fps = parseFps() ?? 1
const stepSec = 1 / fps
const preferSingleFrame = file.size >= 20 * 1024 * 1024
const maxFrames = preferSingleFrame ? 30 : 0
const reloadEvery = preferSingleFrame ? 10 : 0
let sentBytes = 0
const maxWidth =
file.size >= 40 * 1024 * 1024 ? 360 : preferSingleFrame ? 480 : 720
ctx.postMessage({
type: "LOG",
message: `[config] fileMB=${Math.round(file.size / 1024 / 1024)} fps=${fps} maxWidth=${maxWidth} preferSingleFrame=${preferSingleFrame}`,
})
const outputExt = (() => {
for (let i = args.length - 1; i >= 0; i -= 1) {
const v = args[i]
if (typeof v !== "string") continue
const m = v.match(/\.([a-zA-Z0-9]+)$/)
if (m) return m[1].toLowerCase()
}
return "jpg"
})()
const buildSingleFrameArgs = (
outputPattern: string,
startNumber: number
) => {
const vf = findArgValue("-vf")
const qv = findArgValue("-q:v")
const baseFilter =
vf
?.replace(/(?:^|,)fps=\d+(?:\.\d+)?(?=,|$)/g, "")
.replace(/^,|,$/g, "")
.replace(/,,+/g, ",") ?? null
const patchedFilter = (() => {
if (!baseFilter || !baseFilter.trim()) return null
let f = baseFilter
f = f.replace(/min\(\s*720\s*,\s*iw\s*\)/g, `min(${maxWidth},iw)`)
f = f.replace(/min\(\s*480\s*,\s*iw\s*\)/g, `min(${maxWidth},iw)`)
f = f.replace(/min\(\s*360\s*,\s*iw\s*\)/g, `min(${maxWidth},iw)`)
if (!/(?:^|,)format=/.test(f)) f = `${f},format=yuv420p`
return f
})()
const out: string[] = []
if (patchedFilter && patchedFilter.trim())
out.push("-vf", patchedFilter)
if (qv) {
const qNum = Number(qv)
if (Number.isFinite(qNum) && preferSingleFrame) {
out.push("-q:v", String(Math.max(qNum, 6)))
} else {
out.push("-q:v", qv)
}
}
out.push("-pix_fmt", "yuv420p")
out.push("-start_number", String(startNumber))
out.push("-frames:v", "1", outputPattern)
return out
}
const segmentCandidates = preferSingleFrame
? []
: [2, 1.2, 1].filter((v) => v >= stepSec && v > 0)
const trySegmentDuration = async (segmentDurationSec: number) => {
let t = 0
let emptyStreak = 0
let producedAny = false
while (t < durationSec) {
logThrottled(
`[exec] mode=segment t=${t.toFixed(3)} dur=${segmentDurationSec}`
)
const segmentArgs = [
"-ss",
t.toString(),
"-t",
segmentDurationSec.toString(),
"-i",
inputPath,
"-threads",
"1",
"-an",
"-sn",
"-dn",
...args,
]
const ret = await ffmpeg.exec(segmentArgs)
if (ret !== 0) {
logThrottled(
`[exec] mode=segment ret=${ret} t=${t.toFixed(3)} dur=${segmentDurationSec}`
)
}
const files = await ffmpeg.listDir(".")
const imageFiles = files.filter(
(f) =>
!f.isDir &&
f.name.startsWith("frame_") &&
f.name.endsWith(`.${outputExt}`)
)
if (imageFiles.length === 0) {
emptyStreak += 1
if (emptyStreak >= Math.ceil(5 / segmentDurationSec)) break
} else {
emptyStreak = 0
producedAny = true
}
const segmentFiles: { name: string; data: Uint8Array }[] = []
const transfer: ArrayBuffer[] = []
for (const f of imageFiles) {
const raw = (await ffmpeg.readFile(f.name)) as Uint8Array
const copy = raw.slice()
segmentFiles.push({
name: `t_${t.toFixed(3)}_${f.name}`,
data: copy,
})
transfer.push(copy.buffer)
await ffmpeg.deleteFile(f.name)
}
if (segmentFiles.length > 0) {
ctx.postMessage(
{ type: "SEGMENT_DATA", files: segmentFiles },
transfer
)
}
ctx.postMessage({
type: "PROGRESS",
progress: Math.min((t + segmentDurationSec) / durationSec, 1),
})
t += segmentDurationSec
}
if (!producedAny) {
throw new Error("NO_FRAMES")
}
}
const runSingleFrameMode = async () => {
let index = 0
let emptyStreak = 0
const pattern = `frame_%06d.${outputExt}`
const reloadCore = async (reason: string) => {
if (!lastBaseURL) return
ctx.postMessage({ type: "LOG", message: `[reload] ${reason}` })
await unmountInput()
ffmpeg.terminate()
loaded = false
await ffmpeg.load({
coreURL: `${lastBaseURL}/ffmpeg-core.js`,
wasmURL: `${lastBaseURL}/ffmpeg-core.wasm`,
})
loaded = true
await mountInput()
}
for (let t = 0; t < durationSec; t += stepSec) {
if (maxFrames > 0 && index >= maxFrames) break
if (reloadEvery > 0 && index > 0 && index % reloadEvery === 0) {
await reloadCore(`periodic frame=${index}`)
}
const outputName = `frame_${String(index).padStart(6, "0")}.${outputExt}`
const singleFrameArgs = [
"-ss",
t.toString(),
"-i",
inputPath,
"-threads",
"1",
"-an",
"-sn",
"-dn",
...(buildSingleFrameArgs(pattern, index) || []),
]
try {
const ret = await ffmpeg.exec(singleFrameArgs)
ctx.postMessage({
type: "LOG",
message: `[exec] mode=single ret=${ret} t=${t.toFixed(3)} out=${outputName}`,
})
} catch (e) {
const msg = normalizeError(e)
if (isOom(msg)) throw e
ctx.postMessage({
type: "LOG",
message: `[exec] mode=single error t=${t.toFixed(3)}: ${msg}`,
})
if (
msg.includes("At least one output file must be specified")
) {
ctx.postMessage({
type: "LOG",
message: `[exec] mode=single argsTail=${JSON.stringify(
singleFrameArgs.slice(-12)
)}`,
})
}
await ffmpeg.deleteFile(outputName).catch(() => {})
index += 1
continue
}
try {
const raw = (await ffmpeg.readFile(outputName)) as Uint8Array
const copy = raw.slice()
sentBytes += copy.byteLength
await ffmpeg.deleteFile(outputName).catch(() => {})
ctx.postMessage(
{
type: "SEGMENT_DATA",
files: [{ name: outputName, data: copy }],
},
[copy.buffer]
)
logThrottled(
`[frame] idx=${index} t=${t.toFixed(3)} size=${copy.byteLength} sentMB=${Math.round((sentBytes / 1024 / 1024) * 10) / 10}`
)
} catch (e) {
void e
emptyStreak += 1
if (emptyStreak >= 3) break
index += 1
continue
}
emptyStreak = 0
ctx.postMessage({
type: "PROGRESS",
progress: Math.min((t + stepSec) / durationSec, 1),
})
index += 1
}
}
let segmentOk = false
for (const seg of segmentCandidates) {
try {
ctx.postMessage({
type: "LOG",
message: `[mode] trying segmentDuration=${seg}s stepSec=${stepSec}s`,
})
await trySegmentDuration(seg)
segmentOk = true
break
} catch (e) {
const msg = normalizeError(e)
if (msg.includes("NO_FRAMES")) break
if (
msg.includes("Output file is empty") ||
msg.includes("nothing was encoded")
) {
break
}
if (!isOom(msg)) throw e
}
}
if (!segmentOk) {
ctx.postMessage({
type: "LOG",
message: `[mode] segment mode unavailable; fallback to single-frame sampling`,
})
await runSingleFrameMode()
}
// Send DONE without data
ctx.postMessage({ type: "DONE" })
} catch (e: any) {
const errorMessage =
typeof e === "string" ? e : e?.message || String(e)
if (errorMessage.includes("memory access out of bounds")) {
try {
ffmpeg.terminate()
loaded = false
ctx.postMessage({
type: "LOG",
message: `[fatal] ffmpeg terminated due to: ${errorMessage}`,
})
if (lastBaseURL) {
await ffmpeg.load({
coreURL: `${lastBaseURL}/ffmpeg-core.js`,
wasmURL: `${lastBaseURL}/ffmpeg-core.wasm`,
})
loaded = true
ctx.postMessage({
type: "LOG",
message: `[fatal] ffmpeg reloaded`,
})
}
} catch (e2) {
ctx.postMessage({
type: "LOG",
message: `[fatal] reload failed: ${
typeof e2 === "string"
? e2
: (e2 as any)?.message || String(e2)
}`,
})
}
ctx.postMessage({ type: "FATAL", error: errorMessage })
return
}
throw e
} finally {
// Cleanup input
try {
await unmountInput()
} catch (e) {
ctx.postMessage({
type: "LOG",
message: `[cleanup] ${typeof e === "string" ? e : (e as any)?.message || String(e)}`,
})
}
}
break
}
} catch (error: any) {
const msg =
typeof error === "string" ? error : error?.message || String(error)
ctx.postMessage({ type: "ERROR", error: msg })
}
}

View File

@@ -0,0 +1,7 @@
"use client"
import VideoFrameExtractor from "./VideoFrameExtractor"
export default function ComponentLabelFfmpegPage() {
return <VideoFrameExtractor />
}