feat(label): video process

components/label/LabelNossr.tsx

@@ -12,12 +12,13 @@ import ScaleComponent from "./components/ScaleComponent"
 
 import { Box, Flex, LoadingOverlay, Stack } from "@mantine/core"
 import { showNotification } from "@mantine/notifications"
-import { getLabelResult } from "./api/label"
+import { getLabelResult, getServerImage } from "./api/label"
 import { Comment, WorkLoad } from "./api/label/typing"
 import { getProjectDetail } from "./api/project"
 import { Project } from "./api/project/typing"
 import { getTaskList, getTaskWorkFlow } from "./api/task"
 import { Task } from "./api/task/typing"
+import { processVideo } from "@/app/component/label/video2image/processVideo"
 import BottomTools from "./components/BottomTools"
 import ConfirmModal from "./components/ConfirmModal"
 import { splitWord } from "./components/EditorContainer"
@@ -29,7 +30,13 @@ import RightQATools from "./components/RightQATools"
 import RightTaskTools from "./components/RightTaskTools"
 import ScaleToolContainer from "./components/ScaleToolContainer"
 import TopTools from "./components/TopTools"
-import { useKeyEventStore, useLabelStore, useObjectStore } from "./store"
+import {
+  useImagesStore,
+  useKeyEventStore,
+  useLabelStore,
+  useObjectStore,
+  useVideoFrameStore,
+} from "./store"
 import { usePermissionStore } from "./store/auth"
 import { useBottomToolsStore } from "./useBottomToolsStore"
 import { useDescToolsStore } from "./useDescToolsStore"
@@ -74,6 +81,16 @@ const getSegmentPoints = (arr: Array<number[]>) => {
   return arr.map((item) => [item[0], item[1]])
 }
 
+const videoExtPattern = /\.(mp4|mov|avi|mkv|webm|h264|264|ts|m4v)$/i
+
+const normalizeNamePrefix = (name: string) => {
+  const value = name
+    .replace(/[^a-zA-Z0-9_-]+/g, "_")
+    .replace(/_+/g, "_")
+    .replace(/^_|_$/g, "")
+  return value || "video"
+}
+
 interface LabelProps {
   headerHeight: number
   leftWidth: number
@@ -218,6 +235,91 @@ const LabelPage = ({
     setQaOperations,
   ])
 
+  const normalizeVideoTaskData = useCallback(
+    async (detail: Task.DataProps) => {
+      try {
+        const sourceNames = detail.data_name
+          .map((item) => item?.name?.[0] || "")
+          .filter((name) => !!name)
+        if (!sourceNames.length || !projectId) return detail
+
+        const taskDataType = Number(
+          (detail as { [key: string]: any })?.data_type ??
+            (detail as { [key: string]: any })?.task_data_type ??
+            -1
+        )
+        const isVideoTask =
+          taskDataType === 2 ||
+          sourceNames.some((name) => videoExtPattern.test(name))
+        if (!isVideoTask) return detail
+
+        const imagesMap = new Map(useImagesStore.getState().images)
+        let imageMapUpdated = false
+        const normalizedDataName: Task.DataProps["data_name"] = []
+
+        for (const videoName of sourceNames) {
+          const cacheKey = `${projectId}:${taskId}:${videoName}`
+          let frameNames =
+            useVideoFrameStore.getState().videoFrames.get(cacheKey) ?? []
+
+          const hasAllCachedFrames =
+            frameNames.length > 0 &&
+            frameNames.every((frameName) => {
+              const cache = imagesMap.get(frameName)
+              return typeof cache === "string" && cache.length > 0
+            })
+
+          if (!hasAllCachedFrames) {
+            const base64Video = await getServerImage({
+              data_names: [videoName],
+              data_type: 2,
+              project_id: projectId,
+            })
+            const frameData = await processVideo({
+              base64Data: String(base64Video || ""),
+              fileName: videoName,
+              namePrefix: normalizeNamePrefix(
+                `${projectId}_${taskId}_${videoName.replace(/\.[^.]+$/, "")}`
+              ),
+            })
+            frameNames = frameData.map((frame) => frame.name)
+            frameData.forEach((frame) => {
+              imagesMap.set(frame.name, frame.dataUrl)
+            })
+            imageMapUpdated = true
+            useVideoFrameStore.getState().setVideoFrames(cacheKey, frameNames)
+          }
+
+          frameNames.forEach((frameName) => {
+            normalizedDataName.push({
+              name: [frameName] as [string],
+              related_images: [],
+            })
+          })
+        }
+
+        if (imageMapUpdated) {
+          useImagesStore.getState().setImages(imagesMap)
+        }
+        if (!normalizedDataName.length) return detail
+
+        return {
+          ...detail,
+          data_name: normalizedDataName,
+        }
+      } catch (error) {
+        console.log(error)
+        showNotification({
+          color: "red",
+          title: "视频解码失败",
+          message: "视频帧生成失败，已回退到原始数据",
+        })
+        return detail
+      }
+    },
+    [projectId, taskId]
+  )
+
   const asyncGetTaskDetail = useCallback(async () => {
     if (taskId) {
       try {
@@ -230,6 +332,7 @@ const LabelPage = ({
           page_size: 1,
         })
         let detail = res.task_list[0]
+        detail = await normalizeVideoTaskData(detail)
         setTaskDetail(detail)
         // 帧列表
         useBottomToolsStore
@@ -579,7 +682,14 @@ const LabelPage = ({
         usePaperStore.getState().setLoadingData(false)
       }
     }
-  }, [setLabel, setLabelTime, setStateStack, taskId, user_id])
+  }, [
+    normalizeVideoTaskData,
+    setLabel,
+    setLabelTime,
+    setStateStack,
+    taskId,
+    user_id,
+  ])
 
   // 保存时更新当前workload
   const updateOldWorkLoad = (data: WorkLoad) => {

components/label/api/task/typing.ts

@@ -2,6 +2,8 @@ export namespace Task {
   export interface DataProps {
     id: number
     create_date: string
+    data_type?: number
+    task_data_type?: number
     data_name: {
       name: [string]
       related_images: []
@@ -62,6 +64,7 @@ export namespace Task {
   }
 
   export interface ListRequest {
+    data_type?: number
     current_uid?: number[]
     data_name?: string
     first_commit_label_time_end?: string

components/label/store.ts

@@ -498,6 +498,12 @@ interface ImageState {
   setImages: (value: any) => void
 }
 
+interface VideoFrameState {
+  videoFrames: Map<string, string[]>
+  setVideoFrames: (key: string, value: string[]) => void
+  removeVideoFrames: (key: string) => void
+}
+
 export const useImagesStore = create(
   persist<ImageState>(
     (storeSet) => ({
@@ -516,3 +522,33 @@ export const useImagesStore = create(
     }
   )
 )
+
+export const useVideoFrameStore = create(
+  persist<VideoFrameState>(
+    (storeSet) => ({
+      videoFrames: new Map(),
+      setVideoFrames: (key: string, value: string[]) =>
+        storeSet((state) => {
+          const nextMap = new Map(state.videoFrames)
+          nextMap.set(key, value)
+          return {
+            ...state,
+            videoFrames: nextMap,
+          }
+        }),
+      removeVideoFrames: (key: string) =>
+        storeSet((state) => {
+          const nextMap = new Map(state.videoFrames)
+          nextMap.delete(key)
+          return {
+            ...state,
+            videoFrames: nextMap,
+          }
+        }),
+    }),
+    {
+      name: "video-frame-storage",
+      storage: storage,
+    }
+  )
+)