fix(label): label result

components/label/LabelNossr.tsx

@@ -14,7 +14,12 @@ import { processVideo } from "@/app/component/label/video2image/processVideo"
 import { Box, Flex, LoadingOverlay, Stack } from "@mantine/core"
 import { showNotification } from "@mantine/notifications"
 import { getLabelResult, getServerImage } from "./api/label"
-import { Comment, WorkLoad } from "./api/label/typing"
+import {
+  Comment,
+  ImageObjects,
+  LabelResult,
+  WorkLoad,
+} from "./api/label/typing"
 import { getProjectDetail } from "./api/project"
 import { Project } from "./api/project/typing"
 import { getTaskList, getTaskWorkFlow } from "./api/task"
@@ -91,6 +96,29 @@ const normalizeNamePrefix = (name: string) => {
   return value || "video"
 }
 
+interface ResultImageEntry {
+  frameName: string
+  imageObjects: ImageObjects
+}
+
+const expandLabelResultImages = (item: LabelResult): ResultImageEntry[] => {
+  if (item.image_objects) {
+    return [
+      {
+        frameName: item.data_name,
+        imageObjects: item.image_objects,
+      },
+    ]
+  }
+  if (!item.video_objects) return []
+  return Object.entries(item.video_objects).map(([frameName, imageObjects]) => {
+    return {
+      frameName,
+      imageObjects,
+    }
+  })
+}
+
 interface LabelProps {
   headerHeight: number
   leftWidth: number
@@ -237,84 +265,95 @@ const LabelPage = ({
 
   const normalizeVideoTaskData = useCallback(
     async (detail: Task.DataProps) => {
-      try {
-        const sourceNames = detail.data_name
-          .map((item) => item?.name?.[0] || "")
-          .filter((name) => !!name)
-        if (!sourceNames.length || !projectId) return detail
+      const sourceNames = detail.data_name
+        .map((item) => item?.name?.[0] || "")
+        .filter((name) => !!name)
+      if (!sourceNames.length || !projectId || !taskId) return detail
 
-        const taskDataType = Number(
-          (detail as { [key: string]: any })?.data_type ??
-            (detail as { [key: string]: any })?.task_data_type ??
-            -1
-        )
-        const isVideoTask =
-          taskDataType === 2 ||
-          sourceNames.some((name) => videoExtPattern.test(name))
-        if (!isVideoTask) return detail
+      const taskDataType = Number(
+        (detail as { [key: string]: any })?.data_type ??
+          (detail as { [key: string]: any })?.task_data_type ??
+          -1
+      )
+      const isVideoTask =
+        taskDataType === 2 ||
+        taskDataType === 7 ||
+        sourceNames.some((name) => videoExtPattern.test(name))
+      if (!isVideoTask) return detail
 
-        const imagesMap = new Map(useImagesStore.getState().images)
-        let imageMapUpdated = false
-        const normalizedDataName: Task.DataProps["data_name"] = []
+      const imagesMap = new Map(useImagesStore.getState().images)
+      const frameMapUpdates = new Map<string, string[]>()
+      let imageMapUpdated = false
+      const normalizedDataName: Task.DataProps["data_name"] = []
 
-        for (const videoName of sourceNames) {
-          const cacheKey = `${projectId}:${taskId}:${videoName}`
-          let frameNames =
-            useVideoFrameStore.getState().videoFrames.get(cacheKey) ?? []
+      for (const videoName of sourceNames) {
+        const cacheKey = `${projectId}:${taskId}:${videoName}`
+        let frameNames =
+          useVideoFrameStore.getState().videoFrames.get(cacheKey) ?? []
 
-          const hasAllCachedFrames =
-            frameNames.length > 0 &&
-            frameNames.every((frameName) => {
-              const cache = imagesMap.get(frameName)
-              return typeof cache === "string" && cache.length > 0
-            })
+        const hasAllCachedFrames =
+          frameNames.length > 0 &&
+          frameNames.every((frameName) => {
+            const cache = imagesMap.get(frameName)
+            return typeof cache === "string" && cache.length > 0
+          })
 
-          if (!hasAllCachedFrames) {
+        if (!hasAllCachedFrames) {
+          let frameData: Awaited<ReturnType<typeof processVideo>> = []
+          try {
             const base64Video = await getServerImage({
               data_names: [videoName],
               data_type: 2,
               project_id: projectId,
             })
-            const frameData = await processVideo({
+            frameData = await processVideo({
               base64Data: String(base64Video || ""),
               fileName: videoName,
               namePrefix: normalizeNamePrefix(
                 `${projectId}_${taskId}_${videoName.replace(/\.[^.]+$/, "")}`
               ),
             })
-            frameNames = frameData.map((frame) => frame.name)
-            frameData.forEach((frame) => {
-              imagesMap.set(frame.name, frame.dataUrl)
-            })
-            imageMapUpdated = true
-            useVideoFrameStore.getState().setVideoFrames(cacheKey, frameNames)
+          } catch (error) {
+            const message = error instanceof Error ? error.message : "未知错误"
+            throw new Error(`视频帧生成失败：${videoName}（${message}）`)
           }
-
-          frameNames.forEach((frameName) => {
-            normalizedDataName.push({
-              name: [frameName] as [string],
-              related_images: [],
-            })
+          frameNames = frameData.map((frame) => frame.name)
+          if (!frameNames.length) {
+            throw new Error(`视频帧生成失败：${videoName}`)
+          }
+          frameData.forEach((frame) => {
+            imagesMap.set(frame.name, frame.dataUrl)
           })
+          imageMapUpdated = true
+          frameMapUpdates.set(cacheKey, frameNames)
         }
 
-        if (imageMapUpdated) {
-          useImagesStore.getState().setImages(imagesMap)
+        if (!frameNames.length) {
+          throw new Error(`视频帧映射为空：${videoName}`)
         }
-        if (!normalizedDataName.length) return detail
 
-        return {
-          ...detail,
-          data_name: normalizedDataName,
-        }
-      } catch (error) {
-        console.log(error)
-        showNotification({
-          color: "red",
-          title: "视频解码失败",
-          message: "视频帧生成失败，已回退到原始数据",
+        frameNames.forEach((frameName) => {
+          normalizedDataName.push({
+            name: [frameName] as [string],
+            related_images: [],
+          })
         })
-        return detail
+      }
+
+      if (!normalizedDataName.length) {
+        throw new Error("未生成可用视频帧")
+      }
+
+      if (imageMapUpdated) {
+        useImagesStore.getState().setImages(imagesMap)
+      }
+      frameMapUpdates.forEach((frameNames, cacheKey) => {
+        useVideoFrameStore.getState().setVideoFrames(cacheKey, frameNames)
+      })
+
+      return {
+        ...detail,
+        data_name: normalizedDataName,
       }
     },
     [projectId, taskId]
@@ -385,6 +424,9 @@ const LabelPage = ({
         // 获取并处理当前任务标注结果数据
         const dataRes = await getLabelResult(taskId)
         const { results } = dataRes
+        const resultImageEntries = results.flatMap((item) =>
+          expandLabelResultImages(item)
+        )
 
         let currentWorkTime = 0
         if (label_status === 2) {
@@ -411,75 +453,75 @@ const LabelPage = ({
           review_key_frame_size: {},
           question_size: {},
         }
-        results.forEach((item) => {
-          const { image_objects } = item
-          if (image_objects) {
-            const { annotations, images, key_frame, desc } = image_objects
+        resultImageEntries.forEach(({ frameName, imageObjects }) => {
+          const { annotations, images, key_frame, desc } = imageObjects
+          const rasterName = frameName || images.file_name
+          if (rasterName) {
             usePaperStore
               .getState()
-              .setRasterSize(images.file_name, [images.width, images.height])
-            annotations.forEach((annotation) => {
-              let reviewed = false
-              const { comment, prelabel, uid } = annotation
-              if (prelabel)
-                workLoad.prelabel_modify_size[uid]
-                  ? workLoad.prelabel_modify_size[uid]++
-                  : (workLoad.prelabel_modify_size[uid] = 1)
-              else
-                workLoad.object_size[uid]
-                  ? workLoad.object_size[uid]++
-                  : (workLoad.object_size[uid] = 1)
-              if (comment) {
-                comment.forEach((c) => {
-                  if (c.comment_type > 0 && c.uid === user_id) reviewed = true
-                  if (c.comment_type === 2 && c.uid === user_id)
-                    workLoad.comment_size++
-                })
-              }
-              if (reviewed) workLoad.review_size++
-            })
-            if (key_frame) {
-              if (label_status === 2) {
-                let id = image_objects.label1_uid!
-                if (id)
-                  workLoad.key_frame_size[id]
-                    ? workLoad.key_frame_size[id]++
-                    : (workLoad.key_frame_size[id] = 1)
-              } else if (label_status === 4) {
-                let id = image_objects.review1_uid!
-                if (id)
-                  workLoad.review_key_frame_size[id]
-                    ? workLoad.review_key_frame_size[id]++
-                    : (workLoad.review_key_frame_size[id] = 1)
-              } else if (label_status === 6) {
-                let id = image_objects.review2_uid!
-                if (id)
-                  workLoad.review_key_frame_size[id]
-                    ? workLoad.review_key_frame_size[id]++
-                    : (workLoad.review_key_frame_size[id] = 1)
-              }
+              .setRasterSize(rasterName, [images.width, images.height])
+          }
+          annotations.forEach((annotation) => {
+            let reviewed = false
+            const { comment, prelabel, uid } = annotation
+            if (prelabel)
+              workLoad.prelabel_modify_size[uid]
+                ? workLoad.prelabel_modify_size[uid]++
+                : (workLoad.prelabel_modify_size[uid] = 1)
+            else
+              workLoad.object_size[uid]
+                ? workLoad.object_size[uid]++
+                : (workLoad.object_size[uid] = 1)
+            if (comment) {
+              comment.forEach((c) => {
+                if (c.comment_type > 0 && c.uid === user_id) reviewed = true
+                if (c.comment_type === 2 && c.uid === user_id)
+                  workLoad.comment_size++
+              })
             }
-            if (useDescToolsStore.getState().qaOperations.length) {
-              if (desc) {
-                const { other_desc } = desc
-                const descData = other_desc ? JSON.parse(other_desc) : {}
-                Object.values(descData).forEach((questionArr: any) => {
-                  questionArr.forEach((q: any) => {
-                    const v: any = Object.values(q)[0]
-                    if (v.uid) {
-                      workLoad.question_size[v.uid]
-                        ? workLoad.question_size[v.uid]++
-                        : (workLoad.question_size[v.uid] = 1)
-                    }
-                    if (v.tag) {
-                      const uid = usePermissionStore.getState().user_id
-                      workLoad.review_question_size[uid]
-                        ? workLoad.review_question_size[uid]++
-                        : (workLoad.review_question_size[uid] = 1)
-                    }
-                  })
+            if (reviewed) workLoad.review_size++
+          })
+          if (key_frame) {
+            if (label_status === 2) {
+              let id = imageObjects.label1_uid!
+              if (id)
+                workLoad.key_frame_size[id]
+                  ? workLoad.key_frame_size[id]++
+                  : (workLoad.key_frame_size[id] = 1)
+            } else if (label_status === 4) {
+              let id = imageObjects.review1_uid!
+              if (id)
+                workLoad.review_key_frame_size[id]
+                  ? workLoad.review_key_frame_size[id]++
+                  : (workLoad.review_key_frame_size[id] = 1)
+            } else if (label_status === 6) {
+              let id = imageObjects.review2_uid!
+              if (id)
+                workLoad.review_key_frame_size[id]
+                  ? workLoad.review_key_frame_size[id]++
+                  : (workLoad.review_key_frame_size[id] = 1)
+            }
+          }
+          if (useDescToolsStore.getState().qaOperations.length) {
+            if (desc) {
+              const { other_desc } = desc
+              const descData = other_desc ? JSON.parse(other_desc) : {}
+              Object.values(descData).forEach((questionArr: any) => {
+                questionArr.forEach((q: any) => {
+                  const v: any = Object.values(q)[0]
+                  if (v.uid) {
+                    workLoad.question_size[v.uid]
+                      ? workLoad.question_size[v.uid]++
+                      : (workLoad.question_size[v.uid] = 1)
+                  }
+                  if (v.tag) {
+                    const uid = usePermissionStore.getState().user_id
+                    workLoad.review_question_size[uid]
+                      ? workLoad.review_question_size[uid]++
+                      : (workLoad.review_question_size[uid] = 1)
+                  }
                 })
-              }
+              })
             }
           }
         })
@@ -492,180 +534,178 @@ const LabelPage = ({
         // 重置 PathGroupIds
         useRightToolsStore.getState().setPathGroupIds([])
         // 生成标注结果
-        results.forEach((item) => {
-          const { data_name, image_objects } = item
-          if (image_objects) {
-            const categoryMap = new Map()
-            const { annotations, image_groups, desc } = image_objects
-            // 赋值用过的 id
-            annotations.map((annotation) => {
-              useRightToolsStore.getState().pushPathId(annotation.id)
+        resultImageEntries.forEach(({ frameName, imageObjects }) => {
+          const key = frameName
+          if (!key) return
+          const categoryMap = new Map()
+          const { annotations, image_groups, desc } = imageObjects
+          // 赋值用过的 id
+          annotations.map((annotation) => {
+            useRightToolsStore.getState().pushPathId(annotation.id)
+          })
+          // 赋值用过的 groupId
+          image_groups &&
+            Object.keys(image_groups)?.map((groupId) => {
+              useRightToolsStore.getState().pushPathGroupId(+groupId)
             })
-            // 赋值用过的 groupId
-            image_groups &&
-              Object.keys(image_groups)?.map((groupId) => {
-                useRightToolsStore.getState().pushPathGroupId(+groupId)
-              })
-            annotations.forEach((annotation) => {
-              const { id, category_id, comment } = annotation
-              let current_comment: Comment[] = []
-              if (comment) {
-                current_comment = flowCommentArr.map((c) => {
-                  const { turn, times } = c
-                  let modified_comment = null
-                  comment.forEach((item) => {
-                    if (item.turn === turn && item.times === times)
-                      modified_comment = item
-                  })
-                  return modified_comment ? modified_comment : c
+          annotations.forEach((annotation) => {
+            const { id, category_id, comment } = annotation
+            let current_comment: Comment[] = []
+            if (comment) {
+              current_comment = flowCommentArr.map((c) => {
+                const { turn, times } = c
+                let modified_comment = null
+                comment.forEach((item) => {
+                  if (item.turn === turn && item.times === times)
+                    modified_comment = item
                 })
-              } else {
-                current_comment = [...flowCommentArr]
-              }
-              let detail: any = {
-                ...annotation,
-                comment: current_comment,
-              }
-              // group info
-              if (image_groups) {
-                const key = findGroupKey(image_groups as any, id)
-                if (key !== -1) detail.parentGroupId = key
-              }
-              const categoryKey = `${category_id}`
-              let arr = categoryMap.has(categoryKey)
-                ? categoryMap.get(categoryKey)
-                : []
-              if (annotation.rect) {
-                // arr.push([id, getRectPoints(annotation.rect), detail, []]);
-                arr.push([id, [getRectPoints(annotation.rect)], detail, []])
-              }
-
-              if (annotation.line_segment) {
-                arr.push([
-                  id,
-                  annotation.line_segment.map((item) => getSegmentPoints(item)),
-                  detail,
-                  [],
-                ])
-              }
-
-              if (annotation.segmentation) {
-                arr.push([
-                  id,
-                  // getSegmentPoints(annotation.segmentation[0]),
-                  annotation.segmentation.map((item) => getSegmentPoints(item)),
-                  detail,
-                  annotation.hollow_segmentation?.map((item) =>
-                    getSegmentPoints(item)
-                  ) || [],
-                ])
-              }
-              if (annotation.key_points) {
-                arr.push([
-                  id,
-                  // getSegmentPoints(annotation.segmentation[0]),
-                  [
-                    getSegmentPoints(
-                      annotation.key_points.map((item) => item.point)
-                    ),
-                  ],
-                  Object.assign(detail, {
-                    circles: annotation.key_points,
-                  }),
-                  [],
-                ])
-              }
-
-              categoryMap.set(categoryKey, arr)
-            })
-            const key = data_name
-            taskLabelData.set(key, categoryMap)
-            // 关键帧信息
-            const keyFrame = {
-              key_frame: image_objects.key_frame || false,
-              label1_ts: image_objects.label1_ts || 0,
-              label1_uid: image_objects.label1_uid || 0,
-              review1_ts: image_objects.review1_ts || 0,
-              review1_uid: image_objects.review1_uid || 0,
-              review2_ts: image_objects.review2_ts || 0,
-              review2_uid: image_objects.review2_uid || 0,
+                return modified_comment ? modified_comment : c
+              })
+            } else {
+              current_comment = [...flowCommentArr]
             }
-            useBottomToolsStore.getState().setKeyFrameData(key, keyFrame)
-            // 大语言模型 两种数据情况
-            if (useDescToolsStore.getState().qaOperations.length) {
-              if (desc) {
-                const { other_desc } = desc
-                let current_data: any = {}
-                const descData = other_desc ? JSON.parse(other_desc) : {}
-                Object.entries(descData).forEach(
-                  ([label_class, questionArr]: any) => {
-                    let arr = questionArr.map((question: any) => {
-                      let [name, val]: any = Object.entries(question)[0]
-                      return {
-                        [name]: {
-                          ...val,
-                          value: val.value
-                            ? `${val.value}${splitWord}${val.value}`
-                            : undefined,
-                          comment: val.comment
-                            ? `${val.comment}${splitWord}${val.comment}`
-                            : undefined,
-                        },
-                      }
-                    })
-                    current_data[label_class] = arr
-                  }
-                )
-                const qaData = useDescToolsStore.getState().qaData
-                // 无文本描述时 赋予预设值
-                if (JSON.stringify(current_data) === "{}") {
-                  current_data = qaData.get("init")
-                }
-                const qaMap = structuredClone(qaData)
-                qaMap.set(key, current_data)
-                useDescToolsStore.getState().setQaData(qaMap)
-              }
-            } else if (useDescToolsStore.getState().descOperations.length) {
-              if (desc) {
-                const { meta_operation, other_desc } = desc
-                const currentMetaMap = structuredClone(
-                  useDescToolsStore.getState().metaData
-                )
-                currentMetaMap.set(key, meta_operation || [])
-                useDescToolsStore.getState().setMetaData(currentMetaMap)
-                const currentDescMap = structuredClone(
-                  useDescToolsStore.getState().descData
-                )
-                let descMap = new Map()
-                const descData = other_desc ? JSON.parse(other_desc) : {}
-                console.log(descData)
-                Object.entries(descData).forEach(
-                  ([label_class, { value, is_correct, comment }]: any) => {
-                    descMap.set(label_class, {
-                      value:
-                        (typeof value === "string"
-                          ? `${value}${splitWord}${value}`
-                          : value) || undefined,
-                      is_correct: is_correct || undefined,
-                      comment: comment || undefined,
-                    })
-                  }
-                )
-                currentDescMap.set(key, descMap)
-                useDescToolsStore.getState().setDescData(currentDescMap)
-              }
+            let detail: any = {
+              ...annotation,
+              comment: current_comment,
             }
-            image_groups &&
-              pathGroupData.set(
-                key,
-                new Map(
-                  Object.entries(image_groups).map(([key, value]: any) => [
-                    key * 1,
-                    value,
-                  ])
-                )
-              )
+            // group info
+            if (image_groups) {
+              const key = findGroupKey(image_groups as any, id)
+              if (key !== -1) detail.parentGroupId = key
+            }
+            const categoryKey = `${category_id}`
+            let arr = categoryMap.has(categoryKey)
+              ? categoryMap.get(categoryKey)
+              : []
+            if (annotation.rect) {
+              // arr.push([id, getRectPoints(annotation.rect), detail, []]);
+              arr.push([id, [getRectPoints(annotation.rect)], detail, []])
+            }
+
+            if (annotation.line_segment) {
+              arr.push([
+                id,
+                annotation.line_segment.map((item) => getSegmentPoints(item)),
+                detail,
+                [],
+              ])
+            }
+
+            if (annotation.segmentation) {
+              arr.push([
+                id,
+                // getSegmentPoints(annotation.segmentation[0]),
+                annotation.segmentation.map((item) => getSegmentPoints(item)),
+                detail,
+                annotation.hollow_segmentation?.map((item) =>
+                  getSegmentPoints(item)
+                ) || [],
+              ])
+            }
+            if (annotation.key_points) {
+              arr.push([
+                id,
+                // getSegmentPoints(annotation.segmentation[0]),
+                [
+                  getSegmentPoints(
+                    annotation.key_points.map((item) => item.point)
+                  ),
+                ],
+                Object.assign(detail, {
+                  circles: annotation.key_points,
+                }),
+                [],
+              ])
+            }
+
+            categoryMap.set(categoryKey, arr)
+          })
+          taskLabelData.set(key, categoryMap)
+          // 关键帧信息
+          const keyFrame = {
+            key_frame: imageObjects.key_frame || false,
+            label1_ts: imageObjects.label1_ts || 0,
+            label1_uid: imageObjects.label1_uid || 0,
+            review1_ts: imageObjects.review1_ts || 0,
+            review1_uid: imageObjects.review1_uid || 0,
+            review2_ts: imageObjects.review2_ts || 0,
+            review2_uid: imageObjects.review2_uid || 0,
           }
+          useBottomToolsStore.getState().setKeyFrameData(key, keyFrame)
+          // 大语言模型 两种数据情况
+          if (useDescToolsStore.getState().qaOperations.length) {
+            if (desc) {
+              const { other_desc } = desc
+              let current_data: any = {}
+              const descData = other_desc ? JSON.parse(other_desc) : {}
+              Object.entries(descData).forEach(
+                ([label_class, questionArr]: any) => {
+                  let arr = questionArr.map((question: any) => {
+                    let [name, val]: any = Object.entries(question)[0]
+                    return {
+                      [name]: {
+                        ...val,
+                        value: val.value
+                          ? `${val.value}${splitWord}${val.value}`
+                          : undefined,
+                        comment: val.comment
+                          ? `${val.comment}${splitWord}${val.comment}`
+                          : undefined,
+                      },
+                    }
+                  })
+                  current_data[label_class] = arr
+                }
+              )
+              const qaData = useDescToolsStore.getState().qaData
+              // 无文本描述时 赋予预设值
+              if (JSON.stringify(current_data) === "{}") {
+                current_data = qaData.get("init")
+              }
+              const qaMap = structuredClone(qaData)
+              qaMap.set(key, current_data)
+              useDescToolsStore.getState().setQaData(qaMap)
+            }
+          } else if (useDescToolsStore.getState().descOperations.length) {
+            if (desc) {
+              const { meta_operation, other_desc } = desc
+              const currentMetaMap = structuredClone(
+                useDescToolsStore.getState().metaData
+              )
+              currentMetaMap.set(key, meta_operation || [])
+              useDescToolsStore.getState().setMetaData(currentMetaMap)
+              const currentDescMap = structuredClone(
+                useDescToolsStore.getState().descData
+              )
+              let descMap = new Map()
+              const descData = other_desc ? JSON.parse(other_desc) : {}
+              console.log(descData)
+              Object.entries(descData).forEach(
+                ([label_class, { value, is_correct, comment }]: any) => {
+                  descMap.set(label_class, {
+                    value:
+                      (typeof value === "string"
+                        ? `${value}${splitWord}${value}`
+                        : value) || undefined,
+                    is_correct: is_correct || undefined,
+                    comment: comment || undefined,
+                  })
+                }
+              )
+              currentDescMap.set(key, descMap)
+              useDescToolsStore.getState().setDescData(currentDescMap)
+            }
+          }
+          image_groups &&
+            pathGroupData.set(
+              key,
+              new Map(
+                Object.entries(image_groups).map(([key, value]: any) => [
+                  key * 1,
+                  value,
+                ])
+              )
+            )
         })
         setLabel(taskLabelData)
         // 获取任务详情时设置状态栈初始值
@@ -675,9 +715,18 @@ const LabelPage = ({
       } catch (err) {
         console.log(err)
         setOldWorkLoad(null)
+        setTaskDetail(undefined)
+        useBottomToolsStore.getState().setAllItems([])
         setLabel(new Map())
         setStateStack([])
         useRightToolsStore.getState().setPathGroupMap(new Map())
+        if (err instanceof Error && err.message.includes("视频")) {
+          showNotification({
+            color: "red",
+            title: "视频解码失败",
+            message: err.message,
+          })
+        }
       } finally {
         usePaperStore.getState().setLoadingData(false)
       }