diff --git a/components/label/LabelNossr.tsx b/components/label/LabelNossr.tsx index e3330a8..1875527 100644 --- a/components/label/LabelNossr.tsx +++ b/components/label/LabelNossr.tsx @@ -2267,6 +2267,7 @@ const LabelPage = ({ // 追踪 if (e.key === "i") { e.preventDefault() + // sam2 暂未有图片接口 paperContainerRef.current.renderTrackingAnnotation() } diff --git a/components/label/api/label/index.ts b/components/label/api/label/index.ts index c32f8e8..47f4f58 100644 --- a/components/label/api/label/index.ts +++ b/components/label/api/label/index.ts @@ -108,18 +108,6 @@ export const getLabelImage = (activeImage: string, path: string) => { }) } -// 辅助标注 -export const getAuxiliaryAnnotation = (data: any) => { - return httpFetch({ - url: BASE_LABEL_API + "/api/model_server/sam2/sa", - method: "POST", - body: JSON.stringify(data), - headerAttr: { - responseType: "arraybuffer", - }, - }) -} - const BASE_SAM_API = process.env.NEXT_PUBLIC_ENV === "production" ? "http://172.16.103.224:8000" @@ -127,7 +115,7 @@ const BASE_SAM_API = ? "http://172.30.21.211:8000" : "http://172.30.21.211:8000" -export const getAuxiliaryAnnotation2 = (data: { +export const getAuxiliaryAnnotation = (data: { project_id: number image_name: string prompt: { @@ -146,18 +134,6 @@ export const getAuxiliaryAnnotation2 = (data: { }) } -// 模型标注 追踪 -export const getTrackingAuxiliaryAnnotation = (data: any) => { - return httpFetch({ - url: BASE_LABEL_API + "/api/model_server/sam2/vos", - method: "POST", - body: JSON.stringify(data), - headerAttr: { - responseType: "arraybuffer", - }, - }) -} - // 用户登录 export const userLogin = (data: { name: string; password: string }) => { return httpFetch({ diff --git a/components/label/components/PaperContainer.renderTrackingAnnotation.md b/components/label/components/PaperContainer.renderTrackingAnnotation.md deleted file mode 100644 index ef53d7b..0000000 --- a/components/label/components/PaperContainer.renderTrackingAnnotation.md +++ /dev/null @@ -1,96 +0,0 @@ -# `renderTrackingAnnotation` 流程说明 - -来源文件： - -- `labelimage/components/label/components/PaperContainer.tsx` -- `labelimage/components/label/LabelNossr.tsx` -- `labelimage/components/label/api/label/index.ts` - -## 概述 - -`renderTrackingAnnotation` 是单对象跨帧追踪的入口方法。 - -它会把当前激活图片上选中的标注对象作为 seed，将其轮廓数据发送给追踪模型接口，再把模型返回的后续帧轮廓结果写回 `label store`。 - -## Mermaid - -```mermaid -flowchart TD - A["在 `LabelNossr.tsx` 中按下 `i`"] --> B["调用 `paperContainerRef.current.renderTrackingAnnotation()`"] - B --> C["执行 `updateLoadingFlag(true)`
并显示加载通知"] - C --> D["从 store 读取运行时状态：
`selectedPath[activeImage]`
`selectedItems`
`rasterSize[activeImage]`
`rasterScale[activeImage]`"] - - D --> E{"是否恰好选中了一个标注对象？"} - E -->|未选中| F["提示错误：
`请先选中标注对象后再进行模型调用`"] - E -->|选中了多个| G["提示错误：
`请勿选择多个标注对象进行模型调用`"] - E -->|是| H["在 `label.get(activeImage)` 中查找
满足 `item[0] === id` 的标注对象"] - - H --> I{"是否找到 `categoryId` 和 `data`？"} - I -->|否| Z["跳过请求参数构造"] - I -->|是| J["提取源标注几何数据：
`data[1]` => 外轮廓 `segmentation`
`data[3]` => 镂空轮廓 `hollow_segmentation`"] - - J --> K["将画布坐标转换为原图坐标：
`Math.round(x / currentScale)`
`Math.round(y / currentScale)`"] - K --> L["构造请求参数：
`project_id`
`file_names = [activeImage, ...selectedItems]`
`image_hw = [height, width]`
`contours`
`hollows`"] - L --> M["使用 `msgpack.encode(params)` 序列化"] - M --> N["通过 `getTrackingAuxiliaryAnnotation()`
POST 到 `/api/model_server/sam2/vos`"] - N --> O["以 `arraybuffer` 形式接收响应体"] - O --> P["使用
`msgpack.decode(new Uint8Array(res))`
解码响应数据"] - - P --> Q{"resData.msg === 'success'？"} - Q -->|否| R["内层流程仅执行 `console.log(error/resData)`"] - Q -->|是| S["读取响应中的 `future_contours`"] - - S --> T["克隆当前 `label store`：
`safeClone(useLabelStore.getState().label)`"] - T --> U["遍历每个未来帧结果：
目标文件 = `file_names[index + 1]`"] - U --> V["读取目标帧缩放比例：
`rasterScale[fileName]`"] - V --> W["基于 `pathIds` 生成 `newId` 并注册"] - W --> X["将原图坐标还原为画布坐标：
`x * imgScale`
`y * imgScale`"] - X --> Y["将返回轮廓拆分为：
`segmentation`
`hollow_segmentation`"] - Y --> AA["构造标注元组：
`[newId, segmentation, detail, hollow_segmentation]`"] - AA --> AB["将结果合并到
`Map>`"] - AB --> AC["持久化结果：
`setLabel(nowTaskData)`
`pushStateStack(nowTaskData)`"] - AC --> AD["显示成功通知"] - - F --> AE["finally：执行 `updateLoadingFlag(false)`"] - G --> AE - Z --> AE - R --> AE - AD --> AE - C --> AF["外层 `try/catch`"] - AF -->|异常| AG["显示通用失败通知：
`模型生成失败`"] - AG --> AE -``` - -## 数据结构 - -被追踪的源标注对象，使用和普通多边形标注一致的元组结构： - -```ts -;[id, segmentation, detail, hollow_segmentation] -``` - -各字段含义： - -- `item[0]`：标注 id -- `item[1]`：外轮廓多边形 -- `item[2]`：详情元数据 -- `item[3]`：镂空轮廓多边形 - -## 为什么使用 `arraybuffer` - -这条追踪请求并不是从头到尾都按普通 JSON 方式传输。 - -- 请求参数先通过 `msgpack.encode(params)` 打包。 -- API 封装层把这份打包后的数据发给后端。 -- 响应体以 `arraybuffer` 的形式返回。 -- `renderTrackingAnnotation` 再通过 `msgpack.decode(new Uint8Array(res))` 还原出 JavaScript 对象。 - -因此这里必须配置 `responseType: "arraybuffer"`，因为前端预期拿到的是二进制响应体，再自行解码。 - -## 备注 - -- 当前帧是 seed 帧，模型返回的结果会写入 `file_names[index + 1]` 对应的后续帧。 -- 这个方法更新的是 `label store`，不会直接在当前页面把其他帧的结果立刻画出来。 -- 这个方法只支持一次追踪一个被选中的对象。 -- 内层请求失败时目前主要是 `console.log`，不一定总会显示专门的失败提示。 -- 这个方法里生成的 `detail.image_id` 使用的是 `activeImage`，建议再结合整体数据模型确认一下是否符合预期。 diff --git a/components/label/components/PaperContainer.tsx b/components/label/components/PaperContainer.tsx index fd5c1fc..e8c197a 100644 --- a/components/label/components/PaperContainer.tsx +++ b/components/label/components/PaperContainer.tsx @@ -18,7 +18,7 @@ import { import { useForm } from "@mantine/form" import { notifications } from "@mantine/notifications" import dayjs from "dayjs" -import msgpack from "msgpack-lite" +// import msgpack from "msgpack-lite" import paper from "paper" import React, { DispatchWithoutAction, @@ -31,12 +31,7 @@ import React, { useRef, useState, } from "react" -import { - getAuxiliaryAnnotation, - getAuxiliaryAnnotation2, - getServerImage, - getTrackingAuxiliaryAnnotation, -} from "../api/label" +import { getAuxiliaryAnnotation, getServerImage } from "../api/label" import { Project } from "../api/project/typing" import { labelimagePerformanceConfig } from "../config/performance" import { LabelState } from "../LabelNossr" @@ -185,6 +180,7 @@ const PaperContainer = ( ref: React.Ref | undefined ) => { const { forceUpdate, projectDetail, labelState, updateLoadingFlag } = props + false && updateLoadingFlag const containerRef = useRef(null) const contextMenuRef = useRef(null) const [imgSize, setImageSize] = useState<{ @@ -884,164 +880,6 @@ const PaperContainer = ( }, [clearPendingFrameSwitch, loadingData]) const renderSupportAnnotation = useCallback( - async ( - points: Array<[number, number]>, - tags: number[], - clear_previous_state = false - ) => { - try { - notifications.show({ - id: "sam2", - message: "模型生成中", - loading: true, - autoClose: false, - }) - const operationSchema = - projectDetail?.label_schema_list?.find( - (item) => - item.category_id === +useTopToolsStore.getState().activeOperation - ) || null - const strokeColor = operationSchema - ? `rgb(${operationSchema.color[0]},${operationSchema.color[1]},${operationSchema.color[2]})` - : "rgb(0,0,0)" - const fillColor = operationSchema - ? `rgba(${operationSchema.color[0]},${operationSchema.color[1]},${operationSchema.color[2]},0.3)` - : "rgba(0,0,0,0.3)" - const blankColor = operationSchema - ? `rgba(${operationSchema.color[0]},${operationSchema.color[1]},${operationSchema.color[2]},0.01)` - : "rgba(0,0,0,0.01)" - const currentScale = - usePaperStore.getState().rasterScale[activeImage] ?? 1 - - const draw = () => { - points.forEach((point, index) => { - const drawPoint = new paper.Path.Circle({ - center: point, - radius: 2, - fillColor: tags[index] === 1 ? "red" : "blue", - strokeColor: tags[index] === 1 ? "red" : "blue", - data: { - id: "support", - type: "point", - }, - }) - drawPoint.parent = usePaperStore.getState().group! - const raster = usePaperStore - .getState() - .group!.getItems({ data: { name: "pic" } })[0] - if (!raster.contains(drawPoint.position)) { - throw new Error("点位超出图片范围") - } - }) - } - console.log(points, tags) - // 绘制点位 - clearSupportAnnotationItem() - draw() - - const picSize = usePaperStore.getState().rasterSize[activeImage] - - const params = { - project_id: projectDetail?.id || 0, - file_name: activeImage, - image_hw: - picSize && picSize.length ? [picSize[1], picSize[0]] : [0, 0], - points_and_box: points.map(([x, y]) => [ - Math.round(x / currentScale), - Math.round(y / currentScale), - ]), - tags, - clear_previous_state, - } - try { - const res = await getAuxiliaryAnnotation( - msgpack.encode(params) as any - ) - const data = msgpack.decode(new Uint8Array(res)) - console.log(data) - if (data.msg === "success") { - const { contours, hollows } = data - const outerPaths: any[] = [] - const innerPaths: any[] = [] - contours.forEach((contour: any, index: number) => { - if (contour.length > 3) { - const path = new paper.Path({ - segments: contour.map(([x, y]: [number, number]) => [ - x * currentScale, - y * currentScale, - ]), - fillColor: blankColor, - strokeColor: strokeColor, - closed: true, - data: { - id: "support", - type: "polygon", - isHollowPolygon: false, - fillColor, - strokeColor, - blankColor, - }, - parent: usePaperStore.getState().group, - strokeScaling: false, - }) - if ( - Math.abs(path.area) < useTopToolsStore.getState().checkSize - ) { - path.remove() - } else { - // path.scaling = usePaperStore.getState().group!.scaling; - if (hollows[index]) { - // 更新标志位 - path.data.isHollowPolygon = true - innerPaths.push(path) - } else { - // 上色 - path.fillColor = new paper.Color(fillColor) - outerPaths.push(path) - } - } - } - }) - const compoundPath = new paper.CompoundPath({ - children: [...outerPaths, ...innerPaths], - data: { - id: "support", - type: "parent", - }, - strokeColor: strokeColor, - strokeWidth: 2, - fillColor: fillColor, - }) - compoundPath.parent = usePaperStore.getState().group! - notifications.show({ - id: "sam2", - message: "模型生成成功", - color: "green", - }) - } - } catch (error) { - console.log(error) - } - } catch (error: any) { - console.log(error) - notifications.show({ - id: "sam2", - message: error.message ?? "模型生成失败", - color: "red", - }) - if (error.message === "点位超出图片范围") { - usePaperStore - .getState() - .supportTool?.emit("keydown", { key: "escape" }) - } - } - }, - [activeImage, projectDetail] - ) - - false && renderSupportAnnotation - - const renderSupportAnnotation2 = useCallback( async ( points: Array<[number, number]>, tags: number[], @@ -1145,7 +983,7 @@ const PaperContainer = ( // const picSize = usePaperStore.getState().rasterSize[activeImage] try { - const data = await getAuxiliaryAnnotation2({ + const data = await getAuxiliaryAnnotation({ project_id: projectId, image_name: activeImage, // project_id: "image_test_bk", @@ -1233,180 +1071,174 @@ const PaperContainer = ( [activeImage, projectDetail] ) - const renderTrackingAnnotation = useCallback(async () => { - updateLoadingFlag(true) - try { - notifications.show({ - id: "sam2", - message: "模型生成中", - loading: true, - autoClose: false, - }) - const ids = - useObjectStore.getState().selectedPath[ - useBottomToolsStore.getState().activeImage - ] - const picArr = useBottomToolsStore.getState().selectedItems - const [width, height] = usePaperStore.getState().rasterSize[ - activeImage - ] ?? [0, 0] - const currentScale = - usePaperStore.getState().rasterScale[activeImage] ?? 1 - if (ids.length === 1) { - const id = ids[0] - let categoryId: any = null - let data: any = null - const activeImageData = useLabelStore.getState().label.get(activeImage)! - for (let [category, objArr] of activeImageData.entries()) { - objArr.forEach((item) => { - if (item[0] === id) { - categoryId = category - data = item - } - }) - } - if (categoryId && data) { - const contours: [number, number][] = [] - const tags: boolean[] = [] - data[1].forEach((item: any) => { - contours.push( - item.map(([x, y]: any) => [ - Math.round(x / currentScale), - Math.round(y / currentScale), - ]) - ) - tags.push(false) - }) - data[3].forEach((item: any) => { - contours.push( - item.map(([x, y]: any) => [ - Math.round(x / currentScale), - Math.round(y / currentScale), - ]) - ) - tags.push(true) - }) - const file_names = [...new Set([activeImage, ...picArr])] - const params = { - project_id: projectDetail?.id || 0, - file_names, - image_hw: [height, width], - contours, - hollows: tags, - } - console.log("传参", params) - try { - const res = await getTrackingAuxiliaryAnnotation( - msgpack.encode(params) - ) - const resData = msgpack.decode(new Uint8Array(res)) - if (resData.msg === "success") { - const { - future_contours, - }: { - future_contours: { - contours: [number, number][][] - hollows: boolean[] - }[] - } = resData - const nowTaskData = safeClone(useLabelStore.getState().label) - future_contours.forEach(({ contours, hollows }, index) => { - const fileName = file_names[index + 1] - const imgScale = - usePaperStore.getState().rasterScale[fileName] ?? 1 - let newId = - useRightToolsStore - .getState() - .pathIds.reduce((a, b) => Math.max(a, b), 0) + 1 - useRightToolsStore.getState().pushPathId(newId) - // 处理点位数据 - let segmentation: Array<[number, number][]> = [] - let hollow_segmentation: Array<[number, number][]> = [] - contours.forEach((contour, index) => { - if (!hollows[index]) - segmentation.push( - contour.map(([x, y]: any) => [x * imgScale, y * imgScale]) - ) - else - hollow_segmentation.push( - contour.map(([x, y]: any) => [x * imgScale, y * imgScale]) - ) - }) - // 详情数据 - let detail = { - ...initialDetail, - uid: usePermissionStore.getState().user_id, - comment: [...useLabelStore.getState().labelDefaultComments], - create_timestamp: dayjs().unix(), - sub_attributes: {}, - image_id: activeImage, - category_id: categoryId, - } - const updateData: any = [ - newId, - segmentation, - detail, - hollow_segmentation, - ] - if (nowTaskData.has(fileName)) { - const categoryMap = nowTaskData.get(fileName)! - if (categoryMap.has(categoryId)) { - const existArr = categoryMap.get(categoryId) - existArr?.push(updateData) - } else { - categoryMap.set(categoryId, [updateData]) - } - } else { - let m = new Map() - m.set(categoryId, [updateData]) - nowTaskData.set(fileName, m) - } - }) - console.log(nowTaskData) - useLabelStore.getState().setLabel(nowTaskData) - useLabelStore.getState().pushStateStack(nowTaskData) - notifications.show({ - id: "sam2", - message: "模型生成成功", - color: "green", - }) - } - } catch (error) { - console.log(error) - } - } - } else if (!ids.length) { - notifications.show({ - id: "sam2", - message: "请先选中标注对象后再进行模型调用", - color: "red", - }) - } else { - notifications.show({ - id: "sam2", - message: "请勿选择多个标注对象进行模型调用", - color: "red", - }) - } - } catch (error) { - console.log(error) - notifications.show({ - id: "sam2", - message: "模型生成失败", - color: "red", - }) - } finally { - updateLoadingFlag(false) - } - }, [activeImage, projectDetail?.id, updateLoadingFlag]) + // const renderTrackingAnnotation = useCallback(async () => { + // updateLoadingFlag(true) + // try { + // notifications.show({ + // id: "sam2", + // message: "模型生成中", + // loading: true, + // autoClose: false, + // }) + // const ids = + // useObjectStore.getState().selectedPath[ + // useBottomToolsStore.getState().activeImage + // ] + // const picArr = useBottomToolsStore.getState().selectedItems + // const [width, height] = usePaperStore.getState().rasterSize[ + // activeImage + // ] ?? [0, 0] + // const currentScale = + // usePaperStore.getState().rasterScale[activeImage] ?? 1 + // if (ids.length === 1) { + // const id = ids[0] + // let categoryId: any = null + // let data: any = null + // const activeImageData = useLabelStore.getState().label.get(activeImage)! + // for (let [category, objArr] of activeImageData.entries()) { + // objArr.forEach((item) => { + // if (item[0] === id) { + // categoryId = category + // data = item + // } + // }) + // } + // if (categoryId && data) { + // const contours: [number, number][] = [] + // const tags: boolean[] = [] + // data[1].forEach((item: any) => { + // contours.push( + // item.map(([x, y]: any) => [ + // Math.round(x / currentScale), + // Math.round(y / currentScale), + // ]) + // ) + // tags.push(false) + // }) + // data[3].forEach((item: any) => { + // contours.push( + // item.map(([x, y]: any) => [ + // Math.round(x / currentScale), + // Math.round(y / currentScale), + // ]) + // ) + // tags.push(true) + // }) + // const file_names = [...new Set([activeImage, ...picArr])] + // const params = { + // project_id: projectDetail?.id || 0, + // file_names, + // image_hw: [height, width], + // contours, + // hollows: tags, + // } + // console.log("传参", params) + // try { + // const res = await getTrackingAuxiliaryAnnotation( + // msgpack.encode(params) + // ) + // const resData = msgpack.decode(new Uint8Array(res)) + // if (resData.msg === "success") { + // const { + // future_contours, + // }: { + // future_contours: { + // contours: [number, number][][] + // hollows: boolean[] + // }[] + // } = resData + // const nowTaskData = safeClone(useLabelStore.getState().label) + // future_contours.forEach(({ contours, hollows }, index) => { + // const fileName = file_names[index + 1] + // const imgScale = + // usePaperStore.getState().rasterScale[fileName] ?? 1 + // let newId = + // useRightToolsStore + // .getState() + // .pathIds.reduce((a, b) => Math.max(a, b), 0) + 1 + // useRightToolsStore.getState().pushPathId(newId) + // // 处理点位数据 + // let segmentation: Array<[number, number][]> = [] + // let hollow_segmentation: Array<[number, number][]> = [] + // contours.forEach((contour, index) => { + // if (!hollows[index]) + // segmentation.push( + // contour.map(([x, y]: any) => [x * imgScale, y * imgScale]) + // ) + // else + // hollow_segmentation.push( + // contour.map(([x, y]: any) => [x * imgScale, y * imgScale]) + // ) + // }) + // // 详情数据 + // let detail = { + // ...initialDetail, + // uid: usePermissionStore.getState().user_id, + // comment: [...useLabelStore.getState().labelDefaultComments], + // create_timestamp: dayjs().unix(), + // sub_attributes: {}, + // image_id: activeImage, + // category_id: categoryId, + // } + // const updateData: any = [ + // newId, + // segmentation, + // detail, + // hollow_segmentation, + // ] + // if (nowTaskData.has(fileName)) { + // const categoryMap = nowTaskData.get(fileName)! + // if (categoryMap.has(categoryId)) { + // const existArr = categoryMap.get(categoryId) + // existArr?.push(updateData) + // } else { + // categoryMap.set(categoryId, [updateData]) + // } + // } else { + // let m = new Map() + // m.set(categoryId, [updateData]) + // nowTaskData.set(fileName, m) + // } + // }) + // console.log(nowTaskData) + // useLabelStore.getState().setLabel(nowTaskData) + // useLabelStore.getState().pushStateStack(nowTaskData) + // notifications.show({ + // id: "sam2", + // message: "模型生成成功", + // color: "green", + // }) + // } + // } catch (error) { + // console.log(error) + // } + // } + // } else if (!ids.length) { + // notifications.show({ + // id: "sam2", + // message: "请先选中标注对象后再进行模型调用", + // color: "red", + // }) + // } else { + // notifications.show({ + // id: "sam2", + // message: "请勿选择多个标注对象进行模型调用", + // color: "red", + // }) + // } + // } catch (error) { + // console.log(error) + // notifications.show({ + // id: "sam2", + // message: "模型生成失败", + // color: "red", + // }) + // } finally { + // updateLoadingFlag(false) + // } + // }, [activeImage, projectDetail?.id, updateLoadingFlag]) - // const getBase64 = (file: any): Promise => - // new Promise((resolve, reject) => { - // const reader = new FileReader(); - // reader.readAsDataURL(file); - // reader.onload = () => resolve(reader.result as string); - // reader.onerror = (error) => reject(error); - // }); + const renderTrackingAnnotation = useCallback(async () => {}, []) useEffect(() => { if (setup) { @@ -1439,7 +1271,7 @@ const PaperContainer = ( let pointTool = new paper.Tool() initPointTool(pointTool, renderCircle, renderPath, forceUpdate) let supportTool = new paper.Tool() - initSupportTool(supportTool, renderSupportAnnotation2) + initSupportTool(supportTool, renderSupportAnnotation) } return () => { newGroup.remove() @@ -1455,7 +1287,7 @@ const PaperContainer = ( initRectangleTool, initSupportTool, initViewTool, - renderSupportAnnotation2, + renderSupportAnnotation, setup, ]) @@ -4148,7 +3980,7 @@ const PaperContainer = ( renderPolygons, handleCrosshairMove, renderTrackingAnnotation, - renderSupportAnnotation2, + renderSupportAnnotation, saveSupportAnnotationData, copyAnnotationDataToMultiFrame, }))