paperclip-bot/labelimage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(sam2): code

Browse Source
This commit is contained in:
zhangheng
2026-04-10 13:51:52 +08:00
parent 140df9202d
commit 033f11fb9b
4 changed files with 176 additions and 463 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
components/label/LabelNossr.tsx
View File

@@ -2267,6 +2267,7 @@ const LabelPage = ({
// 追踪
if (e.key === "i") {
e.preventDefault()
// sam2 暂未有图片接口
paperContainerRef.current.renderTrackingAnnotation()
}

26
components/label/api/label/index.ts
View File

@@ -108,18 +108,6 @@ export const getLabelImage = (activeImage: string, path: string) => {
})
}
// 辅助标注
export const getAuxiliaryAnnotation = (data: any) => {
return httpFetch<any>({
url: BASE_LABEL_API + "/api/model_server/sam2/sa",
method: "POST",
body: JSON.stringify(data),
headerAttr: {
responseType: "arraybuffer",
},
})
}
const BASE_SAM_API =
process.env.NEXT_PUBLIC_ENV === "production"
? "http://172.16.103.224:8000"
@@ -127,7 +115,7 @@ const BASE_SAM_API =
? "http://172.30.21.211:8000"
: "http://172.30.21.211:8000"
export const getAuxiliaryAnnotation2 = (data: {
export const getAuxiliaryAnnotation = (data: {
project_id: number
image_name: string
prompt: {
@@ -146,18 +134,6 @@ export const getAuxiliaryAnnotation2 = (data: {
})
}
// 模型标注 追踪
export const getTrackingAuxiliaryAnnotation = (data: any) => {
return httpFetch<any>({
url: BASE_LABEL_API + "/api/model_server/sam2/vos",
method: "POST",
body: JSON.stringify(data),
headerAttr: {
responseType: "arraybuffer",
},
})
}
// 用户登录
export const userLogin = (data: { name: string; password: string }) => {
return httpFetch<LoginInfo>({

96
components/label/components/PaperContainer.renderTrackingAnnotation.md
View File

@@ -1,96 +0,0 @@
# `renderTrackingAnnotation` 流程说明
来源文件：
- `labelimage/components/label/components/PaperContainer.tsx`
- `labelimage/components/label/LabelNossr.tsx`
- `labelimage/components/label/api/label/index.ts`
## 概述
`renderTrackingAnnotation` 是单对象跨帧追踪的入口方法。
它会把当前激活图片上选中的标注对象作为 seed，将其轮廓数据发送给追踪模型接口，再把模型返回的后续帧轮廓结果写回 `label store`。
## Mermaid
```mermaid
flowchart TD
A["在 `LabelNossr.tsx` 中按下 `i`"] --> B["调用 `paperContainerRef.current.renderTrackingAnnotation()`"]
B --> C["执行 `updateLoadingFlag(true)`<br/>并显示加载通知"]
C --> D["从 store 读取运行时状态：<br/>`selectedPath[activeImage]`<br/>`selectedItems`<br/>`rasterSize[activeImage]`<br/>`rasterScale[activeImage]`"]
D --> E{"是否恰好选中了一个标注对象？"}
E -->|未选中| F["提示错误：<br/>`请先选中标注对象后再进行模型调用`"]
E -->|选中了多个| G["提示错误：<br/>`请勿选择多个标注对象进行模型调用`"]
E -->|是| H["在 `label.get(activeImage)` 中查找<br/>满足 `item[0] === id` 的标注对象"]
H --> I{"是否找到 `categoryId` 和 `data`？"}
I -->|否| Z["跳过请求参数构造"]
I -->|是| J["提取源标注几何数据：<br/>`data[1]` => 外轮廓 `segmentation`<br/>`data[3]` => 镂空轮廓 `hollow_segmentation`"]
J --> K["将画布坐标转换为原图坐标：<br/>`Math.round(x / currentScale)`<br/>`Math.round(y / currentScale)`"]
K --> L["构造请求参数：<br/>`project_id`<br/>`file_names = [activeImage, ...selectedItems]`<br/>`image_hw = [height, width]`<br/>`contours`<br/>`hollows`"]
L --> M["使用 `msgpack.encode(params)` 序列化"]
M --> N["通过 `getTrackingAuxiliaryAnnotation()`<br/>POST 到 `/api/model_server/sam2/vos`"]
N --> O["以 `arraybuffer` 形式接收响应体"]
O --> P["使用<br/>`msgpack.decode(new Uint8Array(res))`<br/>解码响应数据"]
P --> Q{"resData.msg === 'success'？"}
Q -->|否| R["内层流程仅执行 `console.log(error/resData)`"]
Q -->|是| S["读取响应中的 `future_contours`"]
S --> T["克隆当前 `label store`：<br/>`safeClone(useLabelStore.getState().label)`"]
T --> U["遍历每个未来帧结果：<br/>目标文件 = `file_names[index + 1]`"]
U --> V["读取目标帧缩放比例：<br/>`rasterScale[fileName]`"]
V --> W["基于 `pathIds` 生成 `newId` 并注册"]
W --> X["将原图坐标还原为画布坐标：<br/>`x * imgScale`<br/>`y * imgScale`"]
X --> Y["将返回轮廓拆分为：<br/>`segmentation`<br/>`hollow_segmentation`"]
Y --> AA["构造标注元组：<br/>`[newId, segmentation, detail, hollow_segmentation]`"]
AA --> AB["将结果合并到<br/>`Map<fileName, Map<categoryId, annotation[]>>`"]
AB --> AC["持久化结果：<br/>`setLabel(nowTaskData)`<br/>`pushStateStack(nowTaskData)`"]
AC --> AD["显示成功通知"]
F --> AE["finally：执行 `updateLoadingFlag(false)`"]
G --> AE
Z --> AE
R --> AE
AD --> AE
C --> AF["外层 `try/catch`"]
AF -->|异常| AG["显示通用失败通知：<br/>`模型生成失败`"]
AG --> AE
```
## 数据结构
被追踪的源标注对象，使用和普通多边形标注一致的元组结构：
```ts
;[id, segmentation, detail, hollow_segmentation]
```
各字段含义：
- `item[0]`：标注 id
- `item[1]`：外轮廓多边形
- `item[2]`：详情元数据
- `item[3]`：镂空轮廓多边形
## 为什么使用 `arraybuffer`
这条追踪请求并不是从头到尾都按普通 JSON 方式传输。
- 请求参数先通过 `msgpack.encode(params)` 打包。
- API 封装层把这份打包后的数据发给后端。
- 响应体以 `arraybuffer` 的形式返回。
- `renderTrackingAnnotation` 再通过 `msgpack.decode(new Uint8Array(res))` 还原出 JavaScript 对象。
因此这里必须配置 `responseType: "arraybuffer"`，因为前端预期拿到的是二进制响应体，再自行解码。
## 备注
- 当前帧是 seed 帧，模型返回的结果会写入 `file_names[index + 1]` 对应的后续帧。
- 这个方法更新的是 `label store`，不会直接在当前页面把其他帧的结果立刻画出来。
- 这个方法只支持一次追踪一个被选中的对象。
- 内层请求失败时目前主要是 `console.log`，不一定总会显示专门的失败提示。
- 这个方法里生成的 `detail.image_id` 使用的是 `activeImage`，建议再结合整体数据模型确认一下是否符合预期。

516
components/label/components/PaperContainer.tsx
View File

@@ -18,7 +18,7 @@ import {
import { useForm } from "@mantine/form"
import { notifications } from "@mantine/notifications"
import dayjs from "dayjs"
import msgpack from "msgpack-lite"
// import msgpack from "msgpack-lite"
import paper from "paper"
import React, {
DispatchWithoutAction,
@@ -31,12 +31,7 @@ import React, {
useRef,
useState,
} from "react"
import {
getAuxiliaryAnnotation,
getAuxiliaryAnnotation2,
getServerImage,
getTrackingAuxiliaryAnnotation,
} from "../api/label"
import { getAuxiliaryAnnotation, getServerImage } from "../api/label"
import { Project } from "../api/project/typing"
import { labelimagePerformanceConfig } from "../config/performance"
import { LabelState } from "../LabelNossr"
@@ -185,6 +180,7 @@ const PaperContainer = (
ref: React.Ref<unknown> | undefined
) => {
const { forceUpdate, projectDetail, labelState, updateLoadingFlag } = props
false && updateLoadingFlag
const containerRef = useRef<any>(null)
const contextMenuRef = useRef<HTMLDivElement>(null)
const [imgSize, setImageSize] = useState<{
@@ -884,164 +880,6 @@ const PaperContainer = (
}, [clearPendingFrameSwitch, loadingData])
const renderSupportAnnotation = useCallback(
async (
points: Array<[number, number]>,
tags: number[],
clear_previous_state = false
) => {
try {
notifications.show({
id: "sam2",
message: "模型生成中",
loading: true,
autoClose: false,
})
const operationSchema =
projectDetail?.label_schema_list?.find(
(item) =>
item.category_id === +useTopToolsStore.getState().activeOperation
) || null
const strokeColor = operationSchema
? `rgb(${operationSchema.color[0]},${operationSchema.color[1]},${operationSchema.color[2]})`
: "rgb(0,0,0)"
const fillColor = operationSchema
? `rgba(${operationSchema.color[0]},${operationSchema.color[1]},${operationSchema.color[2]},0.3)`
: "rgba(0,0,0,0.3)"
const blankColor = operationSchema
? `rgba(${operationSchema.color[0]},${operationSchema.color[1]},${operationSchema.color[2]},0.01)`
: "rgba(0,0,0,0.01)"
const currentScale =
usePaperStore.getState().rasterScale[activeImage] ?? 1
const draw = () => {
points.forEach((point, index) => {
const drawPoint = new paper.Path.Circle({
center: point,
radius: 2,
fillColor: tags[index] === 1 ? "red" : "blue",
strokeColor: tags[index] === 1 ? "red" : "blue",
data: {
id: "support",
type: "point",
},
})
drawPoint.parent = usePaperStore.getState().group!
const raster = usePaperStore
.getState()
.group!.getItems({ data: { name: "pic" } })[0]
if (!raster.contains(drawPoint.position)) {
throw new Error("点位超出图片范围")
}
})
}
console.log(points, tags)
// 绘制点位
clearSupportAnnotationItem()
draw()
const picSize = usePaperStore.getState().rasterSize[activeImage]
const params = {
project_id: projectDetail?.id || 0,
file_name: activeImage,
image_hw:
picSize && picSize.length ? [picSize[1], picSize[0]] : [0, 0],
points_and_box: points.map(([x, y]) => [
Math.round(x / currentScale),
Math.round(y / currentScale),
]),
tags,
clear_previous_state,
}
try {
const res = await getAuxiliaryAnnotation(
msgpack.encode(params) as any
)
const data = msgpack.decode(new Uint8Array(res))
console.log(data)
if (data.msg === "success") {
const { contours, hollows } = data
const outerPaths: any[] = []
const innerPaths: any[] = []
contours.forEach((contour: any, index: number) => {
if (contour.length > 3) {
const path = new paper.Path({
segments: contour.map(([x, y]: [number, number]) => [
x * currentScale,
y * currentScale,
]),
fillColor: blankColor,
strokeColor: strokeColor,
closed: true,
data: {
id: "support",
type: "polygon",
isHollowPolygon: false,
fillColor,
strokeColor,
blankColor,
},
parent: usePaperStore.getState().group,
strokeScaling: false,
})
if (
Math.abs(path.area) < useTopToolsStore.getState().checkSize
) {
path.remove()
} else {
// path.scaling = usePaperStore.getState().group!.scaling;
if (hollows[index]) {
// 更新标志位
path.data.isHollowPolygon = true
innerPaths.push(path)
} else {
// 上色
path.fillColor = new paper.Color(fillColor)
outerPaths.push(path)
}
}
}
})
const compoundPath = new paper.CompoundPath({
children: [...outerPaths, ...innerPaths],
data: {
id: "support",
type: "parent",
},
strokeColor: strokeColor,
strokeWidth: 2,
fillColor: fillColor,
})
compoundPath.parent = usePaperStore.getState().group!
notifications.show({
id: "sam2",
message: "模型生成成功",
color: "green",
})
}
} catch (error) {
console.log(error)
}
} catch (error: any) {
console.log(error)
notifications.show({
id: "sam2",
message: error.message ?? "模型生成失败",
color: "red",
})
if (error.message === "点位超出图片范围") {
usePaperStore
.getState()
.supportTool?.emit("keydown", { key: "escape" })
}
}
},
[activeImage, projectDetail]
)
false && renderSupportAnnotation
const renderSupportAnnotation2 = useCallback(
async (
points: Array<[number, number]>,
tags: number[],
@@ -1145,7 +983,7 @@ const PaperContainer = (
// const picSize = usePaperStore.getState().rasterSize[activeImage]
try {
const data = await getAuxiliaryAnnotation2({
const data = await getAuxiliaryAnnotation({
project_id: projectId,
image_name: activeImage,
// project_id: "image_test_bk",
@@ -1233,180 +1071,174 @@ const PaperContainer = (
[activeImage, projectDetail]
)
const renderTrackingAnnotation = useCallback(async () => {
updateLoadingFlag(true)
try {
notifications.show({
id: "sam2",
message: "模型生成中",
loading: true,
autoClose: false,
})
const ids =
useObjectStore.getState().selectedPath[
useBottomToolsStore.getState().activeImage
]
const picArr = useBottomToolsStore.getState().selectedItems
const [width, height] = usePaperStore.getState().rasterSize[
activeImage
] ?? [0, 0]
const currentScale =
usePaperStore.getState().rasterScale[activeImage] ?? 1
if (ids.length === 1) {
const id = ids[0]
let categoryId: any = null
let data: any = null
const activeImageData = useLabelStore.getState().label.get(activeImage)!
for (let [category, objArr] of activeImageData.entries()) {
objArr.forEach((item) => {
if (item[0] === id) {
categoryId = category
data = item
}
})
}
if (categoryId && data) {
const contours: [number, number][] = []
const tags: boolean[] = []
data[1].forEach((item: any) => {
contours.push(
item.map(([x, y]: any) => [
Math.round(x / currentScale),
Math.round(y / currentScale),
])
)
tags.push(false)
})
data[3].forEach((item: any) => {
contours.push(
item.map(([x, y]: any) => [
Math.round(x / currentScale),
Math.round(y / currentScale),
])
)
tags.push(true)
})
const file_names = [...new Set([activeImage, ...picArr])]
const params = {
project_id: projectDetail?.id || 0,
file_names,
image_hw: [height, width],
contours,
hollows: tags,
}
console.log("传参", params)
try {
const res = await getTrackingAuxiliaryAnnotation(
msgpack.encode(params)
)
const resData = msgpack.decode(new Uint8Array(res))
if (resData.msg === "success") {
const {
future_contours,
}: {
future_contours: {
contours: [number, number][][]
hollows: boolean[]
}[]
} = resData
const nowTaskData = safeClone(useLabelStore.getState().label)
future_contours.forEach(({ contours, hollows }, index) => {
const fileName = file_names[index + 1]
const imgScale =
usePaperStore.getState().rasterScale[fileName] ?? 1
let newId =
useRightToolsStore
.getState()
.pathIds.reduce((a, b) => Math.max(a, b), 0) + 1
useRightToolsStore.getState().pushPathId(newId)
// 处理点位数据
let segmentation: Array<[number, number][]> = []
let hollow_segmentation: Array<[number, number][]> = []
contours.forEach((contour, index) => {
if (!hollows[index])
segmentation.push(
contour.map(([x, y]: any) => [x * imgScale, y * imgScale])
)
else
hollow_segmentation.push(
contour.map(([x, y]: any) => [x * imgScale, y * imgScale])
)
})
// 详情数据
let detail = {
...initialDetail,
uid: usePermissionStore.getState().user_id,
comment: [...useLabelStore.getState().labelDefaultComments],
create_timestamp: dayjs().unix(),
sub_attributes: {},
image_id: activeImage,
category_id: categoryId,
}
const updateData: any = [
newId,
segmentation,
detail,
hollow_segmentation,
]
if (nowTaskData.has(fileName)) {
const categoryMap = nowTaskData.get(fileName)!
if (categoryMap.has(categoryId)) {
const existArr = categoryMap.get(categoryId)
existArr?.push(updateData)
} else {
categoryMap.set(categoryId, [updateData])
}
} else {
let m = new Map()
m.set(categoryId, [updateData])
nowTaskData.set(fileName, m)
}
})
console.log(nowTaskData)
useLabelStore.getState().setLabel(nowTaskData)
useLabelStore.getState().pushStateStack(nowTaskData)
notifications.show({
id: "sam2",
message: "模型生成成功",
color: "green",
})
}
} catch (error) {
console.log(error)
}
}
} else if (!ids.length) {
notifications.show({
id: "sam2",
message: "请先选中标注对象后再进行模型调用",
color: "red",
})
} else {
notifications.show({
id: "sam2",
message: "请勿选择多个标注对象进行模型调用",
color: "red",
})
}
} catch (error) {
console.log(error)
notifications.show({
id: "sam2",
message: "模型生成失败",
color: "red",
})
} finally {
updateLoadingFlag(false)
}
}, [activeImage, projectDetail?.id, updateLoadingFlag])
// const renderTrackingAnnotation = useCallback(async () => {
// updateLoadingFlag(true)
// try {
// notifications.show({
// id: "sam2",
// message: "模型生成中",
// loading: true,
// autoClose: false,
// })
// const ids =
// useObjectStore.getState().selectedPath[
// useBottomToolsStore.getState().activeImage
// ]
// const picArr = useBottomToolsStore.getState().selectedItems
// const [width, height] = usePaperStore.getState().rasterSize[
// activeImage
// ] ?? [0, 0]
// const currentScale =
// usePaperStore.getState().rasterScale[activeImage] ?? 1
// if (ids.length === 1) {
// const id = ids[0]
// let categoryId: any = null
// let data: any = null
// const activeImageData = useLabelStore.getState().label.get(activeImage)!
// for (let [category, objArr] of activeImageData.entries()) {
// objArr.forEach((item) => {
// if (item[0] === id) {
// categoryId = category
// data = item
// }
// })
// }
// if (categoryId && data) {
// const contours: [number, number][] = []
// const tags: boolean[] = []
// data[1].forEach((item: any) => {
// contours.push(
// item.map(([x, y]: any) => [
// Math.round(x / currentScale),
// Math.round(y / currentScale),
// ])
// )
// tags.push(false)
// })
// data[3].forEach((item: any) => {
// contours.push(
// item.map(([x, y]: any) => [
// Math.round(x / currentScale),
// Math.round(y / currentScale),
// ])
// )
// tags.push(true)
// })
// const file_names = [...new Set([activeImage, ...picArr])]
// const params = {
// project_id: projectDetail?.id || 0,
// file_names,
// image_hw: [height, width],
// contours,
// hollows: tags,
// }
// console.log("传参", params)
// try {
// const res = await getTrackingAuxiliaryAnnotation(
// msgpack.encode(params)
// )
// const resData = msgpack.decode(new Uint8Array(res))
// if (resData.msg === "success") {
// const {
// future_contours,
// }: {
// future_contours: {
// contours: [number, number][][]
// hollows: boolean[]
// }[]
// } = resData
// const nowTaskData = safeClone(useLabelStore.getState().label)
// future_contours.forEach(({ contours, hollows }, index) => {
// const fileName = file_names[index + 1]
// const imgScale =
// usePaperStore.getState().rasterScale[fileName] ?? 1
// let newId =
// useRightToolsStore
// .getState()
// .pathIds.reduce((a, b) => Math.max(a, b), 0) + 1
// useRightToolsStore.getState().pushPathId(newId)
// // 处理点位数据
// let segmentation: Array<[number, number][]> = []
// let hollow_segmentation: Array<[number, number][]> = []
// contours.forEach((contour, index) => {
// if (!hollows[index])
// segmentation.push(
// contour.map(([x, y]: any) => [x * imgScale, y * imgScale])
// )
// else
// hollow_segmentation.push(
// contour.map(([x, y]: any) => [x * imgScale, y * imgScale])
// )
// })
// // 详情数据
// let detail = {
// ...initialDetail,
// uid: usePermissionStore.getState().user_id,
// comment: [...useLabelStore.getState().labelDefaultComments],
// create_timestamp: dayjs().unix(),
// sub_attributes: {},
// image_id: activeImage,
// category_id: categoryId,
// }
// const updateData: any = [
// newId,
// segmentation,
// detail,
// hollow_segmentation,
// ]
// if (nowTaskData.has(fileName)) {
// const categoryMap = nowTaskData.get(fileName)!
// if (categoryMap.has(categoryId)) {
// const existArr = categoryMap.get(categoryId)
// existArr?.push(updateData)
// } else {
// categoryMap.set(categoryId, [updateData])
// }
// } else {
// let m = new Map()
// m.set(categoryId, [updateData])
// nowTaskData.set(fileName, m)
// }
// })
// console.log(nowTaskData)
// useLabelStore.getState().setLabel(nowTaskData)
// useLabelStore.getState().pushStateStack(nowTaskData)
// notifications.show({
// id: "sam2",
// message: "模型生成成功",
// color: "green",
// })
// }
// } catch (error) {
// console.log(error)
// }
// }
// } else if (!ids.length) {
// notifications.show({
// id: "sam2",
// message: "请先选中标注对象后再进行模型调用",
// color: "red",
// })
// } else {
// notifications.show({
// id: "sam2",
// message: "请勿选择多个标注对象进行模型调用",
// color: "red",
// })
// }
// } catch (error) {
// console.log(error)
// notifications.show({
// id: "sam2",
// message: "模型生成失败",
// color: "red",
// })
// } finally {
// updateLoadingFlag(false)
// }
// }, [activeImage, projectDetail?.id, updateLoadingFlag])
// const getBase64 = (file: any): Promise<string> =>
// new Promise((resolve, reject) => {
// const reader = new FileReader();
// reader.readAsDataURL(file);
// reader.onload = () => resolve(reader.result as string);
// reader.onerror = (error) => reject(error);
// });
const renderTrackingAnnotation = useCallback(async () => {}, [])
useEffect(() => {
if (setup) {
@@ -1439,7 +1271,7 @@ const PaperContainer = (
let pointTool = new paper.Tool()
initPointTool(pointTool, renderCircle, renderPath, forceUpdate)
let supportTool = new paper.Tool()
initSupportTool(supportTool, renderSupportAnnotation2)
initSupportTool(supportTool, renderSupportAnnotation)
}
return () => {
newGroup.remove()
@@ -1455,7 +1287,7 @@ const PaperContainer = (
initRectangleTool,
initSupportTool,
initViewTool,
renderSupportAnnotation2,
renderSupportAnnotation,
setup,
])
@@ -4148,7 +3980,7 @@ const PaperContainer = (
renderPolygons,
handleCrosshairMove,
renderTrackingAnnotation,
renderSupportAnnotation2,
renderSupportAnnotation,
saveSupportAnnotationData,
copyAnnotationDataToMultiFrame,
}))