paperclip-bot/labelimage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(sam2): code

Browse Source
This commit is contained in:
zhangheng
2026-04-10 13:51:52 +08:00
parent 140df9202d
commit 033f11fb9b
4 changed files with 176 additions and 463 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
components/label/LabelNossr.tsx
View File

@@ -2267,6 +2267,7 @@ const LabelPage = ({
// 追踪 // 追踪
if (e.key === "i") { if (e.key === "i") {
e.preventDefault() e.preventDefault()
// sam2 暂未有图片接口
paperContainerRef.current.renderTrackingAnnotation() paperContainerRef.current.renderTrackingAnnotation()
} }

26
components/label/api/label/index.ts
View File

@@ -108,18 +108,6 @@ export const getLabelImage = (activeImage: string, path: string) => {
}) })
} }
// 辅助标注
export const getAuxiliaryAnnotation = (data: any) => {
return httpFetch<any>({
url: BASE_LABEL_API + "/api/model_server/sam2/sa",
method: "POST",
body: JSON.stringify(data),
headerAttr: {
responseType: "arraybuffer",
},
})
}
const BASE_SAM_API = const BASE_SAM_API =
process.env.NEXT_PUBLIC_ENV === "production" process.env.NEXT_PUBLIC_ENV === "production"
? "http://172.16.103.224:8000" ? "http://172.16.103.224:8000"
@@ -127,7 +115,7 @@ const BASE_SAM_API =
? "http://172.30.21.211:8000" ? "http://172.30.21.211:8000"
: "http://172.30.21.211:8000" : "http://172.30.21.211:8000"
export const getAuxiliaryAnnotation2 = (data: { export const getAuxiliaryAnnotation = (data: {
project_id: number project_id: number
image_name: string image_name: string
prompt: { prompt: {
@@ -146,18 +134,6 @@ export const getAuxiliaryAnnotation2 = (data: {
}) })
} }
// 模型标注 追踪
export const getTrackingAuxiliaryAnnotation = (data: any) => {
return httpFetch<any>({
url: BASE_LABEL_API + "/api/model_server/sam2/vos",
method: "POST",
body: JSON.stringify(data),
headerAttr: {
responseType: "arraybuffer",
},
})
}
// 用户登录 // 用户登录
export const userLogin = (data: { name: string; password: string }) => { export const userLogin = (data: { name: string; password: string }) => {
return httpFetch<LoginInfo>({ return httpFetch<LoginInfo>({

96
components/label/components/PaperContainer.renderTrackingAnnotation.md
View File

@@ -1,96 +0,0 @@
# `renderTrackingAnnotation` 流程说明
来源文件：
- `labelimage/components/label/components/PaperContainer.tsx`
- `labelimage/components/label/LabelNossr.tsx`
- `labelimage/components/label/api/label/index.ts`
## 概述
`renderTrackingAnnotation` 是单对象跨帧追踪的入口方法。
它会把当前激活图片上选中的标注对象作为 seed，将其轮廓数据发送给追踪模型接口，再把模型返回的后续帧轮廓结果写回 `label store`。
## Mermaid
```mermaid
flowchart TD
A["在 `LabelNossr.tsx` 中按下 `i`"] --> B["调用 `paperContainerRef.current.renderTrackingAnnotation()`"]
B --> C["执行 `updateLoadingFlag(true)`<br/>并显示加载通知"]
C --> D["从 store 读取运行时状态：<br/>`selectedPath[activeImage]`<br/>`selectedItems`<br/>`rasterSize[activeImage]`<br/>`rasterScale[activeImage]`"]
D --> E{"是否恰好选中了一个标注对象？"}
E -->|未选中| F["提示错误：<br/>`请先选中标注对象后再进行模型调用`"]
E -->|选中了多个| G["提示错误：<br/>`请勿选择多个标注对象进行模型调用`"]
E -->|是| H["在 `label.get(activeImage)` 中查找<br/>满足 `item[0] === id` 的标注对象"]
H --> I{"是否找到 `categoryId` 和 `data`？"}
I -->|否| Z["跳过请求参数构造"]
I -->|是| J["提取源标注几何数据：<br/>`data[1]` => 外轮廓 `segmentation`<br/>`data[3]` => 镂空轮廓 `hollow_segmentation`"]
J --> K["将画布坐标转换为原图坐标：<br/>`Math.round(x / currentScale)`<br/>`Math.round(y / currentScale)`"]
K --> L["构造请求参数：<br/>`project_id`<br/>`file_names = [activeImage, ...selectedItems]`<br/>`image_hw = [height, width]`<br/>`contours`<br/>`hollows`"]
L --> M["使用 `msgpack.encode(params)` 序列化"]
M --> N["通过 `getTrackingAuxiliaryAnnotation()`<br/>POST 到 `/api/model_server/sam2/vos`"]
N --> O["以 `arraybuffer` 形式接收响应体"]
O --> P["使用<br/>`msgpack.decode(new Uint8Array(res))`<br/>解码响应数据"]
P --> Q{"resData.msg === 'success'？"}
Q -->|否| R["内层流程仅执行 `console.log(error/resData)`"]
Q -->|是| S["读取响应中的 `future_contours`"]
S --> T["克隆当前 `label store`：<br/>`safeClone(useLabelStore.getState().label)`"]
T --> U["遍历每个未来帧结果：<br/>目标文件 = `file_names[index + 1]`"]
U --> V["读取目标帧缩放比例：<br/>`rasterScale[fileName]`"]
V --> W["基于 `pathIds` 生成 `newId` 并注册"]
W --> X["将原图坐标还原为画布坐标：<br/>`x * imgScale`<br/>`y * imgScale`"]
X --> Y["将返回轮廓拆分为：<br/>`segmentation`<br/>`hollow_segmentation`"]
Y --> AA["构造标注元组：<br/>`[newId, segmentation, detail, hollow_segmentation]`"]
AA --> AB["将结果合并到<br/>`Map<fileName, Map<categoryId, annotation[]>>`"]
AB --> AC["持久化结果：<br/>`setLabel(nowTaskData)`<br/>`pushStateStack(nowTaskData)`"]
AC --> AD["显示成功通知"]
F --> AE["finally：执行 `updateLoadingFlag(false)`"]
G --> AE
Z --> AE
R --> AE
AD --> AE
C --> AF["外层 `try/catch`"]
AF -->|异常| AG["显示通用失败通知：<br/>`模型生成失败`"]
AG --> AE
```
## 数据结构
被追踪的源标注对象，使用和普通多边形标注一致的元组结构：
```ts
;[id, segmentation, detail, hollow_segmentation]
```
各字段含义：
- `item[0]`：标注 id
- `item[1]`：外轮廓多边形
- `item[2]`：详情元数据
- `item[3]`：镂空轮廓多边形
## 为什么使用 `arraybuffer`
这条追踪请求并不是从头到尾都按普通 JSON 方式传输。
- 请求参数先通过 `msgpack.encode(params)` 打包。
- API 封装层把这份打包后的数据发给后端。
- 响应体以 `arraybuffer` 的形式返回。
- `renderTrackingAnnotation` 再通过 `msgpack.decode(new Uint8Array(res))` 还原出 JavaScript 对象。
因此这里必须配置 `responseType: "arraybuffer"`，因为前端预期拿到的是二进制响应体，再自行解码。
## 备注
- 当前帧是 seed 帧，模型返回的结果会写入 `file_names[index + 1]` 对应的后续帧。
- 这个方法更新的是 `label store`，不会直接在当前页面把其他帧的结果立刻画出来。
- 这个方法只支持一次追踪一个被选中的对象。
- 内层请求失败时目前主要是 `console.log`，不一定总会显示专门的失败提示。
- 这个方法里生成的 `detail.image_id` 使用的是 `activeImage`，建议再结合整体数据模型确认一下是否符合预期。

516
components/label/components/PaperContainer.tsx
View File

@@ -18,7 +18,7 @@ import {
import { useForm } from "@mantine/form" import { useForm } from "@mantine/form"
import { notifications } from "@mantine/notifications" import { notifications } from "@mantine/notifications"
import dayjs from "dayjs" import dayjs from "dayjs"
import msgpack from "msgpack-lite" // import msgpack from "msgpack-lite"
import paper from "paper" import paper from "paper"
import React, { import React, {
DispatchWithoutAction, DispatchWithoutAction,
@@ -31,12 +31,7 @@ import React, {
useRef, useRef,
useState, useState,
} from "react" } from "react"
import { import { getAuxiliaryAnnotation, getServerImage } from "../api/label"
getAuxiliaryAnnotation,
getAuxiliaryAnnotation2,
getServerImage,
getTrackingAuxiliaryAnnotation,
} from "../api/label"
import { Project } from "../api/project/typing" import { Project } from "../api/project/typing"
import { labelimagePerformanceConfig } from "../config/performance" import { labelimagePerformanceConfig } from "../config/performance"
import { LabelState } from "../LabelNossr" import { LabelState } from "../LabelNossr"
@@ -185,6 +180,7 @@ const PaperContainer = (
ref: React.Ref<unknown> | undefined ref: React.Ref<unknown> | undefined
) => { ) => {
const { forceUpdate, projectDetail, labelState, updateLoadingFlag } = props const { forceUpdate, projectDetail, labelState, updateLoadingFlag } = props
false && updateLoadingFlag
const containerRef = useRef<any>(null) const containerRef = useRef<any>(null)
const contextMenuRef = useRef<HTMLDivElement>(null) const contextMenuRef = useRef<HTMLDivElement>(null)
const [imgSize, setImageSize] = useState<{ const [imgSize, setImageSize] = useState<{
@@ -884,164 +880,6 @@ const PaperContainer = (
}, [clearPendingFrameSwitch, loadingData]) }, [clearPendingFrameSwitch, loadingData])
const renderSupportAnnotation = useCallback( const renderSupportAnnotation = useCallback(
async (
points: Array<[number, number]>,
tags: number[],
clear_previous_state = false
) => {
try {
notifications.show({
id: "sam2",
message: "模型生成中",
loading: true,
autoClose: false,
})
const operationSchema =
projectDetail?.label_schema_list?.find(
(item) =>
item.category_id === +useTopToolsStore.getState().activeOperation
) || null
const strokeColor = operationSchema
? `rgb(${operationSchema.color[0]},${operationSchema.color[1]},${operationSchema.color[2]})`
: "rgb(0,0,0)"
const fillColor = operationSchema
? `rgba(${operationSchema.color[0]},${operationSchema.color[1]},${operationSchema.color[2]},0.3)`
: "rgba(0,0,0,0.3)"
const blankColor = operationSchema
? `rgba(${operationSchema.color[0]},${operationSchema.color[1]},${operationSchema.color[2]},0.01)`
: "rgba(0,0,0,0.01)"
const currentScale =
usePaperStore.getState().rasterScale[activeImage] ?? 1
const draw = () => {
points.forEach((point, index) => {
const drawPoint = new paper.Path.Circle({
center: point,
radius: 2,
fillColor: tags[index] === 1 ? "red" : "blue",
strokeColor: tags[index] === 1 ? "red" : "blue",
data: {
id: "support",
type: "point",
},
})
drawPoint.parent = usePaperStore.getState().group!
const raster = usePaperStore
.getState()
.group!.getItems({ data: { name: "pic" } })[0]
if (!raster.contains(drawPoint.position)) {
throw new Error("点位超出图片范围")
}
})
}
console.log(points, tags)
// 绘制点位
clearSupportAnnotationItem()
draw()
const picSize = usePaperStore.getState().rasterSize[activeImage]
const params = {
project_id: projectDetail?.id || 0,
file_name: activeImage,
image_hw:
picSize && picSize.length ? [picSize[1], picSize[0]] : [0, 0],
points_and_box: points.map(([x, y]) => [
Math.round(x / currentScale),
Math.round(y / currentScale),
]),
tags,
clear_previous_state,
}
try {
const res = await getAuxiliaryAnnotation(
msgpack.encode(params) as any
)
const data = msgpack.decode(new Uint8Array(res))
console.log(data)
if (data.msg === "success") {
const { contours, hollows } = data
const outerPaths: any[] = []
const innerPaths: any[] = []
contours.forEach((contour: any, index: number) => {
if (contour.length > 3) {
const path = new paper.Path({
segments: contour.map(([x, y]: [number, number]) => [
x * currentScale,
y * currentScale,
]),
fillColor: blankColor,
strokeColor: strokeColor,
closed: true,
data: {
id: "support",
type: "polygon",
isHollowPolygon: false,
fillColor,
strokeColor,
blankColor,
},
parent: usePaperStore.getState().group,
strokeScaling: false,
})
if (
Math.abs(path.area) < useTopToolsStore.getState().checkSize
) {
path.remove()
} else {
// path.scaling = usePaperStore.getState().group!.scaling;
if (hollows[index]) {
// 更新标志位
path.data.isHollowPolygon = true
innerPaths.push(path)
} else {
// 上色
path.fillColor = new paper.Color(fillColor)
outerPaths.push(path)
}
}
}
})
const compoundPath = new paper.CompoundPath({
children: [...outerPaths, ...innerPaths],
data: {
id: "support",
type: "parent",
},
strokeColor: strokeColor,
strokeWidth: 2,
fillColor: fillColor,
})
compoundPath.parent = usePaperStore.getState().group!
notifications.show({
id: "sam2",
message: "模型生成成功",
color: "green",
})
}
} catch (error) {
console.log(error)
}
} catch (error: any) {
console.log(error)
notifications.show({
id: "sam2",
message: error.message ?? "模型生成失败",
color: "red",
})
if (error.message === "点位超出图片范围") {
usePaperStore
.getState()
.supportTool?.emit("keydown", { key: "escape" })
}
}
},
[activeImage, projectDetail]
)
false && renderSupportAnnotation
const renderSupportAnnotation2 = useCallback(
async ( async (
points: Array<[number, number]>, points: Array<[number, number]>,
tags: number[], tags: number[],
@@ -1145,7 +983,7 @@ const PaperContainer = (
// const picSize = usePaperStore.getState().rasterSize[activeImage] // const picSize = usePaperStore.getState().rasterSize[activeImage]
try { try {
const data = await getAuxiliaryAnnotation2({ const data = await getAuxiliaryAnnotation({
project_id: projectId, project_id: projectId,
image_name: activeImage, image_name: activeImage,
// project_id: "image_test_bk", // project_id: "image_test_bk",
@@ -1233,180 +1071,174 @@ const PaperContainer = (
[activeImage, projectDetail] [activeImage, projectDetail]
) )
const renderTrackingAnnotation = useCallback(async () => { // const renderTrackingAnnotation = useCallback(async () => {
updateLoadingFlag(true) // updateLoadingFlag(true)
try { // try {
notifications.show({ // notifications.show({
id: "sam2", // id: "sam2",
message: "模型生成中", // message: "模型生成中",
loading: true, // loading: true,
autoClose: false, // autoClose: false,
}) // })
const ids = // const ids =
useObjectStore.getState().selectedPath[ // useObjectStore.getState().selectedPath[
useBottomToolsStore.getState().activeImage // useBottomToolsStore.getState().activeImage
] // ]
const picArr = useBottomToolsStore.getState().selectedItems // const picArr = useBottomToolsStore.getState().selectedItems
const [width, height] = usePaperStore.getState().rasterSize[ // const [width, height] = usePaperStore.getState().rasterSize[
activeImage // activeImage
] ?? [0, 0] // ] ?? [0, 0]
const currentScale = // const currentScale =
usePaperStore.getState().rasterScale[activeImage] ?? 1 // usePaperStore.getState().rasterScale[activeImage] ?? 1
if (ids.length === 1) { // if (ids.length === 1) {
const id = ids[0] // const id = ids[0]
let categoryId: any = null // let categoryId: any = null
let data: any = null // let data: any = null
const activeImageData = useLabelStore.getState().label.get(activeImage)! // const activeImageData = useLabelStore.getState().label.get(activeImage)!
for (let [category, objArr] of activeImageData.entries()) { // for (let [category, objArr] of activeImageData.entries()) {
objArr.forEach((item) => { // objArr.forEach((item) => {
if (item[0] === id) { // if (item[0] === id) {
categoryId = category // categoryId = category
data = item // data = item
} // }
}) // })
} // }
if (categoryId && data) { // if (categoryId && data) {
const contours: [number, number][] = [] // const contours: [number, number][] = []
const tags: boolean[] = [] // const tags: boolean[] = []
data[1].forEach((item: any) => { // data[1].forEach((item: any) => {
contours.push( // contours.push(
item.map(([x, y]: any) => [ // item.map(([x, y]: any) => [
Math.round(x / currentScale), // Math.round(x / currentScale),
Math.round(y / currentScale), // Math.round(y / currentScale),
]) // ])
) // )
tags.push(false) // tags.push(false)
}) // })
data[3].forEach((item: any) => { // data[3].forEach((item: any) => {
contours.push( // contours.push(
item.map(([x, y]: any) => [ // item.map(([x, y]: any) => [
Math.round(x / currentScale), // Math.round(x / currentScale),
Math.round(y / currentScale), // Math.round(y / currentScale),
]) // ])
) // )
tags.push(true) // tags.push(true)
}) // })
const file_names = [...new Set([activeImage, ...picArr])] // const file_names = [...new Set([activeImage, ...picArr])]
const params = { // const params = {
project_id: projectDetail?.id || 0, // project_id: projectDetail?.id || 0,
file_names, // file_names,
image_hw: [height, width], // image_hw: [height, width],
contours, // contours,
hollows: tags, // hollows: tags,
} // }
console.log("传参", params) // console.log("传参", params)
try { // try {
const res = await getTrackingAuxiliaryAnnotation( // const res = await getTrackingAuxiliaryAnnotation(
msgpack.encode(params) // msgpack.encode(params)
) // )
const resData = msgpack.decode(new Uint8Array(res)) // const resData = msgpack.decode(new Uint8Array(res))
if (resData.msg === "success") { // if (resData.msg === "success") {
const { // const {
future_contours, // future_contours,
}: { // }: {
future_contours: { // future_contours: {
contours: [number, number][][] // contours: [number, number][][]
hollows: boolean[] // hollows: boolean[]
}[] // }[]
} = resData // } = resData
const nowTaskData = safeClone(useLabelStore.getState().label) // const nowTaskData = safeClone(useLabelStore.getState().label)
future_contours.forEach(({ contours, hollows }, index) => { // future_contours.forEach(({ contours, hollows }, index) => {
const fileName = file_names[index + 1] // const fileName = file_names[index + 1]
const imgScale = // const imgScale =
usePaperStore.getState().rasterScale[fileName] ?? 1 // usePaperStore.getState().rasterScale[fileName] ?? 1
let newId = // let newId =
useRightToolsStore // useRightToolsStore
.getState() // .getState()
.pathIds.reduce((a, b) => Math.max(a, b), 0) + 1 // .pathIds.reduce((a, b) => Math.max(a, b), 0) + 1
useRightToolsStore.getState().pushPathId(newId) // useRightToolsStore.getState().pushPathId(newId)
// 处理点位数据 // // 处理点位数据
let segmentation: Array<[number, number][]> = [] // let segmentation: Array<[number, number][]> = []
let hollow_segmentation: Array<[number, number][]> = [] // let hollow_segmentation: Array<[number, number][]> = []
contours.forEach((contour, index) => { // contours.forEach((contour, index) => {
if (!hollows[index]) // if (!hollows[index])
segmentation.push( // segmentation.push(
contour.map(([x, y]: any) => [x * imgScale, y * imgScale]) // contour.map(([x, y]: any) => [x * imgScale, y * imgScale])
) // )
else // else
hollow_segmentation.push( // hollow_segmentation.push(
contour.map(([x, y]: any) => [x * imgScale, y * imgScale]) // contour.map(([x, y]: any) => [x * imgScale, y * imgScale])
) // )
}) // })
// 详情数据 // // 详情数据
let detail = { // let detail = {
...initialDetail, // ...initialDetail,
uid: usePermissionStore.getState().user_id, // uid: usePermissionStore.getState().user_id,
comment: [...useLabelStore.getState().labelDefaultComments], // comment: [...useLabelStore.getState().labelDefaultComments],
create_timestamp: dayjs().unix(), // create_timestamp: dayjs().unix(),
sub_attributes: {}, // sub_attributes: {},
image_id: activeImage, // image_id: activeImage,
category_id: categoryId, // category_id: categoryId,
} // }
const updateData: any = [ // const updateData: any = [
newId, // newId,
segmentation, // segmentation,
detail, // detail,
hollow_segmentation, // hollow_segmentation,
] // ]
if (nowTaskData.has(fileName)) { // if (nowTaskData.has(fileName)) {
const categoryMap = nowTaskData.get(fileName)! // const categoryMap = nowTaskData.get(fileName)!
if (categoryMap.has(categoryId)) { // if (categoryMap.has(categoryId)) {
const existArr = categoryMap.get(categoryId) // const existArr = categoryMap.get(categoryId)
existArr?.push(updateData) // existArr?.push(updateData)
} else { // } else {
categoryMap.set(categoryId, [updateData]) // categoryMap.set(categoryId, [updateData])
} // }
} else { // } else {
let m = new Map() // let m = new Map()
m.set(categoryId, [updateData]) // m.set(categoryId, [updateData])
nowTaskData.set(fileName, m) // nowTaskData.set(fileName, m)
} // }
}) // })
console.log(nowTaskData) // console.log(nowTaskData)
useLabelStore.getState().setLabel(nowTaskData) // useLabelStore.getState().setLabel(nowTaskData)
useLabelStore.getState().pushStateStack(nowTaskData) // useLabelStore.getState().pushStateStack(nowTaskData)
notifications.show({ // notifications.show({
id: "sam2", // id: "sam2",
message: "模型生成成功", // message: "模型生成成功",
color: "green", // color: "green",
}) // })
} // }
} catch (error) { // } catch (error) {
console.log(error) // console.log(error)
} // }
} // }
} else if (!ids.length) { // } else if (!ids.length) {
notifications.show({ // notifications.show({
id: "sam2", // id: "sam2",
message: "请先选中标注对象后再进行模型调用", // message: "请先选中标注对象后再进行模型调用",
color: "red", // color: "red",
}) // })
} else { // } else {
notifications.show({ // notifications.show({
id: "sam2", // id: "sam2",
message: "请勿选择多个标注对象进行模型调用", // message: "请勿选择多个标注对象进行模型调用",
color: "red", // color: "red",
}) // })
} // }
} catch (error) { // } catch (error) {
console.log(error) // console.log(error)
notifications.show({ // notifications.show({
id: "sam2", // id: "sam2",
message: "模型生成失败", // message: "模型生成失败",
color: "red", // color: "red",
}) // })
} finally { // } finally {
updateLoadingFlag(false) // updateLoadingFlag(false)
} // }
}, [activeImage, projectDetail?.id, updateLoadingFlag]) // }, [activeImage, projectDetail?.id, updateLoadingFlag])
// const getBase64 = (file: any): Promise<string> => const renderTrackingAnnotation = useCallback(async () => {}, [])
// new Promise((resolve, reject) => {
// const reader = new FileReader();
// reader.readAsDataURL(file);
// reader.onload = () => resolve(reader.result as string);
// reader.onerror = (error) => reject(error);
// });
useEffect(() => { useEffect(() => {
if (setup) { if (setup) {
@@ -1439,7 +1271,7 @@ const PaperContainer = (
let pointTool = new paper.Tool() let pointTool = new paper.Tool()
initPointTool(pointTool, renderCircle, renderPath, forceUpdate) initPointTool(pointTool, renderCircle, renderPath, forceUpdate)
let supportTool = new paper.Tool() let supportTool = new paper.Tool()
initSupportTool(supportTool, renderSupportAnnotation2) initSupportTool(supportTool, renderSupportAnnotation)
} }
return () => { return () => {
newGroup.remove() newGroup.remove()
@@ -1455,7 +1287,7 @@ const PaperContainer = (
initRectangleTool, initRectangleTool,
initSupportTool, initSupportTool,
initViewTool, initViewTool,
renderSupportAnnotation2, renderSupportAnnotation,
setup, setup,
]) ])
@@ -4148,7 +3980,7 @@ const PaperContainer = (
renderPolygons, renderPolygons,
handleCrosshairMove, handleCrosshairMove,
renderTrackingAnnotation, renderTrackingAnnotation,
renderSupportAnnotation2, renderSupportAnnotation,
saveSupportAnnotationData, saveSupportAnnotationData,
copyAnnotationDataToMultiFrame, copyAnnotationDataToMultiFrame,
})) }))