diff --git a/components/label/api/label/index.ts b/components/label/api/label/index.ts index d328016..b4a3e1b 100644 --- a/components/label/api/label/index.ts +++ b/components/label/api/label/index.ts @@ -120,6 +120,25 @@ export const getAuxiliaryAnnotation = (data: any) => { }) } +export const getAuxiliaryAnnotation2 = (data: { + project_id: string + image_name: string + prompt: { + foreground_points?: Array<[number, number]> + background_points?: Array<[number, number]> + boxes?: Array<[number, number, number, number]> + } +}) => { + return httpFetch<{ + contours: Array> + hollows: Array + }>({ + url: "http://172.30.21.211:9000/image_predictor", + method: "POST", + body: JSON.stringify(data), + }) +} + // 模型标注 追踪 export const getTrackingAuxiliaryAnnotation = (data: any) => { return httpFetch({ diff --git a/components/label/components/PaperContainer.tsx b/components/label/components/PaperContainer.tsx index 027758f..60af9e9 100644 --- a/components/label/components/PaperContainer.tsx +++ b/components/label/components/PaperContainer.tsx @@ -32,6 +32,7 @@ import React, { } from "react" import { getAuxiliaryAnnotation, + getAuxiliaryAnnotation2, getServerImage, getTrackingAuxiliaryAnnotation, } from "../api/label" @@ -92,6 +93,40 @@ const renderRaster = (paperGroup: paper.Group, option: any) => { return new paper.Raster(Object.assign({ parent: paperGroup }, option)) } +type NormalizedPoint = [number, number] + +const normalizePoint = (value: any): NormalizedPoint | null => { + if (value instanceof paper.Point) return [value.x, value.y] + if (value instanceof paper.Segment) { + return [value.point.x, value.point.y] + } + if (Array.isArray(value)) { + if ( + value.length >= 2 && + typeof value[0] === "number" && + typeof value[1] === "number" + ) { + return [value[0], value[1]] + } + if (value.length && Array.isArray(value[0])) { + return normalizePoint(value[0]) + } + } + if (value && typeof value.x === "number" && typeof value.y === "number") { + return [value.x, value.y] + } + if (value?.point) { + return normalizePoint(value.point) + } + return null +} + +const normalizeContour = (contour: any[] = []): NormalizedPoint[] => { + return contour + .map((point) => normalizePoint(point)) + .filter((point): point is NormalizedPoint => point !== null) +} + const PaperContainer = ( props: PaperComponentProps, ref: React.Ref | undefined @@ -304,7 +339,6 @@ const PaperContainer = ( tags, clear_previous_state, } - console.log("传参", params) try { const res = await getAuxiliaryAnnotation( msgpack.encode(params) as any @@ -391,6 +425,177 @@ const PaperContainer = ( [activeImage, projectDetail] ) + false && renderSupportAnnotation + + const renderSupportAnnotation2 = useCallback( + async ( + points: Array<[number, number]>, + tags: number[], + clear_previous_state = false + ) => { + void clear_previous_state + try { + notifications.show({ + id: "sam2", + message: "模型生成中", + loading: true, + autoClose: false, + }) + const operationSchema = + projectDetail?.label_schema_list?.find( + (item) => + item.category_id === +useTopToolsStore.getState().activeOperation + ) || null + const strokeColor = operationSchema + ? `rgb(${operationSchema.color[0]},${operationSchema.color[1]},${operationSchema.color[2]})` + : "rgb(0,0,0)" + const fillColor = operationSchema + ? `rgba(${operationSchema.color[0]},${operationSchema.color[1]},${operationSchema.color[2]},0.3)` + : "rgba(0,0,0,0.3)" + const blankColor = operationSchema + ? `rgba(${operationSchema.color[0]},${operationSchema.color[1]},${operationSchema.color[2]},0.01)` + : "rgba(0,0,0,0.01)" + const currentScale = + usePaperStore.getState().rasterScale[activeImage] ?? 1 + + const draw = () => { + points.forEach((point, index) => { + const drawPoint = new paper.Path.Circle({ + center: point, + radius: 2, + fillColor: tags[index] === 1 ? "red" : "blue", + strokeColor: tags[index] === 1 ? "red" : "blue", + data: { + id: "support", + type: "point", + }, + }) + drawPoint.parent = usePaperStore.getState().group! + const raster = usePaperStore + .getState() + .group!.getItems({ data: { name: "pic" } })[0] + if (!raster.contains(drawPoint.position)) { + throw new Error("点位超出图片范围") + } + }) + } + console.log(points, tags) + clearSupportAnnotationItem() + draw() + + const normalizedPoints = points.map(([x, y]) => [ + Math.round(x / currentScale), + Math.round(y / currentScale), + ]) as Array<[number, number]> + const foregroundPoints = normalizedPoints.filter( + (_point, index) => tags[index] === 1 + ) + const background_points = normalizedPoints.filter( + (_point, index) => tags[index] !== 1 + ) + const promptPoints = foregroundPoints.length + ? foregroundPoints + : normalizedPoints + // const promptBox = + // promptPoints.length > 1 ? getContourBox(promptPoints) : null + const projectId = + projectDetail?.name || + projectDetail?.rpath || + `${projectDetail?.id || ""}` + + if (!projectId) { + throw new Error("缺少 project_id,无法调用模型接口") + } + if (!promptPoints.length) { + throw new Error("缺少有效点位,无法调用模型接口") + } + // const picSize = usePaperStore.getState().rasterSize[activeImage] + + try { + const data = await getAuxiliaryAnnotation2({ + project_id: projectId, + image_name: activeImage, + // project_id: "image_test_bk", + prompt: { + foreground_points: promptPoints, + background_points: background_points, + // ...(promptBox ? { boxes: [promptBox] } : {}), + }, + }) + const { contours = [], hollows = [] } = data || {} + const outerPaths: any[] = [] + const innerPaths: any[] = [] + contours.forEach((contour: any, index: number) => { + const normalizedContour = normalizeContour(contour) + if (normalizedContour.length > 3) { + const path = new paper.Path({ + segments: normalizedContour.map(([x, y]) => [ + x * currentScale, + y * currentScale, + ]), + fillColor: blankColor, + strokeColor: strokeColor, + closed: true, + data: { + id: "support", + type: "polygon", + isHollowPolygon: false, + fillColor, + strokeColor, + blankColor, + }, + parent: usePaperStore.getState().group, + strokeScaling: false, + }) + if (Math.abs(path.area) < useTopToolsStore.getState().checkSize) { + path.remove() + } else { + if (hollows[index]) { + path.data.isHollowPolygon = true + innerPaths.push(path) + } else { + path.fillColor = new paper.Color(fillColor) + outerPaths.push(path) + } + } + } + }) + const compoundPath = new paper.CompoundPath({ + children: [...outerPaths, ...innerPaths], + data: { + id: "support", + type: "parent", + }, + strokeColor: strokeColor, + strokeWidth: 2, + fillColor: fillColor, + }) + compoundPath.parent = usePaperStore.getState().group! + notifications.show({ + id: "sam2", + message: "模型生成成功", + color: "green", + }) + } catch (error) { + console.log(error) + } + } catch (error: any) { + console.log(error) + notifications.show({ + id: "sam2", + message: error.message ?? "模型生成失败", + color: "red", + }) + if (error.message === "点位超出图片范围") { + usePaperStore + .getState() + .supportTool?.emit("keydown", { key: "escape" }) + } + } + }, + [activeImage, projectDetail] + ) + const renderTrackingAnnotation = useCallback(async () => { updateLoadingFlag(true) try { @@ -423,7 +628,6 @@ const PaperContainer = ( } }) } - console.log(data) if (categoryId && data) { const contours: [number, number][] = [] const tags: boolean[] = [] @@ -459,7 +663,6 @@ const PaperContainer = ( msgpack.encode(params) ) const resData = msgpack.decode(new Uint8Array(res)) - console.log(resData) if (resData.msg === "success") { const { future_contours, @@ -599,7 +802,7 @@ const PaperContainer = ( let pointTool = new paper.Tool() initPointTool(pointTool, renderCircle, renderPath, forceUpdate) let supportTool = new paper.Tool() - initSupportTool(supportTool, renderSupportAnnotation) + initSupportTool(supportTool, renderSupportAnnotation2) } return () => { newGroup.remove() @@ -615,7 +818,7 @@ const PaperContainer = ( initRectangleTool, initSupportTool, initViewTool, - renderSupportAnnotation, + renderSupportAnnotation2, setup, ]) @@ -2349,6 +2552,7 @@ const PaperContainer = ( renderPolygons, handleCrosshairMove, renderTrackingAnnotation, + renderSupportAnnotation2, saveSupportAnnotationData, copyAnnotationDataToMultiFrame, }))