Dwinzo-Beta
/
Dwinzo_dev
4
0
Fork 3
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/simulation-armbot-v2' into v2

This commit is contained in:
Jerald-Golden-B 2025-05-03 10:10:11 +05:30
parent 94c306c813 0e4005f31e
commit b8d24e2d32
15 changed files with 752 additions and 258 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
app/src/modules/builder/IntialLoad/loadInitialFloorItems.ts
View File

@ -71,7 +71,7 @@ async function loadInitialFloorItems(
// Check Three.js Cache // Check Three.js Cache
const cachedModel = THREE.Cache.get(item.modelfileID!); const cachedModel = THREE.Cache.get(item.modelfileID!);
if (cachedModel) { if (cachedModel) {
// console.log(`[Cache] Fetching ${item.modelName}`); //
processLoadedModel(cachedModel.scene.clone(), item, itemsGroup, setFloorItems, addEvent); processLoadedModel(cachedModel.scene.clone(), item, itemsGroup, setFloorItems, addEvent);
modelsLoaded++; modelsLoaded++;
checkLoadingCompletion(modelsLoaded, modelsToLoad, dracoLoader, resolve); checkLoadingCompletion(modelsLoaded, modelsToLoad, dracoLoader, resolve);
@ -81,7 +81,7 @@ async function loadInitialFloorItems(
// Check IndexedDB // Check IndexedDB
const indexedDBModel = await retrieveGLTF(item.modelfileID!); const indexedDBModel = await retrieveGLTF(item.modelfileID!);
if (indexedDBModel) { if (indexedDBModel) {
// console.log(`[IndexedDB] Fetching ${item.modelName}`); //
const blobUrl = URL.createObjectURL(indexedDBModel); const blobUrl = URL.createObjectURL(indexedDBModel);
loader.load(blobUrl, (gltf) => { loader.load(blobUrl, (gltf) => {
URL.revokeObjectURL(blobUrl); URL.revokeObjectURL(blobUrl);
@ -102,7 +102,7 @@ async function loadInitialFloorItems(
} }
// Fetch from Backend // Fetch from Backend
// console.log(`[Backend] Fetching ${item.modelName}`); //
const modelUrl = `${url_Backend_dwinzo}/api/v2/AssetFile/${item.modelfileID!}`; const modelUrl = `${url_Backend_dwinzo}/api/v2/AssetFile/${item.modelfileID!}`;
loader.load(modelUrl, async (gltf) => { loader.load(modelUrl, async (gltf) => {
const modelBlob = await fetch(modelUrl).then((res) => res.blob()); const modelBlob = await fetch(modelUrl).then((res) => res.blob());
@ -120,7 +120,7 @@ async function loadInitialFloorItems(
); );
}); });
} else { } else {
// console.log(`Item ${item.modelName} is not near`); //
setFloorItems((prevItems) => [ setFloorItems((prevItems) => [
...(prevItems || []), ...(prevItems || []),
{ {
@ -281,8 +281,8 @@ function processLoadedModel(
actionName: "Action 1", actionName: "Action 1",
actionType: "pickAndPlace", actionType: "pickAndPlace",
process: { process: {
startPoint: [0, 0, 0], startPoint: null,
endPoint: [0, 0, 0] endPoint: null
}, },
triggers: [] triggers: []
} }

12
app/src/modules/builder/geomentries/assets/assetManager.ts
View File

@ -18,7 +18,7 @@ export default async function assetManager(
const taskId = ++currentTaskId; // Increment taskId for each call const taskId = ++currentTaskId; // Increment taskId for each call
activePromises.set(taskId, true); // Mark task as active activePromises.set(taskId, true); // Mark task as active
// console.log("Received message from worker:", data); //
if (data.toRemove.length > 0) { if (data.toRemove.length > 0) {
data.toRemove.forEach((uuid: string) => { data.toRemove.forEach((uuid: string) => {
@ -58,7 +58,7 @@ export default async function assetManager(
// Check Three.js Cache // Check Three.js Cache
const cachedModel = THREE.Cache.get(item.modelfileID!); const cachedModel = THREE.Cache.get(item.modelfileID!);
if (cachedModel) { if (cachedModel) {
// console.log(`[Cache] Fetching ${item.modelName}`); //
processLoadedModel(cachedModel.scene.clone(), item, itemsGroup, resolve); processLoadedModel(cachedModel.scene.clone(), item, itemsGroup, resolve);
return; return;
} }
@ -66,7 +66,7 @@ export default async function assetManager(
// Check IndexedDB // Check IndexedDB
const indexedDBModel = await retrieveGLTF(item.modelfileID!); const indexedDBModel = await retrieveGLTF(item.modelfileID!);
if (indexedDBModel) { if (indexedDBModel) {
// console.log(`[IndexedDB] Fetching ${item.modelName}`); //
const blobUrl = URL.createObjectURL(indexedDBModel); const blobUrl = URL.createObjectURL(indexedDBModel);
loader.load( loader.load(
blobUrl, blobUrl,
@ -86,7 +86,7 @@ export default async function assetManager(
} }
// Fetch from Backend // Fetch from Backend
// console.log(`[Backend] Fetching ${item.modelName}`); //
loader.load( loader.load(
modelUrl, modelUrl,
async (gltf) => { async (gltf) => {
@ -114,14 +114,14 @@ export default async function assetManager(
const existingModel = itemsGroup?.current?.getObjectByProperty("uuid", item.modelUuid); const existingModel = itemsGroup?.current?.getObjectByProperty("uuid", item.modelUuid);
if (existingModel) { if (existingModel) {
// console.log(`Model ${item.modelName} already exists in the scene.`); //
resolve(); resolve();
return; return;
} }
const model = gltf; const model = gltf;
model.uuid = item.modelUuid; model.uuid = item.modelUuid;
model.userData = { name: item.modelName, modelId: item.modelfileID, modelUuid: item.modelUuid }; model.userData = { name: item.modelName, modelId: item.modelfileID, modelUuid: item.modelUuid, eventData: item.eventData };
model.scale.set(...CONSTANTS.assetConfig.defaultScaleBeforeGsap); model.scale.set(...CONSTANTS.assetConfig.defaultScaleBeforeGsap);
model.position.set(...item.position); model.position.set(...item.position);
model.rotation.set(item.rotation.x, item.rotation.y, item.rotation.z); model.rotation.set(item.rotation.x, item.rotation.y, item.rotation.z);

2
app/src/modules/scene/controls/selectionControls/moveControls.tsx
View File

@ -222,6 +222,8 @@ function MoveControls({ movedObjects, setMovedObjects, itemsGroupRef, copiedObje
event event
); );
} }
newFloorItem.eventData = eventData;
} }
} }

2
app/src/modules/scene/controls/selectionControls/rotateControls.tsx
View File

@ -222,6 +222,8 @@ function RotateControls({ rotatedObjects, setRotatedObjects, movedObjects, setMo
event event
); );
} }
newFloorItem.eventData = eventData;
} }
} }

62
app/src/modules/simulation/events/points/creator/pointsCreator.tsx
View File

@ -21,7 +21,7 @@ function PointsCreator() {
const [transformMode, setTransformMode] = useState<"translate" | "rotate" | null>(null); const [transformMode, setTransformMode] = useState<"translate" | "rotate" | null>(null);
const sphereRefs = useRef<{ [key: string]: THREE.Mesh }>({}); const sphereRefs = useRef<{ [key: string]: THREE.Mesh }>({});
const { selectedEventSphere, setSelectedEventSphere, clearSelectedEventSphere, } = useSelectedEventSphere(); const { selectedEventSphere, setSelectedEventSphere, clearSelectedEventSphere, } = useSelectedEventSphere();
const { setSelectedEventData, clearSelectedEventData } = useSelectedEventData(); const { selectedEventData,setSelectedEventData, clearSelectedEventData } = useSelectedEventData();
const { isDragging } = useIsDragging(); const { isDragging } = useIsDragging();
const { isRotating } = useIsRotating(); const { isRotating } = useIsRotating();
@ -30,6 +30,7 @@ function PointsCreator() {
const eventData = getEventByModelUuid( const eventData = getEventByModelUuid(
selectedEventSphere.userData.modelUuid selectedEventSphere.userData.modelUuid
); );
if (eventData) { if (eventData) {
setSelectedEventData(eventData, selectedEventSphere.userData.pointUuid); setSelectedEventData(eventData, selectedEventSphere.userData.pointUuid);
} else { } else {
@ -134,7 +135,10 @@ function PointsCreator() {
{events.map((event, i) => { {events.map((event, i) => {
if (event.type === "transfer") { if (event.type === "transfer") {
return ( return (
<group key={i} position={[...event.position]} rotation={[...event.rotation]} > <group
key={i}
position={new THREE.Vector3(...event.position)}
>
{event.points.map((point, j) => ( {event.points.map((point, j) => (
<mesh <mesh
name="Event-Sphere" name="Event-Sphere"
@ -146,10 +150,18 @@ function PointsCreator() {
sphereRefs.current[point.uuid] sphereRefs.current[point.uuid]
); );
}} }}
onPointerMissed={() => {
if (selectedEventData?.data.type !== 'vehicle') {
// clearSelectedEventSphere();
}
setTransformMode(null);
}}
key={`${i}-${j}`} key={`${i}-${j}`}
position={new THREE.Vector3(...point.position)} position={new THREE.Vector3(...point.position)}
// rotation={new THREE.Euler(...point.rotation)} userData={{
userData={{ modelUuid: event.modelUuid, pointUuid: point.uuid }} modelUuid: event.modelUuid,
pointUuid: point.uuid,
}}
> >
<sphereGeometry args={[0.1, 16, 16]} /> <sphereGeometry args={[0.1, 16, 16]} />
<meshStandardMaterial color="orange" /> <meshStandardMaterial color="orange" />
@ -159,7 +171,10 @@ function PointsCreator() {
); );
} else if (event.type === "vehicle") { } else if (event.type === "vehicle") {
return ( return (
<group key={i} position={[...event.position]} rotation={[...event.rotation]} > <group
key={i}
position={new THREE.Vector3(...event.position)}
>
<mesh <mesh
name="Event-Sphere" name="Event-Sphere"
uuid={event.point.uuid} uuid={event.point.uuid}
@ -171,8 +186,10 @@ function PointsCreator() {
); );
}} }}
position={new THREE.Vector3(...event.point.position)} position={new THREE.Vector3(...event.point.position)}
// rotation={new THREE.Euler(...event.point.rotation)} userData={{
userData={{ modelUuid: event.modelUuid, pointUuid: event.point.uuid }} modelUuid: event.modelUuid,
pointUuid: event.point.uuid,
}}
> >
<sphereGeometry args={[0.1, 16, 16]} /> <sphereGeometry args={[0.1, 16, 16]} />
<meshStandardMaterial color="blue" /> <meshStandardMaterial color="blue" />
@ -181,7 +198,10 @@ function PointsCreator() {
); );
} else if (event.type === "roboticArm") { } else if (event.type === "roboticArm") {
return ( return (
<group key={i} position={[...event.position]} rotation={[...event.rotation]} > <group
key={i}
position={new THREE.Vector3(...event.position)}
>
<mesh <mesh
name="Event-Sphere" name="Event-Sphere"
uuid={event.point.uuid} uuid={event.point.uuid}
@ -192,9 +212,14 @@ function PointsCreator() {
sphereRefs.current[event.point.uuid] sphereRefs.current[event.point.uuid]
); );
}} }}
onPointerMissed={() => {
setTransformMode(null);
}}
position={new THREE.Vector3(...event.point.position)} position={new THREE.Vector3(...event.point.position)}
// rotation={new THREE.Euler(...event.point.rotation)} userData={{
userData={{ modelUuid: event.modelUuid, pointUuid: event.point.uuid }} modelUuid: event.modelUuid,
pointUuid: event.point.uuid,
}}
> >
<sphereGeometry args={[0.1, 16, 16]} /> <sphereGeometry args={[0.1, 16, 16]} />
<meshStandardMaterial color="green" /> <meshStandardMaterial color="green" />
@ -203,7 +228,10 @@ function PointsCreator() {
); );
} else if (event.type === "machine") { } else if (event.type === "machine") {
return ( return (
<group key={i} position={[...event.position]} rotation={[...event.rotation]} > <group
key={i}
position={new THREE.Vector3(...event.position)}
>
<mesh <mesh
name="Event-Sphere" name="Event-Sphere"
uuid={event.point.uuid} uuid={event.point.uuid}
@ -214,9 +242,15 @@ function PointsCreator() {
sphereRefs.current[event.point.uuid] sphereRefs.current[event.point.uuid]
); );
}} }}
onPointerMissed={() => {
// clearSelectedEventSphere();
setTransformMode(null);
}}
position={new THREE.Vector3(...event.point.position)} position={new THREE.Vector3(...event.point.position)}
// rotation={new THREE.Euler(...event.point.rotation)} userData={{
userData={{ modelUuid: event.modelUuid, pointUuid: event.point.uuid }} modelUuid: event.modelUuid,
pointUuid: event.point.uuid,
}}
> >
<sphereGeometry args={[0.1, 16, 16]} /> <sphereGeometry args={[0.1, 16, 16]} />
<meshStandardMaterial color="purple" /> <meshStandardMaterial color="purple" />
@ -244,4 +278,4 @@ function PointsCreator() {
); );
} }
export default PointsCreator; export default PointsCreator;

246
app/src/modules/simulation/roboticArm/instances/animator/roboticArmAnimator.tsx
View File

@ -1,68 +1,220 @@
import React, { useEffect, useMemo, useRef, useState } from 'react' import React, { useEffect, useMemo, useRef, useState } from 'react';
import { useArmBotStore } from '../../../../../store/simulation/useArmBotStore'; import { useFrame } from '@react-three/fiber';
import { useFrame, useThree } from '@react-three/fiber'; import * as THREE from 'three';
import * as THREE from "three" import { Line } from '@react-three/drei';
import { usePlayButtonStore } from '../../../../../store/usePlayButtonStore'; import {
useAnimationPlaySpeed,
usePauseButtonStore,
usePlayButtonStore,
useResetButtonStore
} from '../../../../../store/usePlayButtonStore';
function RoboticArmAnimator({ armUuid, HandleCallback, currentPhase, ikSolver, targetBone, robot, logStatus, groupRef, processes, armBotCurveRef, path }: any) { function RoboticArmAnimator({
const { armBots } = useArmBotStore(); HandleCallback,
const { scene } = useThree(); restPosition,
const restSpeed = 0.1; ikSolver,
const restPosition = new THREE.Vector3(0, 1, -1.6); targetBone,
const initialCurveRef = useRef<THREE.CatmullRomCurve3 | null>(null); armBot,
const initialStartPositionRef = useRef<THREE.Vector3 | null>(null); logStatus,
const [initialProgress, setInitialProgress] = useState(0); path
const [progress, setProgress] = useState(0); }: any) {
const [needsInitialMovement, setNeedsInitialMovement] = useState(true); const progressRef = useRef(0);
const [isInitializing, setIsInitializing] = useState(true); const curveRef = useRef<THREE.Vector3[] | null>(null);
const { isPlaying } = usePlayButtonStore();
const statusRef = useRef("idle");
// Create a ref for initialProgress
const initialProgressRef = useRef(0);
const [currentPath, setCurrentPath] = useState<[number, number, number][]>([]); const [currentPath, setCurrentPath] = useState<[number, number, number][]>([]);
const [circlePoints, setCirclePoints] = useState<[number, number, number][]>([]);
const [customCurvePoints, setCustomCurvePoints] = useState<THREE.Vector3[] | null>(null);
// Zustand stores
const { isPlaying } = usePlayButtonStore();
const { isPaused } = usePauseButtonStore();
const { isReset } = useResetButtonStore();
const { speed } = useAnimationPlaySpeed();
// Update path state whenever `path` prop changes
useEffect(() => { useEffect(() => {
setCurrentPath(path);
setCurrentPath(path) }, [path]);
}, [path])
// Reset logic when `isPlaying` changes
useEffect(() => { useEffect(() => {
if (!isPlaying) {
setCurrentPath([]);
curveRef.current = null;
}
}, [isPlaying]);
}, [currentPath]) // Handle circle points based on armBot position
useEffect(() => {
const points = generateRingPoints(1.6, 64)
setCirclePoints(points);
}, [armBot.position]);
function generateRingPoints(radius: any, segments: any) {
const points: [number, number, number][] = [];
for (let i = 0; i < segments; i++) {
// Calculate angle for current segment
const angle = (i / segments) * Math.PI * 2;
// Calculate x and z coordinates (y remains the same for a flat ring)
const x = Math.cos(angle) * radius;
const z = Math.sin(angle) * radius;
points.push([x, 1.5, z]);
}
return points;
}
const findNearestIndex = (nearestPoint: [number, number, number], points: [number, number, number][], epsilon = 1e-6) => {
for (let i = 0; i < points.length; i++) {
const [x, y, z] = points[i];
if (
Math.abs(x - nearestPoint[0]) < epsilon &&
Math.abs(y - nearestPoint[1]) < epsilon &&
Math.abs(z - nearestPoint[2]) < epsilon
) {
return i; // Found the matching index
}
}
return -1; // Not found
};
// Handle nearest points and final path (including arc points)
useEffect(() => {
if (circlePoints.length > 0 && currentPath.length > 0) {
const start = currentPath[0];
const end = currentPath[currentPath.length - 1];
const raisedStart = [start[0], start[1] + 0.5, start[2]] as [number, number, number];
const raisedEnd = [end[0], end[1] + 0.5, end[2]] as [number, number, number];
const findNearest = (target: [number, number, number]) => {
return circlePoints.reduce((nearest, point) => {
const distance = Math.hypot(
target[0] - point[0],
target[1] - point[1],
target[2] - point[2]
);
const nearestDistance = Math.hypot(
target[0] - nearest[0],
target[1] - nearest[1],
target[2] - nearest[2]
);
return distance < nearestDistance ? point : nearest;
}, circlePoints[0]);
};
const nearestToStart = findNearest(raisedStart);
const nearestToEnd = findNearest(raisedEnd);
const indexOfNearestStart = findNearestIndex(nearestToStart, circlePoints);
const indexOfNearestEnd = findNearestIndex(nearestToEnd, circlePoints);
// Find clockwise and counter-clockwise distances
const clockwiseDistance = (indexOfNearestEnd - indexOfNearestStart + 64) % 64;
const counterClockwiseDistance = (indexOfNearestStart - indexOfNearestEnd + 64) % 64;
const clockwiseIsShorter = clockwiseDistance <= counterClockwiseDistance;
// Collect arc points between start and end
let arcPoints: [number, number, number][] = [];
if (clockwiseIsShorter) {
if (indexOfNearestStart <= indexOfNearestEnd) {
arcPoints = circlePoints.slice(indexOfNearestStart, indexOfNearestEnd + 1);
} else {
// Wrap around
arcPoints = [
...circlePoints.slice(indexOfNearestStart, 64),
...circlePoints.slice(0, indexOfNearestEnd + 1)
];
}
} else {
if (indexOfNearestStart >= indexOfNearestEnd) {
for (let i = indexOfNearestStart; i !== (indexOfNearestEnd - 1 + 64) % 64; i = (i - 1 + 64) % 64) {
arcPoints.push(circlePoints[i]);
}
} else {
for (let i = indexOfNearestStart; i !== (indexOfNearestEnd - 1 + 64) % 64; i = (i - 1 + 64) % 64) {
arcPoints.push(circlePoints[i]);
}
}
}
// Continue your custom path logic
const pathVectors = [
new THREE.Vector3(start[0], start[1], start[2]), // start
new THREE.Vector3(raisedStart[0], raisedStart[1], raisedStart[2]), // lift up
new THREE.Vector3(nearestToStart[0], raisedStart[1], nearestToStart[2]), // move to arc start
...arcPoints.map(point => new THREE.Vector3(point[0], raisedStart[1], point[2])),
new THREE.Vector3(nearestToEnd[0], raisedEnd[1], nearestToEnd[2]), // move from arc end
new THREE.Vector3(raisedEnd[0], raisedEnd[1], raisedEnd[2]), // lowered end
new THREE.Vector3(end[0], end[1], end[2]) // end
];
const customCurve = new THREE.CatmullRomCurve3(pathVectors, false, 'centripetal', 1);
const generatedPoints = customCurve.getPoints(100);
setCustomCurvePoints(generatedPoints);
}
}, [circlePoints, currentPath]);
// Frame update for animation
useFrame((_, delta) => { useFrame((_, delta) => {
if (!ikSolver || !currentPath || currentPath.length === 0) return; if (!ikSolver) return;
const bone = ikSolver.mesh.skeleton.bones.find( const bone = ikSolver.mesh.skeleton.bones.find((b: any) => b.name === targetBone);
(b: any) => b.name === targetBone
);
if (!bone) return; if (!bone) return;
// Ensure currentPath is a valid array of 3D points, create a CatmullRomCurve3 from it if (isPlaying) {
const curve = new THREE.CatmullRomCurve3( if (!isPaused && customCurvePoints && currentPath.length > 0) {
currentPath.map(point => new THREE.Vector3(point[0], point[1], point[2])) const curvePoints = customCurvePoints;
); const speedAdjustedProgress = progressRef.current + (speed * armBot.speed);
const index = Math.floor(speedAdjustedProgress);
if (index >= curvePoints.length) {
// Reached the end of the curve
HandleCallback();
setCurrentPath([]);
curveRef.current = null;
progressRef.current = 0;
} else {
const point = curvePoints[index];
bone.position.copy(point);
progressRef.current = speedAdjustedProgress;
}
} else if (isPaused) {
logStatus(armBot.modelUuid, 'Simulation Paused');
}
const next = initialProgressRef.current + delta * 0.5; ikSolver.update();
if (next >= 1) { } else if (!isPlaying && currentPath.length === 0) {
// bone.position.copy(restPosition); // Not playing anymore, reset to rest
HandleCallback(); // Call the callback when the path is completed bone.position.copy(restPosition);
initialProgressRef.current = 0; // Set ref to 1 when done ikSolver.update();
} else {
const point = curve.getPoint(next); // Get the interpolated point from the curve
bone.position.copy(point); // Update the bone position along the curve
initialProgressRef.current = next; // Update progress
} }
ikSolver.update();
}); });
return ( return (
<></> <>
) {customCurvePoints && currentPath && isPlaying && (
<mesh rotation={armBot.rotation} position={armBot.position}>
<Line
points={customCurvePoints.map((p) => [p.x, p.y, p.z] as [number, number, number])}
color="green"
lineWidth={5}
dashed={false}
/>
</mesh>
)}
<mesh position={[armBot.position[0], armBot.position[1] + 1.5, armBot.position[2]]} rotation={[-Math.PI / 2, 0, 0]}>
<ringGeometry args={[1.59, 1.61, 64]} />
<meshBasicMaterial color="green" side={THREE.DoubleSide} />
</mesh>
</>
);
} }
export default RoboticArmAnimator; export default RoboticArmAnimator;

307
app/src/modules/simulation/roboticArm/instances/armInstance/roboticArmInstance.tsx
View File

@ -1,61 +1,141 @@
import React, { useEffect, useRef, useState } from 'react' import React, { useEffect, useRef, useState } from 'react'
import IKInstance from '../ikInstance/ikInstance'; import IKInstance from '../ikInstance/ikInstance';
import RoboticArmAnimator from '../animator/roboticArmAnimator'; import RoboticArmAnimator from '../animator/roboticArmAnimator';
import { usePlayButtonStore } from '../../../../../store/usePlayButtonStore'; import { usePauseButtonStore, usePlayButtonStore, useResetButtonStore } from '../../../../../store/usePlayButtonStore';
import { useArmBotStore } from '../../../../../store/simulation/useArmBotStore'; import { useArmBotStore } from '../../../../../store/simulation/useArmBotStore';
import armModel from "../../../../../assets/gltf-glb/rigged/ik_arm_4.glb"; import armModel from "../../../../../assets/gltf-glb/rigged/ik_arm_4.glb";
import { useThree } from "@react-three/fiber"; import { useThree } from "@react-three/fiber";
import { useFloorItems } from '../../../../../store/store'; import { useFloorItems } from '../../../../../store/store';
import useModuleStore from '../../../../../store/useModuleStore'; import useModuleStore from '../../../../../store/useModuleStore';
import { Vector3 } from "three";
import * as THREE from "three"; import * as THREE from "three";
import { useSelectedAction, useSelectedProduct } from '../../../../../store/simulation/useSimulationStore';
import { useProductStore } from '../../../../../store/simulation/useProductStore';
interface Process {
triggerId: string;
startPoint?: Vector3;
endPoint?: Vector3;
speed: number;
}
function RoboticArmInstance({ robot }: { robot: ArmBotStatus }) { function RoboticArmInstance({ armBot }: { armBot: ArmBotStatus }) {
const { isPlaying } = usePlayButtonStore();
const [currentPhase, setCurrentPhase] = useState<(string)>("init"); const [currentPhase, setCurrentPhase] = useState<(string)>("init");
const { scene } = useThree();
const targetBone = "Target";
const { activeModule } = useModuleStore();
const [ikSolver, setIkSolver] = useState<any>(null);
const { addCurrentAction, setArmBotActive, setArmBotState, removeCurrentAction } = useArmBotStore();
const { floorItems } = useFloorItems();
const groupRef = useRef<any>(null);
const [processes, setProcesses] = useState<Process[]>([]);
const [armBotCurvePoints, setArmBotCurvePoints] = useState({ start: [], end: [] })
const restPosition = new THREE.Vector3(0, 1, -1.6);
let armBotCurveRef = useRef<THREE.CatmullRomCurve3 | null>(null)
const [path, setPath] = useState<[number, number, number][]>([]); const [path, setPath] = useState<[number, number, number][]>([]);
const [ikSolver, setIkSolver] = useState<any>(null);
const { scene } = useThree();
const restPosition = new THREE.Vector3(0, 1.75, -1.6);
const targetBone = "Target";
const groupRef = useRef<any>(null);
const pauseTimeRef = useRef<number | null>(null);
const isPausedRef = useRef<boolean>(false);
let startTime: number;
//zustand
const { addCurrentAction, setArmBotActive, setArmBotState, removeCurrentAction } = useArmBotStore();
const { products, getActionByUuid } = useProductStore();
const { selectedProduct } = useSelectedProduct();
const { floorItems } = useFloorItems();
const { activeModule } = useModuleStore();
const { isPlaying } = usePlayButtonStore();
const { isReset, setReset } = useResetButtonStore();
const { isPaused } = usePauseButtonStore();
const { selectedAction } = useSelectedAction();
function firstFrame() {
startTime = performance.now();
step();
}
function step() {
if (isPausedRef.current) {
if (!pauseTimeRef.current) {
pauseTimeRef.current = performance.now();
}
requestAnimationFrame(() => step());
return;
}
if (pauseTimeRef.current) {
const pauseDuration = performance.now() - pauseTimeRef.current;
startTime += pauseDuration;
pauseTimeRef.current = null;
}
const elapsedTime = performance.now() - startTime;
if (elapsedTime < 1500) {
// Wait until 1500ms has passed
requestAnimationFrame(step);
return;
}
if (currentPhase === "picking") {
setArmBotActive(armBot.modelUuid, true);
setArmBotState(armBot.modelUuid, "running");
setCurrentPhase("start-to-end");
startTime = 0
const startPoint = armBot.point.actions[0].process.startPoint;
const endPoint = armBot.point.actions[0].process.endPoint;
if (startPoint && endPoint) {
let curve = createCurveBetweenTwoPoints(
new THREE.Vector3(startPoint[0], startPoint[1], startPoint[2]),
new THREE.Vector3(endPoint[0], endPoint[1], endPoint[2]));
if (curve) {
logStatus(armBot.modelUuid, "picking the object");
setPath(curve.points.map(point => [point.x, point.y, point.z]))
}
}
logStatus(armBot.modelUuid, "Moving armBot from start point to end position.")
} else if (currentPhase === "dropping") {
setArmBotActive(armBot.modelUuid, true);
setArmBotState(armBot.modelUuid, "running");
setCurrentPhase("end-to-rest");
startTime = 0;
const endPoint = armBot.point.actions[0].process.endPoint;
if (endPoint) {
let curve = createCurveBetweenTwoPoints(new THREE.Vector3(endPoint[0], endPoint[1], endPoint[2]), restPosition);
if (curve) {
logStatus(armBot.modelUuid, "dropping the object");
setPath(curve.points.map(point => [point.x, point.y, point.z]));
}
}
logStatus(armBot.modelUuid, "Moving armBot from end point to rest position.")
}
}
useEffect(() => {
isPausedRef.current = isPaused;
}, [isPaused]);
useEffect(() => { useEffect(() => {
let armItems = floorItems?.filter((val: any) => const targetMesh = scene?.getObjectByProperty("uuid", armBot.modelUuid);
val.modelUuid === "3abf5d46-b59e-4e6b-9c02-a4634b64b82d"
); if (targetMesh) {
// Get the first matching item targetMesh.visible = activeModule !== "simulation"
let armItem = armItems?.[0];
if (armItem) {
const targetMesh = scene?.getObjectByProperty("uuid", armItem.modelUuid);
if (targetMesh) {
targetMesh.visible = activeModule !== "simulation"
}
} }
const targetBones = ikSolver?.mesh.skeleton.bones.find( const targetBones = ikSolver?.mesh.skeleton.bones.find(
(b: any) => b.name === targetBone (b: any) => b.name === targetBone
); );
if (isReset) {
logStatus(armBot.modelUuid, "Simulation Play Reset Successfully")
removeCurrentAction(armBot.modelUuid)
setArmBotActive(armBot.modelUuid, true)
setArmBotState(armBot.modelUuid, "running")
setCurrentPhase("init-to-rest");
isPausedRef.current = false
pauseTimeRef.current = null
isPausedRef.current = false
startTime = 0
if (targetBones) {
let curve = createCurveBetweenTwoPoints(targetBones.position, targetBones.position)
if (curve) {
setPath(curve.points.map(point => [point.x, point.y, point.z]));
}
}
setReset(false);
logStatus(armBot.modelUuid, "Moving armBot from initial point to rest position.")
}
if (isPlaying) { if (isPlaying) {
//Moving armBot from initial point to rest position.
if (!robot?.isActive && robot?.state == "idle" && currentPhase == "init") {
setArmBotActive(robot.modelUuid, true) //Moving armBot from initial point to rest position.
setArmBotState(robot.modelUuid, "running") if (!armBot?.isActive && armBot?.state == "idle" && currentPhase == "init") {
setArmBotActive(armBot.modelUuid, true)
setArmBotState(armBot.modelUuid, "running")
setCurrentPhase("init-to-rest"); setCurrentPhase("init-to-rest");
if (targetBones) { if (targetBones) {
let curve = createCurveBetweenTwoPoints(targetBones.position, restPosition) let curve = createCurveBetweenTwoPoints(targetBones.position, restPosition)
@ -63,21 +143,26 @@ function RoboticArmInstance({ robot }: { robot: ArmBotStatus }) {
setPath(curve.points.map(point => [point.x, point.y, point.z])); setPath(curve.points.map(point => [point.x, point.y, point.z]));
} }
} }
logStatus(robot.modelUuid, "Moving armBot from initial point to rest position.") logStatus(armBot.modelUuid, "Moving armBot from initial point to rest position.")
} }
//Waiting for trigger. //Waiting for trigger.
else if (robot && !robot.isActive && robot.state === "idle" && currentPhase === "rest" && !robot.currentAction) { else if (armBot && !armBot.isActive && armBot.state === "idle" && currentPhase === "rest" && !armBot.currentAction) {
logStatus(robot.modelUuid, "Waiting to trigger CurrentAction") logStatus(armBot.modelUuid, "Waiting to trigger CurrentAction")
setTimeout(() => { const timeoutId = setTimeout(() => {
addCurrentAction(robot.modelUuid, 'action-003'); addCurrentAction(armBot.modelUuid, selectedAction?.actionId);
console.log('selectedAction?.actionId: ', selectedAction?.actionId);
}, 3000); }, 3000);
return () => clearTimeout(timeoutId);
} }
else if (robot && !robot.isActive && robot.state === "idle" && currentPhase === "rest" && robot.currentAction) { else if (armBot && !armBot.isActive && armBot.state === "idle" && currentPhase === "rest" && armBot.currentAction) {
if (robot.currentAction) { if (armBot.currentAction) {
setArmBotActive(robot.modelUuid, true);
setArmBotState(robot.modelUuid, "running"); setArmBotActive(armBot.modelUuid, true);
setArmBotState(armBot.modelUuid, "running");
setCurrentPhase("rest-to-start"); setCurrentPhase("rest-to-start");
const startPoint = robot.point.actions[0].process.startPoint; let actiondata = getActionByUuid(selectedProduct.productId, selectedAction.actionId)
console.log('actiondata: ', actiondata);
const startPoint = armBot.point.actions[0].process.startPoint;
if (startPoint) { if (startPoint) {
let curve = createCurveBetweenTwoPoints(restPosition, new THREE.Vector3(startPoint[0], startPoint[1], startPoint[2])); let curve = createCurveBetweenTwoPoints(restPosition, new THREE.Vector3(startPoint[0], startPoint[1], startPoint[2]));
if (curve) { if (curve) {
@ -85,105 +170,109 @@ function RoboticArmInstance({ robot }: { robot: ArmBotStatus }) {
} }
} }
} }
logStatus(robot.modelUuid, "Moving armBot from rest point to start position.") logStatus(armBot.modelUuid, "Moving armBot from rest point to start position.")
} }
else if (robot && !robot.isActive && robot.state === "idle" && currentPhase === "picking" && robot.currentAction) { else if (armBot && !armBot.isActive && armBot.state === "idle" && currentPhase === "picking" && armBot.currentAction) {
setArmBotActive(robot.modelUuid, true); requestAnimationFrame(firstFrame);
setArmBotState(robot.modelUuid, "running"); // setArmBotActive(armBot.modelUuid, true);
setCurrentPhase("start-to-end"); // setArmBotState(armBot.modelUuid, "running");
const startPoint = robot.point.actions[0].process.startPoint; // setCurrentPhase("start-to-end");
const endPoint = robot.point.actions[0].process.endPoint; // const startPoint = armBot.point.actions[0].process.startPoint;
if (startPoint && endPoint) { // const endPoint = armBot.point.actions[0].process.endPoint;
let curve = createCurveBetweenTwoPoints( // if (startPoint && endPoint) {
new THREE.Vector3(startPoint[0], startPoint[1], startPoint[2]), // let curve = createCurveBetweenTwoPoints(
new THREE.Vector3(endPoint[0], endPoint[1], endPoint[2]) // new THREE.Vector3(startPoint[0], startPoint[1], startPoint[2]),
); // new THREE.Vector3(endPoint[0], endPoint[1], endPoint[2]));
if (curve) { // if (curve) {
setTimeout(() => { // setTimeout(() => {
logStatus(robot.modelUuid, "picking the object"); // logStatus(armBot.modelUuid, "picking the object");
setPath(curve.points.map(point => [point.x, point.y, point.z])); // setPath(curve.points.map(point => [point.x, point.y, point.z]));
}, 1500) // }, 1500)
} // }
} // }
logStatus(robot.modelUuid, "Moving armBot from start point to end position.") // logStatus(armBot.modelUuid, "Moving armBot from start point to end position.")
} }
else if (robot && !robot.isActive && robot.state === "idle" && currentPhase === "dropping" && robot.currentAction) { else if (armBot && !armBot.isActive && armBot.state === "idle" && currentPhase === "dropping" && armBot.currentAction) {
setArmBotActive(robot.modelUuid, true); requestAnimationFrame(firstFrame);
setArmBotState(robot.modelUuid, "running"); // setArmBotActive(armBot.modelUuid, true);
setCurrentPhase("end-to-rest"); // setArmBotState(armBot.modelUuid, "running");
const endPoint = robot.point.actions[0].process.endPoint; // setCurrentPhase("end-to-rest");
if (endPoint) { // const endPoint = armBot.point.actions[0].process.endPoint;
let curve = createCurveBetweenTwoPoints(new THREE.Vector3(endPoint[0], endPoint[1], endPoint[2]), restPosition // if (endPoint) {
); // let curve = createCurveBetweenTwoPoints(new THREE.Vector3(endPoint[0], endPoint[1], endPoint[2]), restPosition);
if (curve) { // if (curve) {
setTimeout(() => { // setTimeout(() => {
logStatus(robot.modelUuid, "dropping the object"); // logStatus(armBot.modelUuid, "dropping the object");
setPath(curve.points.map(point => [point.x, point.y, point.z])); // setPath(curve.points.map(point => [point.x, point.y, point.z]));
}, 1500) // }, 1500)
} // }
} // }
logStatus(robot.modelUuid, "Moving armBot from end point to rest position.") // logStatus(armBot.modelUuid, "Moving armBot from end point to rest position.")
} }
} else {
logStatus(armBot.modelUuid, "Simulation Play Stopped")
setArmBotActive(armBot.modelUuid, false)
setArmBotState(armBot.modelUuid, "idle")
setCurrentPhase("init");
setPath([])
removeCurrentAction(armBot.modelUuid)
} }
}, [currentPhase, robot, isPlaying, ikSolver]) }, [currentPhase, armBot, isPlaying, ikSolver, isReset])
function createCurveBetweenTwoPoints(p1: any, p2: any) { function createCurveBetweenTwoPoints(p1: any, p2: any) {
const mid = new THREE.Vector3().addVectors(p1, p2).multiplyScalar(0.5); const mid = new THREE.Vector3().addVectors(p1, p2).multiplyScalar(0.5);
mid.y += 0.5; // mid.y += 0.5;
const points = [p1, mid, p2]; const points = [p1, mid, p2];
return new THREE.CatmullRomCurve3(points); return new THREE.CatmullRomCurve3(points);
} }
const HandleCallback = () => { const HandleCallback = () => {
if (robot.isActive && robot.state == "running" && currentPhase == "init-to-rest") { if (armBot.isActive && armBot.state == "running" && currentPhase == "init-to-rest") {
logStatus(robot.modelUuid, "Callback triggered: rest"); logStatus(armBot.modelUuid, "Callback triggered: rest");
setArmBotActive(robot.modelUuid, false) setArmBotActive(armBot.modelUuid, false)
setArmBotState(robot.modelUuid, "idle") setArmBotState(armBot.modelUuid, "idle")
setCurrentPhase("rest"); setCurrentPhase("rest");
setPath([]) setPath([])
} }
else if (robot.isActive && robot.state == "running" && currentPhase == "rest-to-start") { else if (armBot.isActive && armBot.state == "running" && currentPhase == "rest-to-start") {
logStatus(robot.modelUuid, "Callback triggered: pick."); logStatus(armBot.modelUuid, "Callback triggered: pick.");
setArmBotActive(robot.modelUuid, false) setArmBotActive(armBot.modelUuid, false)
setArmBotState(robot.modelUuid, "idle") setArmBotState(armBot.modelUuid, "idle")
setCurrentPhase("picking"); setCurrentPhase("picking");
setPath([]) setPath([])
} }
else if (robot.isActive && robot.state == "running" && currentPhase == "start-to-end") { else if (armBot.isActive && armBot.state == "running" && currentPhase == "start-to-end") {
logStatus(robot.modelUuid, "Callback triggered: drop."); logStatus(armBot.modelUuid, "Callback triggered: drop.");
setArmBotActive(robot.modelUuid, false) setArmBotActive(armBot.modelUuid, false)
setArmBotState(robot.modelUuid, "idle") setArmBotState(armBot.modelUuid, "idle")
setCurrentPhase("dropping"); setCurrentPhase("dropping");
setPath([]) setPath([])
} }
else if (robot.isActive && robot.state == "running" && currentPhase == "end-to-rest") { else if (armBot.isActive && armBot.state == "running" && currentPhase == "end-to-rest") {
logStatus(robot.modelUuid, "Callback triggered: rest, cycle completed."); logStatus(armBot.modelUuid, "Callback triggered: rest, cycle completed.");
setArmBotActive(robot.modelUuid, false) setArmBotActive(armBot.modelUuid, false)
setArmBotState(robot.modelUuid, "idle") setArmBotState(armBot.modelUuid, "idle")
setCurrentPhase("rest"); setCurrentPhase("rest");
setPath([]) setPath([])
removeCurrentAction(robot.modelUuid) removeCurrentAction(armBot.modelUuid)
} }
} }
const logStatus = (id: string, status: string) => { const logStatus = (id: string, status: string) => {
// //
} }
return ( return (
<> <>
<IKInstance modelUrl={armModel} setIkSolver={setIkSolver} ikSolver={ikSolver} robot={robot} groupRef={groupRef} processes={processes} <IKInstance modelUrl={armModel} setIkSolver={setIkSolver} ikSolver={ikSolver} armBot={armBot} groupRef={groupRef} />
setArmBotCurvePoints={setArmBotCurvePoints} /> <RoboticArmAnimator HandleCallback={HandleCallback} restPosition={restPosition} ikSolver={ikSolver} targetBone={targetBone} armBot={armBot}
<RoboticArmAnimator armUuid={robot?.modelUuid} HandleCallback={HandleCallback} logStatus={logStatus} path={path} currentPhase={currentPhase} />
currentPhase={currentPhase} targetBone={targetBone} ikSolver={ikSolver} robot={robot}
logStatus={logStatus} groupRef={groupRef} processes={processes} armBotCurveRef={armBotCurveRef} path={path} />
</> </>
) )
} }
export default RoboticArmInstance; export default RoboticArmInstance;

26
app/src/modules/simulation/roboticArm/instances/ikInstance/ikInstance.tsx
View File

@ -3,20 +3,19 @@ import * as THREE from "three";
import { DRACOLoader } from "three/examples/jsm/loaders/DRACOLoader"; import { DRACOLoader } from "three/examples/jsm/loaders/DRACOLoader";
import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader"; import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader";
import { clone } from "three/examples/jsm/utils/SkeletonUtils"; import { clone } from "three/examples/jsm/utils/SkeletonUtils";
import { useFrame, useLoader, useThree } from "@react-three/fiber"; import { useLoader, useThree } from "@react-three/fiber";
import { CCDIKSolver, CCDIKHelper, } from "three/examples/jsm/animation/CCDIKSolver"; import { CCDIKSolver, CCDIKHelper, } from "three/examples/jsm/animation/CCDIKSolver";
import { TransformControls } from '@react-three/drei';
type IKInstanceProps = { type IKInstanceProps = {
modelUrl: string; modelUrl: string;
ikSolver: any; ikSolver: any;
setIkSolver: any setIkSolver: any
robot: any; armBot: any;
groupRef: React.RefObject<THREE.Group>; groupRef: any;
processes: any;
setArmBotCurvePoints: any
}; };
function IKInstance({ modelUrl, setIkSolver, ikSolver, robot, groupRef, processes, setArmBotCurvePoints }: IKInstanceProps) { function IKInstance({ modelUrl, setIkSolver, ikSolver, armBot, groupRef }: IKInstanceProps) {
const { scene } = useThree()
const { scene } = useThree();
const gltf = useLoader(GLTFLoader, modelUrl, (loader) => { const gltf = useLoader(GLTFLoader, modelUrl, (loader) => {
const draco = new DRACOLoader(); const draco = new DRACOLoader();
draco.setDecoderPath("https://cdn.jsdelivr.net/npm/three@0.160.0/examples/jsm/libs/draco/"); draco.setDecoderPath("https://cdn.jsdelivr.net/npm/three@0.160.0/examples/jsm/libs/draco/");
@ -25,6 +24,8 @@ function IKInstance({ modelUrl, setIkSolver, ikSolver, robot, groupRef, processe
const cloned = useMemo(() => clone(gltf?.scene), [gltf]); const cloned = useMemo(() => clone(gltf?.scene), [gltf]);
const targetBoneName = "Target"; const targetBoneName = "Target";
const skinnedMeshName = "link_0"; const skinnedMeshName = "link_0";
const [selectedArm, setSelectedArm] = useState<THREE.Group>();
useEffect(() => { useEffect(() => {
if (!gltf) return; if (!gltf) return;
const OOI: any = {}; const OOI: any = {};
@ -66,15 +67,19 @@ function IKInstance({ modelUrl, setIkSolver, ikSolver, robot, groupRef, processe
setIkSolver(solver); setIkSolver(solver);
const helper = new CCDIKHelper(OOI.Skinned_Mesh, iks, 0.05) const helper = new CCDIKHelper(OOI.Skinned_Mesh, iks, 0.05)
// groupRef.current.add(helper);
// scene.add(helper) setSelectedArm(OOI.Target_Bone);
scene.add(helper)
}, [gltf]); }, [gltf]);
return ( return (
<> <>
<group ref={groupRef} position={robot.position} rotation={robot.rotation}> <group ref={groupRef} position={armBot.position} rotation={armBot.rotation} onClick={() => {
setSelectedArm(groupRef.current?.getObjectByName(targetBoneName))
}}>
<primitive <primitive
uuid={"ArmBot-X200"} uuid={"ArmBot-X200"}
object={cloned} object={cloned}
@ -82,6 +87,7 @@ function IKInstance({ modelUrl, setIkSolver, ikSolver, robot, groupRef, processe
name={`arm-bot11`} name={`arm-bot11`}
/> />
</group> </group>
{/* {selectedArm && <TransformControls object={selectedArm} />} */}
</> </>
) )
} }

4
app/src/modules/simulation/roboticArm/instances/roboticArmInstances.tsx
View File

@ -1,4 +1,3 @@
import React from 'react'
import RoboticArmInstance from './armInstance/roboticArmInstance'; import RoboticArmInstance from './armInstance/roboticArmInstance';
import { useArmBotStore } from '../../../../store/simulation/useArmBotStore'; import { useArmBotStore } from '../../../../store/simulation/useArmBotStore';
@ -8,9 +7,8 @@ function RoboticArmInstances() {
return ( return (
<> <>
{armBots?.map((robot: ArmBotStatus) => ( {armBots?.map((robot: ArmBotStatus) => (
<RoboticArmInstance key={robot.modelUuid} robot={robot} /> <RoboticArmInstance key={robot.modelUuid} armBot={robot} />
))} ))}
</> </>
) )
} }

66
app/src/modules/simulation/roboticArm/roboticArm.tsx
View File

@ -2,18 +2,26 @@ import { useEffect } from "react";
import RoboticArmInstances from "./instances/roboticArmInstances"; import RoboticArmInstances from "./instances/roboticArmInstances";
import { useArmBotStore } from "../../../store/simulation/useArmBotStore"; import { useArmBotStore } from "../../../store/simulation/useArmBotStore";
import { useFloorItems } from "../../../store/store"; import { useFloorItems } from "../../../store/store";
import { useSelectedEventData, useSelectedEventSphere, useSelectedProduct } from "../../../store/simulation/useSimulationStore";
import { useProductStore } from "../../../store/simulation/useProductStore";
import { usePlayButtonStore } from "../../../store/usePlayButtonStore";
import ArmBotUI from "../ui/arm/armBotUI";
function RoboticArm() { function RoboticArm() {
const { armBots, addArmBot, removeArmBot } = useArmBotStore(); const { armBots, addArmBot, clearArmBots } = useArmBotStore();
const { products, getProductById } = useProductStore();
const { selectedProduct } = useSelectedProduct();
const { selectedEventSphere } = useSelectedEventSphere();
const { selectedEventData } = useSelectedEventData();
const { isPlaying } = usePlayButtonStore();
const { floorItems } = useFloorItems(); const { floorItems } = useFloorItems();
const armBotStatusSample: RoboticArmEventSchema[] = [ const armBotStatusSample: RoboticArmEventSchema[] = [
{ {
state: "idle", state: "idle",
modelUuid: "3abf5d46-b59e-4e6b-9c02-a4634b64b82d", modelUuid: "8790b4d5-fb6e-49e0-8161-04945fe3fdc4",
modelName: "ArmBot-X200", modelName: "ArmBot-X200",
position: [0.20849215906958463, 0, 0.32079278127773675], position: [4.317833205016363, 0, -3.2829924989068715],
rotation: [-1.3768690876192207e-15, 1.4883085074751308, 1.5407776675834467e-15], rotation: [-1.3768690876192207e-15, 1.4883085074751308, 1.5407776675834467e-15],
type: "roboticArm", type: "roboticArm",
speed: 1.5, speed: 1.5,
@ -29,19 +37,9 @@ function RoboticArm() {
process: { process: {
startPoint: [-1, 2, 1], startPoint: [-1, 2, 1],
endPoint: [-2, 1, -1], endPoint: [-2, 1, -1],
// startPoint: [-2, 1, -1],
// endPoint: [-1, 2, 1],
}, },
// process: {
// "startPoint": [
// 0.37114476008711866,
// 1.9999999999999998,
// 1.8418816116721384
// ],
// "endPoint": [
// -0.42197069459490777,
// 1,
// -3.159515927851809
// ]
// },
triggers: [ triggers: [
{ {
triggerUuid: "trigger-001", triggerUuid: "trigger-001",
@ -95,7 +93,7 @@ function RoboticArm() {
position: [95.94347308985614, 0, 6.742905194869091], position: [95.94347308985614, 0, 6.742905194869091],
rotation: [0, 0, 0], rotation: [0, 0, 0],
type: "roboticArm", type: "roboticArm",
speed: 1.5, speed: 0.1,
point: { point: {
uuid: "point-123", uuid: "point-123",
position: [0, 1.5, 0], position: [0, 1.5, 0],
@ -106,8 +104,10 @@ function RoboticArm() {
actionName: "Pick Component", actionName: "Pick Component",
actionType: "pickAndPlace", actionType: "pickAndPlace",
process: { process: {
startPoint: [2.52543010919071, 0, 8.433681161200905], // startPoint: [2.52543010919071, 0, 8.433681161200905],
endPoint: [95.3438373267953, 0, 9.0279187421610025], // endPoint: [95.3438373267953, 0, 9.0279187421610025],
startPoint: null,
endPoint: null,
}, },
triggers: [ triggers: [
{ {
@ -158,20 +158,32 @@ function RoboticArm() {
]; ];
useEffect(() => { useEffect(() => {
if (selectedProduct.productId) {
removeArmBot(armBotStatusSample[0].modelUuid); const product = getProductById(selectedProduct.productId);
addArmBot('123', armBotStatusSample[0]); if (product) {
// addArmBot('123', armBotStatusSample[1]); clearArmBots();
// addCurrentAction('armbot-xyz-001', 'action-001'); product.eventDatas.forEach(events => {
}, []); if (events.type === 'roboticArm') {
addArmBot(selectedProduct.productId, events);
}
});
}
}
}, [selectedProduct, products]);
useEffect(() => { useEffect(() => {
}, [armBots])
}, [armBots]);
useEffect(() => {
}, [selectedEventData, selectedEventSphere, isPlaying]);
return ( return (
<> <>
<RoboticArmInstances /> <RoboticArmInstances />
{selectedEventSphere && selectedEventData?.data.type === "roboticArm" &&
< ArmBotUI />
}
</> </>
); );
} }

4
app/src/modules/simulation/simulation.tsx
View File

@ -29,14 +29,14 @@ function Simulation() {
return ( return (
<> <>
<Products />
{activeModule === 'simulation' && {activeModule === 'simulation' &&
<> <>
<Points /> <Points />
<Products />
<Materials /> <Materials />
<Trigger /> <Trigger />

19
app/src/modules/simulation/ui/arm/PickDropPoints.tsx
View File

@ -9,7 +9,7 @@ interface PickDropProps {
actionType: "pick" | "drop"; actionType: "pick" | "drop";
actionUuid: string; actionUuid: string;
gltfScene: THREE.Group; gltfScene: THREE.Group;
selectedPoint: THREE.Mesh | null;
handlePointerDown: (e: ThreeEvent<PointerEvent>) => void; handlePointerDown: (e: ThreeEvent<PointerEvent>) => void;
isSelected: boolean; isSelected: boolean;
} }
@ -21,12 +21,10 @@ const PickDropPoints: React.FC<PickDropProps> = ({
actionType, actionType,
actionUuid, actionUuid,
gltfScene, gltfScene,
selectedPoint,
handlePointerDown, handlePointerDown,
isSelected, isSelected,
}) => { }) => {
const groupRef = useRef<THREE.Group>(null); const groupRef = useRef<THREE.Group>(null);
return ( return (
<group <group
ref={groupRef} ref={groupRef}
@ -36,15 +34,24 @@ const PickDropPoints: React.FC<PickDropProps> = ({
: new THREE.Vector3(0, 0, 0) : new THREE.Vector3(0, 0, 0)
} }
onPointerDown={(e) => { onPointerDown={(e) => {
e.stopPropagation(); // Important to prevent event bubbling
e.stopPropagation(); // Prevent event bubbling
if (!isSelected) return; if (!isSelected) return;
handlePointerDown(e); handlePointerDown(e);
}} }}
userData={{ modelUuid, pointUuid, actionType, actionUuid }} userData={{ modelUuid, pointUuid, actionType, actionUuid }}
> >
<primitive <primitive
object={gltfScene.clone()} object={(() => {
position={[0, 0, 0]} // Ensure this stays at origin const cloned = gltfScene.clone();
cloned.traverse((child: any) => {
if (child.isMesh) {
child.userData = { modelUuid, pointUuid, actionType, actionUuid };
}
});
return cloned;
})()}
position={[0, 0, 0]}
scale={[0.5, 0.5, 0.5]} scale={[0.5, 0.5, 0.5]}
/> />
</group> </group>

190
app/src/modules/simulation/ui/arm/armBotUI.tsx Normal file
View File

@ -0,0 +1,190 @@
import React, { useCallback, useEffect, useState } from 'react';
import { useSelectedAction, useSelectedEventData, useSelectedEventSphere, useSelectedProduct } from '../../../../store/simulation/useSimulationStore';
import { useArmBotStore } from '../../../../store/simulation/useArmBotStore';
import { useGLTF } from '@react-three/drei';
import { useThree } from '@react-three/fiber';
import { useProductStore } from '../../../../store/simulation/useProductStore';
import PickDropPoints from './PickDropPoints';
import useDraggableGLTF from './useDraggableGLTF';
import * as THREE from 'three';
import armPick from "../../../../assets/gltf-glb/arm_ui_pick.glb";
import armDrop from "../../../../assets/gltf-glb/arm_ui_drop.glb";
import useModuleStore from '../../../../store/useModuleStore';
type Positions = {
pick: [number, number, number];
drop: [number, number, number];
default: [number, number, number];
};
const ArmBotUI = () => {
const { getEventByModelUuid } = useProductStore();
const { selectedEventData } = useSelectedEventData();
const { selectedProduct } = useSelectedProduct();
const { armBots, updateStartPoint, updateEndPoint } = useArmBotStore();
const { scene } = useThree();
const { selectedAction } = useSelectedAction();
const armUiPick = useGLTF(armPick) as any;
const armUiDrop = useGLTF(armDrop) as any;
const [startPosition, setStartPosition] = useState<[number, number, number]>([0, 0, 0]);
const [endPosition, setEndPosition] = useState<[number, number, number]>([0, 0, 0]);
const [selectedArmBotData, setSelectedArmBotData] = useState<any>(null);
// Fetch and setup selected ArmBot data
useEffect(() => {
if (selectedEventData?.data.type === "roboticArm") {
const selectedArmBot = getEventByModelUuid(selectedProduct.productId, selectedEventData.data.modelUuid);
if (selectedArmBot?.type === "roboticArm") {
setSelectedArmBotData(selectedArmBot);
const defaultPositions = getDefaultPositions(selectedArmBot.modelUuid);
const matchingAction = armBots?.flatMap((robot: ArmBotStatus) => robot.point.actions)
.find((action) => action.actionUuid === selectedAction.actionId);
if (matchingAction) {
const startPoint = matchingAction.process.startPoint;
const pickPosition = (!startPoint || (Array.isArray(startPoint) && startPoint.every(v => v === 0)))
? defaultPositions.pick
: startPoint;
const endPoint = matchingAction.process.endPoint;
const dropPosition = (!endPoint || (Array.isArray(endPoint) && endPoint.every(v => v === 0)))
? defaultPositions.drop
: endPoint;
setStartPosition(pickPosition);
setEndPosition(dropPosition);
}
}
}
}, [selectedEventData, selectedProduct, getEventByModelUuid, selectedAction]);
function getDefaultPositions(modelUuid: string): Positions {
const modelData = getEventByModelUuid(selectedProduct.productId, modelUuid);
if (modelData?.type === "roboticArm") {
const baseX = modelData.point.position?.[0] || 0;
const baseY = modelData.point.position?.[1] || 0;;
const baseZ = modelData.point.position?.[2] || 0;
return {
pick: [baseX, baseY, baseZ + 0.5],
drop: [baseX, baseY, baseZ - 0.5],
default: [baseX, baseY, baseZ],
};
}
return {
pick: [0.5, 1.5, 0],
drop: [-0.5, 1.5, 0],
default: [0, 1.5, 0],
};
}
function getLocalPosition(parentUuid: string, worldPosArray: [number, number, number] | null): [number, number, number] | null {
if (worldPosArray) {
const worldPos = new THREE.Vector3(...worldPosArray);
const parentObject = scene.getObjectByProperty('uuid', parentUuid);
if (parentObject) {
const localPos = worldPos.clone();
parentObject.worldToLocal(localPos);
return [localPos.x, localPos.y, localPos.z];
}
}
return null;
}
const updatePointToState = (obj: THREE.Object3D) => {
const { modelUuid, actionType, actionUuid } = obj.userData;
const newPosition = new THREE.Vector3();
obj.getWorldPosition(newPosition);
const worldPositionArray = newPosition.toArray() as [number, number, number];
if (selectedEventData?.data.type === "roboticArm") {
const selectedArmBot = getEventByModelUuid(selectedProduct.productId, selectedEventData.data.modelUuid);
const armBot = selectedArmBot?.modelUuid === modelUuid ? selectedArmBot : null;
if (!armBot) return;
if (armBot.type === "roboticArm") {
armBot?.point?.actions?.map((action: any) => {
if (action.actionUuid === actionUuid) {
const updatedProcess = { ...action.process };
if (actionType === "pick") {
updatedProcess.startPoint = getLocalPosition(modelUuid, worldPositionArray);
setStartPosition(updatedProcess.startPoint)
updateStartPoint(modelUuid, actionUuid, updatedProcess.startPoint);
} else if (actionType === "drop") {
updatedProcess.endPoint = getLocalPosition(modelUuid, worldPositionArray);
setEndPosition(updatedProcess.endPoint)
updateEndPoint(modelUuid, actionUuid, updatedProcess.endPoint);
}
return {
...action,
process: updatedProcess,
};
}
return action;
});
}
}
}
useEffect(() => {
}, [armBots])
const { handlePointerDown } = useDraggableGLTF(updatePointToState);
if (!selectedArmBotData || !Array.isArray(selectedArmBotData.point?.actions)) {
return null; // avoid rendering if no data yet
}
return (
<>
{selectedArmBotData.point.actions.map((action: any) => {
if (action.actionUuid === selectedAction.actionId) {
return (
<React.Fragment key={action.actionUuid}>
<group
position={new THREE.Vector3(...selectedArmBotData.position)}
rotation={new THREE.Euler(...selectedArmBotData.rotation)}
>
<PickDropPoints
position={startPosition}
modelUuid={selectedArmBotData.modelUuid}
pointUuid={selectedArmBotData.point.uuid}
actionType="pick"
actionUuid={action.actionUuid}
gltfScene={armUiPick.scene}
handlePointerDown={handlePointerDown}
isSelected={true}
/>
<PickDropPoints
position={endPosition}
modelUuid={selectedArmBotData.modelUuid}
pointUuid={selectedArmBotData.point.uuid}
actionType="drop"
actionUuid={action.actionUuid}
gltfScene={armUiDrop.scene}
handlePointerDown={handlePointerDown}
isSelected={true}
/>
</group>
</React.Fragment>
);
} else {
return null; // important! must return something
}
})}
</>
);
};
export default ArmBotUI;

56
app/src/modules/simulation/ui/arm/useDraggableGLTF.ts
View File

@ -1,11 +1,11 @@
import { useRef } from "react"; import { useRef, useState } from "react";
import * as THREE from "three"; import * as THREE from "three";
import { ThreeEvent, useThree } from "@react-three/fiber"; import { ThreeEvent, useThree } from "@react-three/fiber";
type OnUpdateCallback = (object: THREE.Object3D) => void; type OnUpdateCallback = (object: THREE.Object3D) => void;
export default function useDraggableGLTF(onUpdate: OnUpdateCallback) { export default function useDraggableGLTF(onUpdate: OnUpdateCallback) {
const { camera, gl, controls, scene } = useThree(); const { camera, gl, controls } = useThree();
const activeObjRef = useRef<THREE.Object3D | null>(null); const activeObjRef = useRef<THREE.Object3D | null>(null);
const planeRef = useRef<THREE.Plane>( const planeRef = useRef<THREE.Plane>(
new THREE.Plane(new THREE.Vector3(0, 1, 0), 0) new THREE.Plane(new THREE.Vector3(0, 1, 0), 0)
@ -15,6 +15,7 @@ export default function useDraggableGLTF(onUpdate: OnUpdateCallback) {
const raycaster = new THREE.Raycaster(); const raycaster = new THREE.Raycaster();
const pointer = new THREE.Vector2(); const pointer = new THREE.Vector2();
const [objectWorldPos, setObjectWorldPos] = useState(new THREE.Vector3());
const handlePointerDown = (e: ThreeEvent<PointerEvent>) => { const handlePointerDown = (e: ThreeEvent<PointerEvent>) => {
e.stopPropagation(); e.stopPropagation();
@ -33,10 +34,10 @@ export default function useDraggableGLTF(onUpdate: OnUpdateCallback) {
activeObjRef.current = obj; activeObjRef.current = obj;
initialPositionRef.current.copy(obj.position); initialPositionRef.current.copy(obj.position);
// Get world position // Get world position
const objectWorldPos = new THREE.Vector3(); setObjectWorldPos(obj.getWorldPosition(objectWorldPos));
obj.getWorldPosition(objectWorldPos);
// Set plane at the object's Y level // Set plane at the object's Y level
planeRef.current.set(new THREE.Vector3(0, 1, 0), -objectWorldPos.y); planeRef.current.set(new THREE.Vector3(0, 1, 0), -objectWorldPos.y);
@ -61,52 +62,56 @@ export default function useDraggableGLTF(onUpdate: OnUpdateCallback) {
const handlePointerMove = (e: PointerEvent) => { const handlePointerMove = (e: PointerEvent) => {
if (!activeObjRef.current) return; if (!activeObjRef.current) return;
// Check if Shift key is pressed // Check if Shift key is pressed
const isShiftKeyPressed = e.shiftKey; const isShiftKeyPressed = e.shiftKey;
// Get the mouse position relative to the canvas // Get the mouse position relative to the canvas
const rect = gl.domElement.getBoundingClientRect(); const rect = gl.domElement.getBoundingClientRect();
pointer.x = ((e.clientX - rect.left) / rect.width) * 2 - 1; pointer.x = ((e.clientX - rect.left) / rect.width) * 2 - 1;
pointer.y = -((e.clientY - rect.top) / rect.height) * 2 + 1; pointer.y = -((e.clientY - rect.top) / rect.height) * 2 + 1;
// Update raycaster to point to the mouse position // Update raycaster to point to the mouse position
raycaster.setFromCamera(pointer, camera); raycaster.setFromCamera(pointer, camera);
// Create a vector to store intersection point // Create a vector to store intersection point
const intersection = new THREE.Vector3(); const intersection = new THREE.Vector3();
const intersects = raycaster.ray.intersectPlane(planeRef.current, intersection); const intersects = raycaster.ray.intersectPlane(
planeRef.current,
intersection
);
if (!intersects) return; if (!intersects) return;
// Add offset for dragging // Add offset for dragging
intersection.add(offsetRef.current); intersection.add(offsetRef.current);
console.log('intersection: ', intersection);
// Get the parent's world matrix if exists // Get the parent's world matrix if exists
const parent = activeObjRef.current.parent; const parent = activeObjRef.current.parent;
const targetPosition = new THREE.Vector3(); const targetPosition = new THREE.Vector3();
// OnPointerDown
initialPositionRef.current.copy(objectWorldPos);
// OnPointerMove
if (isShiftKeyPressed) { if (isShiftKeyPressed) {
console.log('isShiftKeyPressed: ', isShiftKeyPressed); const { x: initialX, y: initialY } = initialPositionRef.current;
// For Y-axis only movement, maintain original X and Z const { x: objectX, z: objectZ } = objectWorldPos;
console.log('initialPositionRef: ', initialPositionRef);
console.log('intersection.y: ', intersection); const deltaX = intersection.x - initialX;
targetPosition.set(
initialPositionRef.current.x, targetPosition.set(objectX, initialY + deltaX, objectZ);
intersection.y,
initialPositionRef.current.z
);
} else { } else {
// For free movement // For free movement
targetPosition.copy(intersection); targetPosition.copy(intersection);
} }
// Convert world position to local if object is nested inside a parent // Convert world position to local if object is nested inside a parent
if (parent) { if (parent) {
parent.worldToLocal(targetPosition); parent.worldToLocal(targetPosition);
} }
// Update object position // Update object position
activeObjRef.current.position.copy(targetPosition); activeObjRef.current.position.copy(targetPosition);
}; };
@ -126,6 +131,3 @@ export default function useDraggableGLTF(onUpdate: OnUpdateCallback) {
return { handlePointerDown }; return { handlePointerDown };
} }

2
app/src/modules/simulation/vehicle/vehicles.tsx
View File

@ -29,7 +29,7 @@ function Vehicles() {
}, [selectedProduct, products]); }, [selectedProduct, products]);
useEffect(() => { useEffect(() => {
// console.log('vehicles: ', vehicles); //
}, [vehicles]) }, [vehicles])
return ( return (