From 1e715cee504e12d1b89a10ad756b9779bc007c37 Mon Sep 17 00:00:00 2001 From: Jerald-Golden-B Date: Thu, 3 Jul 2025 16:55:30 +0530 Subject: [PATCH] feat: Enhance human event handling and animation management, including state updates and monitoring --- .../builder/asset/functions/addAssetModel.ts | 1 - .../builder/asset/models/model/model.tsx | 55 +++++++-- .../actions/human/useHumanActions.ts | 6 +- .../eventManager/useHumanEventManager.ts | 79 +++++++++++++ .../instances/instance/humanInstance.tsx | 107 +++++++++++++++++- .../checkActiveRoboticArmsInSubsequence.ts | 3 +- .../determineExecutionMachineSequences.ts | 3 +- .../functions/determineExecutionOrder.ts | 4 +- .../functions/determineExecutionSequences.ts | 4 +- .../getConveyorSequencesInProduct.ts | 3 +- .../triggerHandler/useTriggerHandler.ts | 56 ++++++++- .../instances/instance/vehicleInstance.tsx | 18 +-- app/src/store/simulation/useHumanStore.ts | 13 +++ 13 files changed, 316 insertions(+), 36 deletions(-) create mode 100644 app/src/modules/simulation/human/eventManager/useHumanEventManager.ts diff --git a/app/src/modules/builder/asset/functions/addAssetModel.ts b/app/src/modules/builder/asset/functions/addAssetModel.ts index f7614ee..43c4b8a 100644 --- a/app/src/modules/builder/asset/functions/addAssetModel.ts +++ b/app/src/modules/builder/asset/functions/addAssetModel.ts @@ -162,7 +162,6 @@ async function handleModelLoad( // SOCKET if (selectedItem.type) { - console.log('selectedItem: ', selectedItem); const data = PointsCalculator( selectedItem.type, gltf.scene.clone(), diff --git a/app/src/modules/builder/asset/models/model/model.tsx b/app/src/modules/builder/asset/models/model/model.tsx index 3af4bde..ba47e9e 100644 --- a/app/src/modules/builder/asset/models/model/model.tsx +++ b/app/src/modules/builder/asset/models/model/model.tsx @@ -49,6 +49,10 @@ function Model({ asset }: { readonly asset: Asset }) { const { userId, organization } = getUserData(); const mixerRef = useRef(); const actions = useRef<{ [name: string]: THREE.AnimationAction }>({}); + const [currentAnimation, setCurrentAnimation] = useState(null); + const [previousAnimation, setPreviousAnimation] = useState(null); + const [blendFactor, setBlendFactor] = useState(0); + const blendDuration = 0.3; useEffect(() => { setDeletableFloorItem(null); @@ -278,8 +282,16 @@ function Model({ asset }: { readonly asset: Asset }) { } } + const handleAnimationComplete = useCallback(() => { + console.log(`Animation "${currentAnimation}" completed`); + }, [currentAnimation]); + useFrame((_, delta) => { if (mixerRef.current) { + if (blendFactor < 1) { + setBlendFactor(prev => Math.min(prev + delta / blendDuration, 1)); + } + mixerRef.current.update(delta); } }); @@ -288,17 +300,46 @@ function Model({ asset }: { readonly asset: Asset }) { if (asset.animationState && asset.animationState.isPlaying) { if (!mixerRef.current) return; - Object.values(actions.current).forEach((action) => action.stop()); - - const action = actions.current[asset.animationState.current]; - if (action && asset.animationState?.isPlaying) { - const loopMode = asset.animationState.loopAnimation ? THREE.LoopRepeat : THREE.LoopOnce; - action.reset().setLoop(loopMode, loopMode === THREE.LoopRepeat ? Infinity : 1).play(); + if (asset.animationState.current !== currentAnimation) { + setPreviousAnimation(currentAnimation); + setCurrentAnimation(asset.animationState.current); + setBlendFactor(0); } + + const currentAction = actions.current[asset.animationState.current]; + const previousAction = previousAnimation ? actions.current[previousAnimation] : null; + + if (currentAction) { + const loopMode = asset.animationState.loopAnimation ? THREE.LoopRepeat : THREE.LoopOnce; + + currentAction.reset(); + currentAction.setLoop(loopMode, loopMode === THREE.LoopRepeat ? Infinity : 1); + + currentAction.play(); + + mixerRef.current.addEventListener('finished', handleAnimationComplete); + + if (previousAction && blendFactor < 1) { + previousAction.crossFadeTo(currentAction, blendDuration, true); + } + } + + Object.entries(actions.current).forEach(([name, action]) => { + if ((asset.animationState && name !== asset.animationState.current) && name !== previousAnimation) { + action.stop(); + } + }); } else { Object.values(actions.current).forEach((action) => action.stop()); + setCurrentAnimation(null); } - }, [asset.animationState]) + + return () => { + if (mixerRef.current) { + mixerRef.current.removeEventListener('finished', handleAnimationComplete); + } + } + }, [asset.animationState, currentAnimation, previousAnimation, handleAnimationComplete]); return ( { - handleWorker(action); + const handleWorkerAction = useCallback((action: HumanAction, materialId: string) => { + handleWorker(action, materialId); }, [handleWorker]); const handleHumanAction = useCallback((action: HumanAction, materialId: string) => { @@ -13,7 +13,7 @@ export function useHumanActions() { switch (action.actionType) { case 'worker': - handleWorkerAction(action); + handleWorkerAction(action, materialId); break; default: console.warn(`Unknown Human action type: ${action.actionType}`); diff --git a/app/src/modules/simulation/human/eventManager/useHumanEventManager.ts b/app/src/modules/simulation/human/eventManager/useHumanEventManager.ts new file mode 100644 index 0000000..1ab16f7 --- /dev/null +++ b/app/src/modules/simulation/human/eventManager/useHumanEventManager.ts @@ -0,0 +1,79 @@ +import { useEffect, useRef } from 'react'; +import { useFrame } from '@react-three/fiber'; +import { usePauseButtonStore, usePlayButtonStore, useResetButtonStore } from '../../../../store/usePlayButtonStore'; +import { useSceneContext } from '../../../scene/sceneContext'; + +type HumanCallback = { + humanId: string; + actionId: string; + callback: () => void; +}; + +export function useHumanEventManager() { + const { humanStore } = useSceneContext(); + const { getHumanById } = humanStore(); + const callbacksRef = useRef([]); + const isMonitoringRef = useRef(false); + const { isPlaying } = usePlayButtonStore(); + const { isPaused } = usePauseButtonStore(); + const { isReset } = useResetButtonStore(); + + useEffect(() => { + if (isReset) { + callbacksRef.current = []; + } + }, [isReset]) + + // Add a new human to monitor + const addHumanToMonitor = (humanId: string, actionId: string, callback: () => void) => { + // Avoid duplicates + if (!callbacksRef.current.some((entry) => entry.humanId === humanId)) { + callbacksRef.current.push({ humanId, actionId, callback }); + } + + // Start monitoring if not already running + if (!isMonitoringRef.current) { + isMonitoringRef.current = true; + } + }; + + // Remove a human from monitoring + const removeHumanFromMonitor = (humanId: string) => { + callbacksRef.current = callbacksRef.current.filter( + (entry) => entry.humanId !== humanId + ); + + // Stop monitoring if no more humans to track + if (callbacksRef.current.length === 0) { + isMonitoringRef.current = false; + } + }; + + // Check human states every frame + useFrame(() => { + if (!isMonitoringRef.current || callbacksRef.current.length === 0 || !isPlaying || isPaused) return; + + callbacksRef.current.forEach(({ humanId, actionId, callback }) => { + const human = getHumanById(humanId); + if (!human) return; + const action = human.point.actions.find((action) => action.actionUuid === actionId); + if (action && human.isActive === false && human.state === 'idle' && human.isPicking && human.currentLoad < action.loadCapacity) { + callback(); + removeHumanFromMonitor(humanId); // Remove after triggering + } + }); + }); + + // Cleanup on unmount + useEffect(() => { + return () => { + callbacksRef.current = []; + isMonitoringRef.current = false; + }; + }, []); + + return { + addHumanToMonitor, + removeHumanFromMonitor, + }; +} \ No newline at end of file diff --git a/app/src/modules/simulation/human/instances/instance/humanInstance.tsx b/app/src/modules/simulation/human/instances/instance/humanInstance.tsx index 25fb044..6e1ebc0 100644 --- a/app/src/modules/simulation/human/instances/instance/humanInstance.tsx +++ b/app/src/modules/simulation/human/instances/instance/humanInstance.tsx @@ -1,16 +1,113 @@ -import { useEffect } from 'react' -import HumanUi from './humanUi'; +import { useCallback, useEffect, useRef, useState } from 'react'; +import * as THREE from 'three'; +import { NavMeshQuery } from '@recast-navigation/core'; +import { useNavMesh } from '../../../../../store/builder/store'; +import { useAnimationPlaySpeed, usePauseButtonStore, usePlayButtonStore } from '../../../../../store/usePlayButtonStore'; +import { useTriggerHandler } from '../../../triggers/triggerHandler/useTriggerHandler'; +import { useSceneContext } from '../../../../scene/sceneContext'; +import { useProductContext } from '../../../products/productContext'; + +import HumanAnimator from './animator/humanAnimator'; function HumanInstance({ human }: { human: HumanStatus }) { + const { navMesh } = useNavMesh(); + const { isPlaying } = usePlayButtonStore(); + const { materialStore, armBotStore, conveyorStore, vehicleStore, humanStore, storageUnitStore, productStore } = useSceneContext(); + const { removeMaterial, setEndTime } = materialStore(); + const { getStorageUnitById } = storageUnitStore(); + const { getArmBotById } = armBotStore(); + const { getConveyorById } = conveyorStore(); + const { getVehicleById } = vehicleStore(); + const { triggerPointActions } = useTriggerHandler(); + const { getActionByUuid, getEventByModelUuid, getTriggerByUuid } = productStore(); + const { selectedProductStore } = useProductContext(); + const { selectedProduct } = selectedProductStore(); + const { humans, setHumanActive, setHumanState, setHumanPicking, clearCurrentMaterials, setHumanLoad, decrementHumanLoad, removeLastMaterial, getLastMaterial, incrementIdleTime, incrementActiveTime, resetTime } = humanStore(); + + const [currentPhase, setCurrentPhase] = useState('init'); + const [path, setPath] = useState<[number, number, number][]>([]); + const pauseTimeRef = useRef(null); + const idleTimeRef = useRef(0); + const activeTimeRef = useRef(0); + const isPausedRef = useRef(false); + const isSpeedRef = useRef(0); + let startTime: number; + let fixedInterval: number; + const { speed } = useAnimationPlaySpeed(); + const { isPaused } = usePauseButtonStore(); + const previousTimeRef = useRef(null); + const animationFrameIdRef = useRef(null); useEffect(() => { - console.log('human: ', human); - }, [human]) + isPausedRef.current = isPaused; + }, [isPaused]); + + useEffect(() => { + isSpeedRef.current = speed; + }, [speed]); + + const computePath = useCallback( + (start: any, end: any) => { + try { + const navMeshQuery = new NavMeshQuery(navMesh); + const { path: segmentPath } = navMeshQuery.computePath(start, end); + if ( + segmentPath.length > 0 && + Math.round(segmentPath[segmentPath.length - 1].x) == Math.round(end.x) && + Math.round(segmentPath[segmentPath.length - 1].z) == Math.round(end.z) + ) { + return segmentPath?.map(({ x, y, z }) => [x, 0, z] as [number, number, number]) || []; + } else { + console.log("There is no path here...Choose valid path") + const { path: segmentPaths } = navMeshQuery.computePath(start, start); + return segmentPaths.map(({ x, y, z }) => [x, 0, z] as [number, number, number]) || []; + } + } catch { + console.error("Failed to compute path"); + return []; + } + }, + [navMesh] + ); + + function humanStatus(modelId: string, status: string) { + // console.log(`${modelId} , ${status}`); + } + + function reset() { + setCurrentPhase('init'); + setHumanActive(human.modelUuid, false); + setHumanPicking(human.modelUuid, false); + setHumanState(human.modelUuid, 'idle'); + setHumanLoad(human.modelUuid, 0); + setPath([]); + startTime = 0; + isPausedRef.current = false; + pauseTimeRef.current = 0; + resetTime(human.modelUuid) + activeTimeRef.current = 0 + idleTimeRef.current = 0 + previousTimeRef.current = null + if (animationFrameIdRef.current !== null) { + cancelAnimationFrame(animationFrameIdRef.current) + animationFrameIdRef.current = null + } + } + + useEffect(() => { + if (isPlaying) { + + } + else { + reset() + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [human, currentPhase, path, isPlaying]); return ( <> - + ) diff --git a/app/src/modules/simulation/simulator/functions/checkActiveRoboticArmsInSubsequence.ts b/app/src/modules/simulation/simulator/functions/checkActiveRoboticArmsInSubsequence.ts index 8275fb3..58925f3 100644 --- a/app/src/modules/simulation/simulator/functions/checkActiveRoboticArmsInSubsequence.ts +++ b/app/src/modules/simulation/simulator/functions/checkActiveRoboticArmsInSubsequence.ts @@ -124,7 +124,8 @@ function determineExecutionMachineSequences(products: productsSchema): EventsSch event.type === 'vehicle' || event.type === 'machine' || event.type === 'storageUnit' || - event.type === 'roboticArm' + event.type === 'roboticArm' || + event.type === 'human' ) { pointToEventMap.set(event.point.uuid, event); allPoints.push(event.point); diff --git a/app/src/modules/simulation/simulator/functions/determineExecutionMachineSequences.ts b/app/src/modules/simulation/simulator/functions/determineExecutionMachineSequences.ts index 7c2f7ee..a87fe6a 100644 --- a/app/src/modules/simulation/simulator/functions/determineExecutionMachineSequences.ts +++ b/app/src/modules/simulation/simulator/functions/determineExecutionMachineSequences.ts @@ -16,7 +16,8 @@ export async function determineExecutionMachineSequences(products: productsSchem event.type === 'vehicle' || event.type === 'machine' || event.type === 'storageUnit' || - event.type === 'roboticArm' + event.type === 'roboticArm' || + event.type === 'human' ) { pointToEventMap.set(event.point.uuid, event); allPoints.push(event.point); diff --git a/app/src/modules/simulation/simulator/functions/determineExecutionOrder.ts b/app/src/modules/simulation/simulator/functions/determineExecutionOrder.ts index 1f29ab3..d17e2d7 100644 --- a/app/src/modules/simulation/simulator/functions/determineExecutionOrder.ts +++ b/app/src/modules/simulation/simulator/functions/determineExecutionOrder.ts @@ -19,7 +19,9 @@ export function determineExecutionOrder(products: productsSchema): PointsScheme[ } else if (event.type === 'vehicle' || event.type === 'machine' || event.type === 'storageUnit' || - event.type === 'roboticArm') { + event.type === 'roboticArm' || + event.type === 'human' + ) { pointMap.set(event.point.uuid, event.point); allPoints.push(event.point); } diff --git a/app/src/modules/simulation/simulator/functions/determineExecutionSequences.ts b/app/src/modules/simulation/simulator/functions/determineExecutionSequences.ts index bb88fb5..3ba5eb2 100644 --- a/app/src/modules/simulation/simulator/functions/determineExecutionSequences.ts +++ b/app/src/modules/simulation/simulator/functions/determineExecutionSequences.ts @@ -16,7 +16,9 @@ export async function determineExecutionSequences(products: productsSchema): Pro } else if (event.type === 'vehicle' || event.type === 'machine' || event.type === 'storageUnit' || - event.type === 'roboticArm') { + event.type === 'roboticArm' || + event.type === 'human' + ) { pointMap.set(event.point.uuid, event.point); allPoints.push(event.point); } diff --git a/app/src/modules/simulation/simulator/functions/getConveyorSequencesInProduct.ts b/app/src/modules/simulation/simulator/functions/getConveyorSequencesInProduct.ts index 8ad5e10..8f812bf 100644 --- a/app/src/modules/simulation/simulator/functions/getConveyorSequencesInProduct.ts +++ b/app/src/modules/simulation/simulator/functions/getConveyorSequencesInProduct.ts @@ -91,7 +91,8 @@ function determineExecutionMachineSequences(products: productsSchema): EventsSch event.type === 'vehicle' || event.type === 'machine' || event.type === 'storageUnit' || - event.type === 'roboticArm' + event.type === 'roboticArm' || + event.type === 'human' ) { pointToEventMap.set(event.point.uuid, event); allPoints.push(event.point); diff --git a/app/src/modules/simulation/triggers/triggerHandler/useTriggerHandler.ts b/app/src/modules/simulation/triggers/triggerHandler/useTriggerHandler.ts index 5731d9e..9af9d78 100644 --- a/app/src/modules/simulation/triggers/triggerHandler/useTriggerHandler.ts +++ b/app/src/modules/simulation/triggers/triggerHandler/useTriggerHandler.ts @@ -6,9 +6,10 @@ import { useVehicleEventManager } from '../../vehicle/eventManager/useVehicleEve import { useMachineEventManager } from '../../machine/eventManager/useMachineEventManager'; import { useSceneContext } from '../../../scene/sceneContext'; import { useProductContext } from '../../products/productContext'; +import { useHumanEventManager } from '../../human/eventManager/useHumanEventManager'; export function useTriggerHandler() { - const { materialStore, armBotStore, machineStore, conveyorStore, vehicleStore, storageUnitStore, productStore } = useSceneContext(); + const { materialStore, armBotStore, machineStore, conveyorStore, vehicleStore, humanStore, storageUnitStore, productStore } = useSceneContext(); const { selectedProductStore } = useProductContext(); const { handleAction } = useActionHandler(); const { selectedProduct } = selectedProductStore(); @@ -19,7 +20,9 @@ export function useTriggerHandler() { const { addConveyorToMonitor } = useConveyorEventManager(); const { addVehicleToMonitor } = useVehicleEventManager(); const { addMachineToMonitor } = useMachineEventManager(); + const { addHumanToMonitor } = useHumanEventManager(); const { getVehicleById } = vehicleStore(); + const { getHumanById } = humanStore(); const { getMachineById } = machineStore(); const { getStorageUnitById } = storageUnitStore(); const { getMaterialById, setCurrentLocation, setNextLocation, setPreviousLocation, setIsPaused, setIsVisible, setEndTime } = materialStore(); @@ -256,6 +259,57 @@ export function useTriggerHandler() { } else if (toEvent?.type === 'storageUnit') { // Transfer to Storage Unit + } else if (toEvent?.type === 'human') { + // Transfer to Human + if (materialId && trigger.triggeredAsset && trigger.triggeredAsset.triggeredPoint && trigger.triggeredAsset.triggeredAction) { + const material = getMaterialById(materialId); + if (material) { + const triggeredAction = action; + + // Handle current action of the material + handleAction(action, materialId); + + if (material.next) { + const action = getActionByUuid(selectedProduct.productUuid, trigger.triggeredAsset.triggeredAction.actionUuid); + const human = getHumanById(trigger.triggeredAsset?.triggeredModel.modelUuid); + + setPreviousLocation(material.materialId, { + modelUuid: material.current.modelUuid, + pointUuid: material.current.pointUuid, + actionUuid: material.current.actionUuid, + }) + + setCurrentLocation(material.materialId, { + modelUuid: material.next.modelUuid, + pointUuid: material.next.pointUuid, + actionUuid: trigger.triggeredAsset?.triggeredAction?.actionUuid, + }); + + setNextLocation(material.materialId, null); + + if (action) { + + if (human) { + + if (human.isActive === false && human.state === 'idle' && human.isPicking && human.currentLoad < (triggeredAction as HumanAction).loadCapacity) { + + setIsVisible(materialId, false); + + // Handle current action from vehicle + handleAction(action, materialId); + + } else { + + // Handle current action using Event Manager + addHumanToMonitor(human.modelUuid, triggeredAction.actionUuid, () => { + handleAction(action, materialId); + }) + } + } + } + } + } + } } } else if (fromEvent?.type === 'vehicle') { if (toEvent?.type === 'transfer') { diff --git a/app/src/modules/simulation/vehicle/instances/instance/vehicleInstance.tsx b/app/src/modules/simulation/vehicle/instances/instance/vehicleInstance.tsx index 0709109..782d4a0 100644 --- a/app/src/modules/simulation/vehicle/instances/instance/vehicleInstance.tsx +++ b/app/src/modules/simulation/vehicle/instances/instance/vehicleInstance.tsx @@ -1,14 +1,15 @@ import { useCallback, useEffect, useRef, useState } from 'react'; -import VehicleAnimator from '../animator/vehicleAnimator'; import * as THREE from 'three'; import { NavMeshQuery } from '@recast-navigation/core'; import { useNavMesh } from '../../../../../store/builder/store'; import { useAnimationPlaySpeed, usePauseButtonStore, usePlayButtonStore } from '../../../../../store/usePlayButtonStore'; import { useTriggerHandler } from '../../../triggers/triggerHandler/useTriggerHandler'; -import MaterialAnimator from '../animator/materialAnimator'; import { useSceneContext } from '../../../../scene/sceneContext'; import { useProductContext } from '../../../products/productContext'; +import MaterialAnimator from '../animator/materialAnimator'; +import VehicleAnimator from '../animator/vehicleAnimator'; + function VehicleInstance({ agvDetail }: Readonly<{ agvDetail: VehicleStatus }>) { const { navMesh } = useNavMesh(); const { isPlaying } = usePlayButtonStore(); @@ -103,10 +104,6 @@ function VehicleInstance({ agvDetail }: Readonly<{ agvDetail: VehicleStatus }>) new THREE.Vector3(agvDetail?.position[0], agvDetail?.position[1], agvDetail?.position[2]), agvDetail?.point?.action?.pickUpPoint?.position ); - // const toPickupPath = computePath( - // new THREE.Vector3(agvDetail?.position[0], agvDetail?.position[1], agvDetail?.position[2]), - // new THREE.Vector3(agvDetail?.position[0], agvDetail?.position[1], agvDetail?.position[2]) - // ); setPath(toPickupPath); setCurrentPhase('stationed-pickup'); setVehicleState(agvDetail.modelUuid, 'running'); @@ -150,7 +147,6 @@ function VehicleInstance({ agvDetail }: Readonly<{ agvDetail: VehicleStatus }>) // eslint-disable-next-line react-hooks/exhaustive-deps }, [vehicles, currentPhase, path, isPlaying]); - function animate(currentTime: number) { if (previousTimeRef.current === null) { previousTimeRef.current = currentTime; @@ -527,10 +523,4 @@ function VehicleInstance({ agvDetail }: Readonly<{ agvDetail: VehicleStatus }>) ); } -export default VehicleInstance; - - - - - - +export default VehicleInstance; \ No newline at end of file diff --git a/app/src/store/simulation/useHumanStore.ts b/app/src/store/simulation/useHumanStore.ts index a7ed7d5..0c6ebd9 100644 --- a/app/src/store/simulation/useHumanStore.ts +++ b/app/src/store/simulation/useHumanStore.ts @@ -15,6 +15,10 @@ interface HumansStore { setHumanActive: (modelUuid: string, isActive: boolean) => void; setHumanPicking: (modelUuid: string, isPicking: boolean) => void; setHumanLoad: (modelUuid: string, load: number) => void; + setHumanState: ( + modelUuid: string, + newState: HumanStatus["state"] + ) => void; incrementHumanLoad: (modelUuid: string, incrementBy: number) => void; decrementHumanLoad: (modelUuid: string, decrementBy: number) => void; @@ -106,6 +110,15 @@ export const createHumanStore = () => { }); }, + setHumanState: (modelUuid, newState) => { + set((state) => { + const human = state.humans.find(h => h.modelUuid === modelUuid); + if (human) { + human.state = newState; + } + }); + }, + incrementHumanLoad: (modelUuid, incrementBy) => { set((state) => { const human = state.humans.find(h => h.modelUuid === modelUuid);