simultaion

server/src/features/weights/domain/exec_weights_process_scenario.ts

@@ -32,7 +32,6 @@ export class ExecWeightProcessScenario extends CallbackStrategyWithIdQuery {
           if (preTrain) {
             execCommand += ` --pretrain`;
           }
-          console.log(execCommand);
           return await new ExecProcessUseCase().call(
             `${model.project.rootDir}/`,
             execCommand,

ui/package-lock.json

@@ -8,6 +8,10 @@
       "name": "ui-robossembler",
       "version": "0.1.0",
       "dependencies": {
+        "@foxglove/cdr": "^3.3.0",
+        "@foxglove/rosmsg": "^5.0.4",
+        "@foxglove/rosmsg2-serialization": "^2.0.3",
+        "@foxglove/ws-protocol": "^0.7.3",
         "@monaco-editor/react": "^4.6.0",
         "@testing-library/jest-dom": "^5.17.0",
         "@testing-library/react": "^13.4.0",
@@ -53,6 +57,7 @@
         "urdf-loader": "^0.12.1",
         "uuid": "^9.0.1",
         "web-vitals": "^2.1.4",
+        "ws": "^8.17.0",
         "xml-formatter": "^3.6.2"
       },
       "devDependencies": {
@@ -2625,6 +2630,72 @@
         "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
       }
     },
+    "node_modules/@foxglove/cdr": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/@foxglove/cdr/-/cdr-3.3.0.tgz",
+      "integrity": "sha512-CjeA6ka/0cddVzQZY0qTEbExw849C7dD1aGpa+KeEzx0LZNxsQvvhsqWeGjszVM4BzwPcyWHwsiOZv27zbnnFA==",
+      "engines": {
+        "node": ">= 14"
+      }
+    },
+    "node_modules/@foxglove/message-definition": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/@foxglove/message-definition/-/message-definition-0.2.0.tgz",
+      "integrity": "sha512-IQHIGCvBZR8GIua9nEpS+hsMF3gm1bfbrrnjG0rgtcFBWiNuKbzx4vIP8OIwDC+8wtwcFdfJhf4Vp5TPFiUUcQ=="
+    },
+    "node_modules/@foxglove/rosmsg": {
+      "version": "5.0.4",
+      "resolved": "https://registry.npmjs.org/@foxglove/rosmsg/-/rosmsg-5.0.4.tgz",
+      "integrity": "sha512-s0JQLA6Zi0Qh9HtzyAan30sjiMU0u34+fkwccek/Xt+aQjCkSoErP/U4NBP6hCDfp0fBcb7CDSU2ggLFy/rJAQ==",
+      "dependencies": {
+        "@foxglove/message-definition": "^0.3.1",
+        "md5-typescript": "^1.0.5"
+      },
+      "engines": {
+        "node": ">= 14"
+      }
+    },
+    "node_modules/@foxglove/rosmsg/node_modules/@foxglove/message-definition": {
+      "version": "0.3.1",
+      "resolved": "https://registry.npmjs.org/@foxglove/message-definition/-/message-definition-0.3.1.tgz",
+      "integrity": "sha512-nkPowiED67LjcKEC77CprkUG3XvSsFHHR9HEwWCuhnIC2wm0W57T1J+WWvteoArZ7SdGGlKzSYSRFyjQkgmITw=="
+    },
+    "node_modules/@foxglove/rosmsg2-serialization": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/@foxglove/rosmsg2-serialization/-/rosmsg2-serialization-2.0.3.tgz",
+      "integrity": "sha512-atNKdzO8aYJLFbqd8OblUo1V5faWSmM2mlKkDfA/4wI65phGiq+SJOfElbj5nNNtRGgC60BkdIxcRyMQgxym2g==",
+      "dependencies": {
+        "@foxglove/cdr": "^3.0.0",
+        "@foxglove/message-definition": "^0.2.0",
+        "@foxglove/rostime": "^1.1.2"
+      },
+      "engines": {
+        "node": ">= 14"
+      }
+    },
+    "node_modules/@foxglove/rostime": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@foxglove/rostime/-/rostime-1.1.2.tgz",
+      "integrity": "sha512-vWuTJCuGv0xvgwOlrZ1y2MevmNMVxWcUU/HwlmYXi/jUq/kRaACStU18uyuZ3LzdNKaffkti0rTcXWESaQjwQw==",
+      "engines": {
+        "node": ">= 14"
+      }
+    },
+    "node_modules/@foxglove/ws-protocol": {
+      "version": "0.7.3",
+      "resolved": "https://registry.npmjs.org/@foxglove/ws-protocol/-/ws-protocol-0.7.3.tgz",
+      "integrity": "sha512-tgZpg1eT5Gm/C9Qe+BMdfH6P5WoSv/dHTcU61LJr36zqFTNfJ6hjZ+FWba5PLz/yoJd0/xgYUJatzamm43yeWA==",
+      "dependencies": {
+        "debug": "^4",
+        "eventemitter3": "^5.0.1",
+        "tslib": "^2.6.2"
+      }
+    },
+    "node_modules/@foxglove/ws-protocol/node_modules/eventemitter3": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz",
+      "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA=="
+    },
     "node_modules/@humanwhocodes/config-array": {
       "version": "0.11.14",
       "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz",
@@ -11006,6 +11077,26 @@
         }
       }
     },
+    "node_modules/jsdom/node_modules/ws": {
+      "version": "7.5.9",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz",
+      "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==",
+      "engines": {
+        "node": ">=8.3.0"
+      },
+      "peerDependencies": {
+        "bufferutil": "^4.0.1",
+        "utf-8-validate": "^5.0.2"
+      },
+      "peerDependenciesMeta": {
+        "bufferutil": {
+          "optional": true
+        },
+        "utf-8-validate": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/jsesc": {
       "version": "2.5.2",
       "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz",
@@ -11361,6 +11452,11 @@
         "tmpl": "1.0.5"
       }
     },
+    "node_modules/md5-typescript": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/md5-typescript/-/md5-typescript-1.0.5.tgz",
+      "integrity": "sha512-ovAc4EtiNt2dY8JPhPr/wkC9h4U5k/nuClNVcG0Ga3V1rMlYpAY24ZaaymFXJlz+ccJ6UMPo3FSaVKe7czBsXw=="
+    },
     "node_modules/mdn-data": {
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.4.tgz",
@@ -17771,26 +17867,6 @@
         "url": "https://opencollective.com/webpack"
       }
     },
-    "node_modules/webpack-dev-server/node_modules/ws": {
-      "version": "8.16.0",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.16.0.tgz",
-      "integrity": "sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==",
-      "engines": {
-        "node": ">=10.0.0"
-      },
-      "peerDependencies": {
-        "bufferutil": "^4.0.1",
-        "utf-8-validate": ">=5.0.2"
-      },
-      "peerDependenciesMeta": {
-        "bufferutil": {
-          "optional": true
-        },
-        "utf-8-validate": {
-          "optional": true
-        }
-      }
-    },
     "node_modules/webpack-manifest-plugin": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/webpack-manifest-plugin/-/webpack-manifest-plugin-4.1.1.tgz",
@@ -18433,15 +18509,15 @@
       }
     },
     "node_modules/ws": {
-      "version": "7.5.9",
+      "version": "8.17.0",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.0.tgz",
-      "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==",
+      "integrity": "sha512-uJq6108EgZMAl20KagGkzCKfMEjxmKvZHG7Tlq0Z6nOky7YF7aq4mOx6xK8TJ/i1LeK4Qus7INktacctDgY8Ow==",
       "engines": {
-        "node": ">=8.3.0"
+        "node": ">=10.0.0"
       },
       "peerDependencies": {
         "bufferutil": "^4.0.1",
-        "utf-8-validate": "^5.0.2"
+        "utf-8-validate": ">=5.0.2"
       },
       "peerDependenciesMeta": {
         "bufferutil": {

ui/package.json

@@ -3,6 +3,10 @@
   "version": "0.1.0",
   "private": true,
   "dependencies": {
+    "@foxglove/cdr": "^3.3.0",
+    "@foxglove/rosmsg": "^5.0.4",
+    "@foxglove/rosmsg2-serialization": "^2.0.3",
+    "@foxglove/ws-protocol": "^0.7.3",
     "@monaco-editor/react": "^4.6.0",
     "@testing-library/jest-dom": "^5.17.0",
     "@testing-library/react": "^13.4.0",
@@ -48,6 +52,7 @@
     "urdf-loader": "^0.12.1",
     "uuid": "^9.0.1",
     "web-vitals": "^2.1.4",
+    "ws": "^8.17.0",
     "xml-formatter": "^3.6.2"
   },
   "scripts": {

ui/src/core/repository/core_three_repository.ts

@@ -21,7 +21,8 @@ import {
   Quaternion,
   MeshBasicMaterial,
   PlaneGeometry,
-  BoxGeometry
+  BoxGeometry,
+  AxesHelper
 } from "three";
 import { TypedEvent } from "../helper/typed_event";
 import { Result } from "../helper/result";
@@ -34,7 +35,12 @@ import {
 import { SceneMode } from "../../features/scene_manager/model/scene_view";
 import { throttle } from "../helper/throttle";
 import { Asset, InstanceRgbCamera, RobossemblerAssets, SceneSimpleObject } from "../model/robossembler_assets";
- 
+import { LoadingManager } from 'three';
+import URDFLoader, { URDFLink } from 'urdf-loader';
+import { UrdfTransforms, coordsToQuternios, coordsToVector } from "../../features/simulations/tranforms_model";
+
+Object3D.DEFAULT_UP = new Vector3(0, 0, 1);
+
 export enum UserData {
   selectedObject = "selected_object",
   cameraInitialization = "camera_initialization",
@@ -51,6 +57,10 @@ interface IEmissiveCache {
   status: boolean;
   object3d: Object3D<Object3DEventMap>;
 }
+type SceneFrames = { [K in string]: URDFLink; }
+
+
+
 
 export class CoreThreeRepository extends TypedEvent<BaseSceneItemModel> {
   scene = new Scene();
@@ -62,10 +72,14 @@ export class CoreThreeRepository extends TypedEvent<BaseSceneItemModel> {
   orbitControls: OrbitControls;
   htmlSceneWidth: number;
   htmlSceneHeight: number;
+  sceneFrame?: SceneFrames;
   objLoader = new OBJLoader();
   glbLoader = new GLTFLoader();
   daeLoader = new ColladaLoader();
   stlLoader = new STLLoader();
+  manager = new LoadingManager();
+  urdfLoader = new URDFLoader(this.manager);
+
   watcherSceneEditorObject: Function;
 
   constructor(htmlCanvasRef: HTMLCanvasElement, watcherSceneEditorObject: Function) {
@@ -125,12 +139,26 @@ export class CoreThreeRepository extends TypedEvent<BaseSceneItemModel> {
       cube.position.copy(vector);
     }
     this.scene.add(cube);
+    // this.scene.children.
   }
   deleteSceneItem(item: BaseSceneItemModel) {
     const updateScene = this.scene;
     updateScene.children = item.deleteToScene(updateScene);
   }
+  loadUrdf = (urlPath: string) => {
+    this.urdfLoader.load(
+      urlPath,                    // The path to the URDF within the package OR absolute
+      robot => {
 
+        this.scene.add(robot)
+        console.log(robot)
+        // @ts-expect-error
+        this.sceneFrame = robot.frames
+
+
+      }
+    );
+  }
   loadInstances(robossemblerAssets: RobossemblerAssets) {
     robossemblerAssets.instances.forEach(async (el) => {
       if (el instanceof InstanceRgbCamera) {
@@ -142,7 +170,7 @@ export class CoreThreeRepository extends TypedEvent<BaseSceneItemModel> {
         const asset = robossemblerAssets.getAssetAtInstance(el.instanceAt as string);
         this.loader(
           asset.meshPath,
-          () => {},
+          () => { },
           asset.name,
           new Vector3(el.position.x, el.position.y, el.position.z),
           new Quaternion(el.quaternion[0], el.quaternion[1], el.quaternion[2], el.quaternion[3])
@@ -153,7 +181,7 @@ export class CoreThreeRepository extends TypedEvent<BaseSceneItemModel> {
   loadInstance(asset: Asset, loadCallback?: Function) {
     this.loader(
       asset.meshPath,
-      loadCallback ? loadCallback : () => {},
+      loadCallback ? loadCallback : () => { },
       asset.name,
       new Vector3(Number(asset.posX), Number(asset.posY), Number(asset.posZ)),
       new Quaternion(0, 0, 0, 0)
@@ -282,21 +310,11 @@ export class CoreThreeRepository extends TypedEvent<BaseSceneItemModel> {
     this.light();
     this.addListeners();
     const floor = new GridHelper(100, 100, 0x888888, 0x444444);
+    floor.geometry.rotateX(Math.PI * 0.5)
     floor.userData = {};
     floor.userData[UserData.cameraInitialization] = true;
+    floor.up.copy(new Vector3(0, 0, 1))
     this.scene.add(floor);
-
-    const planeMesh = new Mesh(
-      new PlaneGeometry(10, 10, 10, 10),
-      new MeshBasicMaterial({ color: 0x808080, wireframe: true })
-    );
-    planeMesh.userData[UserData.selectedObject] = true;
-    planeMesh.userData[UserData.objectForMagnetism] = true;
-    planeMesh.rotation.x = -Math.PI / 2;
-    this.makeCube(1);
-    this.makeCube(2, new Vector3(20, 0, 10), "yellow");
-
-    this.scene.add(planeMesh);
   }
 
   render() {
@@ -329,8 +347,8 @@ export class CoreThreeRepository extends TypedEvent<BaseSceneItemModel> {
       case "glb":
         this.glbLoader.load(
           url,
-          (result) => {},
+          (result) => { },
-          (err) => {}
+          (err) => { }
         );
         break;
       case "obj":
@@ -349,22 +367,22 @@ export class CoreThreeRepository extends TypedEvent<BaseSceneItemModel> {
               this.scene.add(el);
             });
           },
-          (err) => {}
+          (err) => { }
         );
         break;
       case "dae":
         this.daeLoader.load(
           url,
-          (result) => {},
+          (result) => { },
-          (err) => {}
+          (err) => { }
         );
         break;
       case "stl":
         this.stlLoader.load(
           url,
-          (result) => {},
+          (result) => { },
 
-          (err) => {}
+          (err) => { }
         );
         break;
     }
@@ -451,4 +469,15 @@ export class CoreThreeRepository extends TypedEvent<BaseSceneItemModel> {
     this.camera.lookAt(bsWorld);
     this.orbitControls = new OrbitControls(this.camera, this.htmlCanvasRef);
   }
+  urdfTransforms = (urdfTransforms: UrdfTransforms) => {
+    urdfTransforms.transforms.forEach((transform) => {
+      if (this.sceneFrame) {
+        const currentLink = this.sceneFrame[transform?.child_frame_id ?? ""]
+        const currentLinkParrent = this.sceneFrame[transform?.header.frame_id ?? ""]
+        // currentLink.position.copy(coordsToVector(transform.transform.translation))
+        currentLink.quaternion.copy(coordsToQuternios(transform.transform.rotation))
+      }
+    })
+
+  }
 }

ui/src/core/repository/ros_ws_bridge_repository.ts

@@ -0,0 +1,37 @@
+import { FoxgloveClient } from "@foxglove/ws-protocol"
+import { MessageReader } from '@foxglove/rosmsg2-serialization';
+import { parse as parseMessageDefinition } from '@foxglove/rosmsg';
+import { TypedEvent } from "../helper/typed_event";
+
+export class RosWsBridgeRepository<T> extends TypedEvent<T> {
+    client?: FoxgloveClient;
+    constructor() {
+        super();
+    }
+    connect(topic: string) {
+        const client = new FoxgloveClient({
+            ws: new WebSocket(`ws://localhost:8765`, [FoxgloveClient.SUPPORTED_SUBPROTOCOL]),
+        });
+        const deserializers = new Map();
+        client.on("advertise", (channels) => {
+            for (const channel of channels) {
+                if (channel.topic.isEqual(topic)) {
+                    const subId = client.subscribe(channel.id);
+                    deserializers.set(subId, (data: any) => {
+                        return {
+                            data: data,
+                            channel: channel
+                        }
+                    });
+                }
+            }
+        });
+        client.on("message", ({ subscriptionId, timestamp, data }) => {
+            const channel = deserializers.get(subscriptionId)(data);
+            this.emit(new MessageReader(parseMessageDefinition(channel.channel?.schema!, {
+                ros2: true,
+            })).readMessage(channel.data))
+
+        });
+    }
+}

ui/src/core/routers/routers.tsx

@@ -19,7 +19,7 @@ import {
 import { DataSetScreen, DatasetsScreenPath } from "../../features/dataset/dataset_screen";
 import DetailsScreen, { DetailsScreenPath } from "../../features/details/details_screen";
 import { AssemblesScreen, AssemblesScreenPath } from "../../features/assembles/assembles_screen";
-import SimulationScreen, { SimulationScreenPath } from "../../features/simulations/simulations_screen";
+import { SimulationScreen, SimulationScreenPath } from "../../features/simulations/simulations_screen";
 import { EstimateScreen, EstimateScreenPath } from "../../features/estimate/estimate_screen";
 import { SkillPath, SkillScreen } from "../../features/skils/skills_screen";
 

ui/src/features/all_projects/presentation/all_projects_screen.tsx

@@ -24,7 +24,7 @@ export const AllProjectScreen: React.FunctionComponent = observer(() => {
         needBackButton={false}
         minText="create project?"
         path={CreateProjectScreenPath}
-        isError={store.isError}
+        isError={false}
         isLoading={store.isLoading}
         children={
           <div>

ui/src/features/simulations/simulations_screen.tsx

@@ -1,11 +1,15 @@
 import * as React from "react";
 import { MainPage } from "../../core/ui/pages/main_page";
-
+import { SimulationStore } from "./simulations_store";
-export interface ISimulationScreenProps {}
+interface ISimulationScreenProps { }
 export const SimulationScreenPath = "/simulation";
 
-export default class SimulationScreen extends React.Component<ISimulationScreenProps> {
+export const SimulationScreen = (props: ISimulationScreenProps) => {
-  public render() {
+  const [store] = React.useState(() => new SimulationStore());
-    return <MainPage page={"Симуляция"}></MainPage>;
+  const canvasRef = React.useRef<HTMLCanvasElement>(null);
-  }
+  React.useEffect(() => {
-}
+    store.init();
+    store.loadScene(canvasRef.current!);
+  }, []);
+  return <MainPage page={"Симуляция"} bodyChildren={<canvas ref={canvasRef} style={{ overflow: "hidden" }} />} />;
+}

ui/src/features/simulations/simulations_store.ts

@@ -0,0 +1,29 @@
+import makeAutoObservable from "mobx-store-inheritance";
+import { RosWsBridgeRepository } from "../../core/repository/ros_ws_bridge_repository";
+import { CoreThreeRepository } from "../../core/repository/core_three_repository";
+import { UrdfTransforms } from "./tranforms_model";
+
+export class SimulationStore {
+    rosWsUrdfTransfomsListner: RosWsBridgeRepository<UrdfTransforms> = new RosWsBridgeRepository();
+    coreThereRepository?: CoreThreeRepository;
+
+    constructor() {
+        makeAutoObservable(this)
+    }
+    init = () => {
+        this.rosWsUrdfTransfomsListner.connect('/tf')
+    }
+    async loadScene(canvasRef: HTMLCanvasElement) {
+        await this.loadWebGl(canvasRef);
+        this.coreThereRepository?.loadUrdf('http://localhost:4001/robot.xml');
+        if (this.coreThereRepository?.urdfTransforms) {
+            this.rosWsUrdfTransfomsListner.on(this.coreThereRepository.urdfTransforms)
+
+        }
+    }
+    loadWebGl = async (canvasRef: HTMLCanvasElement) => {
+        this.coreThereRepository = new CoreThreeRepository(canvasRef as HTMLCanvasElement, () => { });
+        this.coreThereRepository.on(() => { });
+        this.coreThereRepository.render();
+    }
+}

ui/src/features/simulations/tranforms_model.ts

@@ -0,0 +1,40 @@
+import { Quaternion, Vector3 } from "three";
+
+export interface UrdfTransforms {
+    transforms: TransformElement[];
+}
+
+export interface TransformElement {
+    header: Header;
+    child_frame_id: string;
+    transform: TransformTransform;
+}
+
+export interface Header {
+    stamp: Stamp;
+    frame_id: string;
+}
+
+export interface Stamp {
+    sec: number;
+    nsec: number;
+}
+
+export interface TransformTransform {
+    translation: Coords;
+    rotation: Coords;
+}
+export interface Coords {
+    x: number;
+    y: number;
+    z: number;
+    w?: number;
+}
+
+export const coordsToVector = (coords: Coords): Vector3 => {
+    return new Vector3(coords.x, coords.y, coords.z)
+}
+export const coordsToQuternios = (coords: Coords): Quaternion => {
+    return new Quaternion(coords.x, coords.y, coords.z, coords.w)
+}
+