diff --git a/server/src/features/weights/domain/exec_weights_process_scenario.ts b/server/src/features/weights/domain/exec_weights_process_scenario.ts index 92fe0df..361da60 100644 --- a/server/src/features/weights/domain/exec_weights_process_scenario.ts +++ b/server/src/features/weights/domain/exec_weights_process_scenario.ts @@ -32,7 +32,6 @@ export class ExecWeightProcessScenario extends CallbackStrategyWithIdQuery { if (preTrain) { execCommand += ` --pretrain`; } - console.log(execCommand); return await new ExecProcessUseCase().call( `${model.project.rootDir}/`, execCommand, diff --git a/ui/package-lock.json b/ui/package-lock.json index bc40a61..20f0dd7 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -8,6 +8,10 @@ "name": "ui-robossembler", "version": "0.1.0", "dependencies": { + "@foxglove/cdr": "^3.3.0", + "@foxglove/rosmsg": "^5.0.4", + "@foxglove/rosmsg2-serialization": "^2.0.3", + "@foxglove/ws-protocol": "^0.7.3", "@monaco-editor/react": "^4.6.0", "@testing-library/jest-dom": "^5.17.0", "@testing-library/react": "^13.4.0", @@ -53,6 +57,7 @@ "urdf-loader": "^0.12.1", "uuid": "^9.0.1", "web-vitals": "^2.1.4", + "ws": "^8.17.0", "xml-formatter": "^3.6.2" }, "devDependencies": { @@ -2625,6 +2630,72 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@foxglove/cdr": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@foxglove/cdr/-/cdr-3.3.0.tgz", + "integrity": "sha512-CjeA6ka/0cddVzQZY0qTEbExw849C7dD1aGpa+KeEzx0LZNxsQvvhsqWeGjszVM4BzwPcyWHwsiOZv27zbnnFA==", + "engines": { + "node": ">= 14" + } + }, + "node_modules/@foxglove/message-definition": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@foxglove/message-definition/-/message-definition-0.2.0.tgz", + "integrity": "sha512-IQHIGCvBZR8GIua9nEpS+hsMF3gm1bfbrrnjG0rgtcFBWiNuKbzx4vIP8OIwDC+8wtwcFdfJhf4Vp5TPFiUUcQ==" + }, + "node_modules/@foxglove/rosmsg": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/@foxglove/rosmsg/-/rosmsg-5.0.4.tgz", + "integrity": "sha512-s0JQLA6Zi0Qh9HtzyAan30sjiMU0u34+fkwccek/Xt+aQjCkSoErP/U4NBP6hCDfp0fBcb7CDSU2ggLFy/rJAQ==", + "dependencies": { + "@foxglove/message-definition": "^0.3.1", + "md5-typescript": "^1.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@foxglove/rosmsg/node_modules/@foxglove/message-definition": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@foxglove/message-definition/-/message-definition-0.3.1.tgz", + "integrity": "sha512-nkPowiED67LjcKEC77CprkUG3XvSsFHHR9HEwWCuhnIC2wm0W57T1J+WWvteoArZ7SdGGlKzSYSRFyjQkgmITw==" + }, + "node_modules/@foxglove/rosmsg2-serialization": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@foxglove/rosmsg2-serialization/-/rosmsg2-serialization-2.0.3.tgz", + "integrity": "sha512-atNKdzO8aYJLFbqd8OblUo1V5faWSmM2mlKkDfA/4wI65phGiq+SJOfElbj5nNNtRGgC60BkdIxcRyMQgxym2g==", + "dependencies": { + "@foxglove/cdr": "^3.0.0", + "@foxglove/message-definition": "^0.2.0", + "@foxglove/rostime": "^1.1.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@foxglove/rostime": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@foxglove/rostime/-/rostime-1.1.2.tgz", + "integrity": "sha512-vWuTJCuGv0xvgwOlrZ1y2MevmNMVxWcUU/HwlmYXi/jUq/kRaACStU18uyuZ3LzdNKaffkti0rTcXWESaQjwQw==", + "engines": { + "node": ">= 14" + } + }, + "node_modules/@foxglove/ws-protocol": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/@foxglove/ws-protocol/-/ws-protocol-0.7.3.tgz", + "integrity": "sha512-tgZpg1eT5Gm/C9Qe+BMdfH6P5WoSv/dHTcU61LJr36zqFTNfJ6hjZ+FWba5PLz/yoJd0/xgYUJatzamm43yeWA==", + "dependencies": { + "debug": "^4", + "eventemitter3": "^5.0.1", + "tslib": "^2.6.2" + } + }, + "node_modules/@foxglove/ws-protocol/node_modules/eventemitter3": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", + "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==" + }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.14", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", @@ -11006,6 +11077,26 @@ } } }, + "node_modules/jsdom/node_modules/ws": { + "version": "7.5.9", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", + "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/jsesc": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", @@ -11361,6 +11452,11 @@ "tmpl": "1.0.5" } }, + "node_modules/md5-typescript": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/md5-typescript/-/md5-typescript-1.0.5.tgz", + "integrity": "sha512-ovAc4EtiNt2dY8JPhPr/wkC9h4U5k/nuClNVcG0Ga3V1rMlYpAY24ZaaymFXJlz+ccJ6UMPo3FSaVKe7czBsXw==" + }, "node_modules/mdn-data": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.4.tgz", @@ -17771,26 +17867,6 @@ "url": "https://opencollective.com/webpack" } }, - "node_modules/webpack-dev-server/node_modules/ws": { - "version": "8.16.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.16.0.tgz", - "integrity": "sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, "node_modules/webpack-manifest-plugin": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/webpack-manifest-plugin/-/webpack-manifest-plugin-4.1.1.tgz", @@ -18433,15 +18509,15 @@ } }, "node_modules/ws": { - "version": "7.5.9", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", - "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "version": "8.17.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.0.tgz", + "integrity": "sha512-uJq6108EgZMAl20KagGkzCKfMEjxmKvZHG7Tlq0Z6nOky7YF7aq4mOx6xK8TJ/i1LeK4Qus7INktacctDgY8Ow==", "engines": { - "node": ">=8.3.0" + "node": ">=10.0.0" }, "peerDependencies": { "bufferutil": "^4.0.1", - "utf-8-validate": "^5.0.2" + "utf-8-validate": ">=5.0.2" }, "peerDependenciesMeta": { "bufferutil": { diff --git a/ui/package.json b/ui/package.json index 72c809e..173d335 100644 --- a/ui/package.json +++ b/ui/package.json @@ -3,6 +3,10 @@ "version": "0.1.0", "private": true, "dependencies": { + "@foxglove/cdr": "^3.3.0", + "@foxglove/rosmsg": "^5.0.4", + "@foxglove/rosmsg2-serialization": "^2.0.3", + "@foxglove/ws-protocol": "^0.7.3", "@monaco-editor/react": "^4.6.0", "@testing-library/jest-dom": "^5.17.0", "@testing-library/react": "^13.4.0", @@ -48,6 +52,7 @@ "urdf-loader": "^0.12.1", "uuid": "^9.0.1", "web-vitals": "^2.1.4", + "ws": "^8.17.0", "xml-formatter": "^3.6.2" }, "scripts": { diff --git a/ui/src/core/repository/core_three_repository.ts b/ui/src/core/repository/core_three_repository.ts index 531c0c2..2bd6b40 100644 --- a/ui/src/core/repository/core_three_repository.ts +++ b/ui/src/core/repository/core_three_repository.ts @@ -21,7 +21,8 @@ import { Quaternion, MeshBasicMaterial, PlaneGeometry, - BoxGeometry + BoxGeometry, + AxesHelper } from "three"; import { TypedEvent } from "../helper/typed_event"; import { Result } from "../helper/result"; @@ -34,7 +35,12 @@ import { import { SceneMode } from "../../features/scene_manager/model/scene_view"; import { throttle } from "../helper/throttle"; import { Asset, InstanceRgbCamera, RobossemblerAssets, SceneSimpleObject } from "../model/robossembler_assets"; - +import { LoadingManager } from 'three'; +import URDFLoader, { URDFLink } from 'urdf-loader'; +import { UrdfTransforms, coordsToQuternios, coordsToVector } from "../../features/simulations/tranforms_model"; + +Object3D.DEFAULT_UP = new Vector3(0, 0, 1); + export enum UserData { selectedObject = "selected_object", cameraInitialization = "camera_initialization", @@ -51,6 +57,10 @@ interface IEmissiveCache { status: boolean; object3d: Object3D; } +type SceneFrames = { [K in string]: URDFLink; } + + + export class CoreThreeRepository extends TypedEvent { scene = new Scene(); @@ -62,10 +72,14 @@ export class CoreThreeRepository extends TypedEvent { orbitControls: OrbitControls; htmlSceneWidth: number; htmlSceneHeight: number; + sceneFrame?: SceneFrames; objLoader = new OBJLoader(); glbLoader = new GLTFLoader(); daeLoader = new ColladaLoader(); stlLoader = new STLLoader(); + manager = new LoadingManager(); + urdfLoader = new URDFLoader(this.manager); + watcherSceneEditorObject: Function; constructor(htmlCanvasRef: HTMLCanvasElement, watcherSceneEditorObject: Function) { @@ -125,12 +139,26 @@ export class CoreThreeRepository extends TypedEvent { cube.position.copy(vector); } this.scene.add(cube); + // this.scene.children. } deleteSceneItem(item: BaseSceneItemModel) { const updateScene = this.scene; updateScene.children = item.deleteToScene(updateScene); } + loadUrdf = (urlPath: string) => { + this.urdfLoader.load( + urlPath, // The path to the URDF within the package OR absolute + robot => { + this.scene.add(robot) + console.log(robot) + // @ts-expect-error + this.sceneFrame = robot.frames + + + } + ); + } loadInstances(robossemblerAssets: RobossemblerAssets) { robossemblerAssets.instances.forEach(async (el) => { if (el instanceof InstanceRgbCamera) { @@ -142,7 +170,7 @@ export class CoreThreeRepository extends TypedEvent { const asset = robossemblerAssets.getAssetAtInstance(el.instanceAt as string); this.loader( asset.meshPath, - () => {}, + () => { }, asset.name, new Vector3(el.position.x, el.position.y, el.position.z), new Quaternion(el.quaternion[0], el.quaternion[1], el.quaternion[2], el.quaternion[3]) @@ -153,7 +181,7 @@ export class CoreThreeRepository extends TypedEvent { loadInstance(asset: Asset, loadCallback?: Function) { this.loader( asset.meshPath, - loadCallback ? loadCallback : () => {}, + loadCallback ? loadCallback : () => { }, asset.name, new Vector3(Number(asset.posX), Number(asset.posY), Number(asset.posZ)), new Quaternion(0, 0, 0, 0) @@ -282,21 +310,11 @@ export class CoreThreeRepository extends TypedEvent { this.light(); this.addListeners(); const floor = new GridHelper(100, 100, 0x888888, 0x444444); + floor.geometry.rotateX(Math.PI * 0.5) floor.userData = {}; floor.userData[UserData.cameraInitialization] = true; + floor.up.copy(new Vector3(0, 0, 1)) this.scene.add(floor); - - const planeMesh = new Mesh( - new PlaneGeometry(10, 10, 10, 10), - new MeshBasicMaterial({ color: 0x808080, wireframe: true }) - ); - planeMesh.userData[UserData.selectedObject] = true; - planeMesh.userData[UserData.objectForMagnetism] = true; - planeMesh.rotation.x = -Math.PI / 2; - this.makeCube(1); - this.makeCube(2, new Vector3(20, 0, 10), "yellow"); - - this.scene.add(planeMesh); } render() { @@ -329,8 +347,8 @@ export class CoreThreeRepository extends TypedEvent { case "glb": this.glbLoader.load( url, - (result) => {}, - (err) => {} + (result) => { }, + (err) => { } ); break; case "obj": @@ -349,22 +367,22 @@ export class CoreThreeRepository extends TypedEvent { this.scene.add(el); }); }, - (err) => {} + (err) => { } ); break; case "dae": this.daeLoader.load( url, - (result) => {}, - (err) => {} + (result) => { }, + (err) => { } ); break; case "stl": this.stlLoader.load( url, - (result) => {}, + (result) => { }, - (err) => {} + (err) => { } ); break; } @@ -451,4 +469,15 @@ export class CoreThreeRepository extends TypedEvent { this.camera.lookAt(bsWorld); this.orbitControls = new OrbitControls(this.camera, this.htmlCanvasRef); } + urdfTransforms = (urdfTransforms: UrdfTransforms) => { + urdfTransforms.transforms.forEach((transform) => { + if (this.sceneFrame) { + const currentLink = this.sceneFrame[transform?.child_frame_id ?? ""] + const currentLinkParrent = this.sceneFrame[transform?.header.frame_id ?? ""] + // currentLink.position.copy(coordsToVector(transform.transform.translation)) + currentLink.quaternion.copy(coordsToQuternios(transform.transform.rotation)) + } + }) + + } } diff --git a/ui/src/core/repository/ros_ws_bridge_repository.ts b/ui/src/core/repository/ros_ws_bridge_repository.ts new file mode 100644 index 0000000..5e45b18 --- /dev/null +++ b/ui/src/core/repository/ros_ws_bridge_repository.ts @@ -0,0 +1,37 @@ +import { FoxgloveClient } from "@foxglove/ws-protocol" +import { MessageReader } from '@foxglove/rosmsg2-serialization'; +import { parse as parseMessageDefinition } from '@foxglove/rosmsg'; +import { TypedEvent } from "../helper/typed_event"; + +export class RosWsBridgeRepository extends TypedEvent { + client?: FoxgloveClient; + constructor() { + super(); + } + connect(topic: string) { + const client = new FoxgloveClient({ + ws: new WebSocket(`ws://localhost:8765`, [FoxgloveClient.SUPPORTED_SUBPROTOCOL]), + }); + const deserializers = new Map(); + client.on("advertise", (channels) => { + for (const channel of channels) { + if (channel.topic.isEqual(topic)) { + const subId = client.subscribe(channel.id); + deserializers.set(subId, (data: any) => { + return { + data: data, + channel: channel + } + }); + } + } + }); + client.on("message", ({ subscriptionId, timestamp, data }) => { + const channel = deserializers.get(subscriptionId)(data); + this.emit(new MessageReader(parseMessageDefinition(channel.channel?.schema!, { + ros2: true, + })).readMessage(channel.data)) + + }); + } +} diff --git a/ui/src/core/routers/routers.tsx b/ui/src/core/routers/routers.tsx index 5797f86..cf292ad 100644 --- a/ui/src/core/routers/routers.tsx +++ b/ui/src/core/routers/routers.tsx @@ -19,7 +19,7 @@ import { import { DataSetScreen, DatasetsScreenPath } from "../../features/dataset/dataset_screen"; import DetailsScreen, { DetailsScreenPath } from "../../features/details/details_screen"; import { AssemblesScreen, AssemblesScreenPath } from "../../features/assembles/assembles_screen"; -import SimulationScreen, { SimulationScreenPath } from "../../features/simulations/simulations_screen"; +import { SimulationScreen, SimulationScreenPath } from "../../features/simulations/simulations_screen"; import { EstimateScreen, EstimateScreenPath } from "../../features/estimate/estimate_screen"; import { SkillPath, SkillScreen } from "../../features/skils/skills_screen"; diff --git a/ui/src/features/all_projects/presentation/all_projects_screen.tsx b/ui/src/features/all_projects/presentation/all_projects_screen.tsx index c7d1041..c29b9cb 100644 --- a/ui/src/features/all_projects/presentation/all_projects_screen.tsx +++ b/ui/src/features/all_projects/presentation/all_projects_screen.tsx @@ -24,7 +24,7 @@ export const AllProjectScreen: React.FunctionComponent = observer(() => { needBackButton={false} minText="create project?" path={CreateProjectScreenPath} - isError={store.isError} + isError={false} isLoading={store.isLoading} children={
diff --git a/ui/src/features/simulations/simulations_screen.tsx b/ui/src/features/simulations/simulations_screen.tsx index b0fd134..9fd1644 100644 --- a/ui/src/features/simulations/simulations_screen.tsx +++ b/ui/src/features/simulations/simulations_screen.tsx @@ -1,11 +1,15 @@ import * as React from "react"; import { MainPage } from "../../core/ui/pages/main_page"; - -export interface ISimulationScreenProps {} +import { SimulationStore } from "./simulations_store"; +interface ISimulationScreenProps { } export const SimulationScreenPath = "/simulation"; -export default class SimulationScreen extends React.Component { - public render() { - return ; - } -} +export const SimulationScreen = (props: ISimulationScreenProps) => { + const [store] = React.useState(() => new SimulationStore()); + const canvasRef = React.useRef(null); + React.useEffect(() => { + store.init(); + store.loadScene(canvasRef.current!); + }, []); + return } />; +} \ No newline at end of file diff --git a/ui/src/features/simulations/simulations_store.ts b/ui/src/features/simulations/simulations_store.ts new file mode 100644 index 0000000..0ecb96c --- /dev/null +++ b/ui/src/features/simulations/simulations_store.ts @@ -0,0 +1,29 @@ +import makeAutoObservable from "mobx-store-inheritance"; +import { RosWsBridgeRepository } from "../../core/repository/ros_ws_bridge_repository"; +import { CoreThreeRepository } from "../../core/repository/core_three_repository"; +import { UrdfTransforms } from "./tranforms_model"; + +export class SimulationStore { + rosWsUrdfTransfomsListner: RosWsBridgeRepository = new RosWsBridgeRepository(); + coreThereRepository?: CoreThreeRepository; + + constructor() { + makeAutoObservable(this) + } + init = () => { + this.rosWsUrdfTransfomsListner.connect('/tf') + } + async loadScene(canvasRef: HTMLCanvasElement) { + await this.loadWebGl(canvasRef); + this.coreThereRepository?.loadUrdf('http://localhost:4001/robot.xml'); + if (this.coreThereRepository?.urdfTransforms) { + this.rosWsUrdfTransfomsListner.on(this.coreThereRepository.urdfTransforms) + + } + } + loadWebGl = async (canvasRef: HTMLCanvasElement) => { + this.coreThereRepository = new CoreThreeRepository(canvasRef as HTMLCanvasElement, () => { }); + this.coreThereRepository.on(() => { }); + this.coreThereRepository.render(); + } +} \ No newline at end of file diff --git a/ui/src/features/simulations/tranforms_model.ts b/ui/src/features/simulations/tranforms_model.ts new file mode 100644 index 0000000..a9850c2 --- /dev/null +++ b/ui/src/features/simulations/tranforms_model.ts @@ -0,0 +1,40 @@ +import { Quaternion, Vector3 } from "three"; + +export interface UrdfTransforms { + transforms: TransformElement[]; +} + +export interface TransformElement { + header: Header; + child_frame_id: string; + transform: TransformTransform; +} + +export interface Header { + stamp: Stamp; + frame_id: string; +} + +export interface Stamp { + sec: number; + nsec: number; +} + +export interface TransformTransform { + translation: Coords; + rotation: Coords; +} +export interface Coords { + x: number; + y: number; + z: number; + w?: number; +} + +export const coordsToVector = (coords: Coords): Vector3 => { + return new Vector3(coords.x, coords.y, coords.z) +} +export const coordsToQuternios = (coords: Coords): Quaternion => { + return new Quaternion(coords.x, coords.y, coords.z, coords.w) +} + \ No newline at end of file