framework/docs/scene_generator.md at dc5921163440106dd72228b3cc3fbbe4df7dfda8

robossembler/framework

Fork 0

Igor Brylyov dc59211634 DOCS: object detection usecase

2025-02-27 16:58:24 +03:00

14 KiB

Raw Blame History

Мета-модель фреймворка

Соответствие Digital Twin Definition Language и мета-модели Robossembler

Слои абстракции

DTDL - BASETYPES, COMPLEXTYPES (Array, Map, ...), DT_TYPES
ROBOSSEMBLER_CONTEXT (Represents)
USE_CASE_CONTEXT (Entities)
USE_CASE_INSTANCE (Product)

Сравнение мета-моделей Digital Twin Definition Language и ROS 2

Сущность	DTDL	ROS	Robonomics
Описывает всё содержимое двойника, включая ниже приведённые сущности	Interface	Interface Spec	-
Передаваемые данные и их тип	Telemetry	Topic	Datalog Hash
Свойство, описывающее какое-то состояние двойника, может быть read-only или read/write; также описывает синхронизацию состояния между компонентами (например, показание датчика записано в облако)	Property	Parameters	Launch Param
Функция или операция, которая может быть осуществлена над двойником (например, `reboot`)	Command	Service / Action	Launch
Структура из примитивных типов (Array, Enum, Map, Object) данных для сериализации (в JSON, Avro, Protobuf)	Schema	IDL	-
Часть интерфейса (отношение part-of с каким-то другим двойником)	Component	-	-
Связь с другим цифровым двойником. Связи могут представлять различные семантические значения. Например, `contains` ("этаж содержит комнату"), `cools` ("hvac cools room"), `isBilledTo` ("счёт выставляется пользователю")	Relationship	-	-

# базовые типы значений (из JSON Schema)
"string","number", "boolean", "array", "object";

{
  "type": "object",
  "properties": {
    "REPRESENT": { 
      "type": { 
        "enum": [ "OBJECT_LINK", "KEY", "FILEPATH", "VALUE", "TREE", "ARRAY", "SEQUENCE" ] 
      }
    }
  }
}

# представления
ENUM REPRESENT = "OBJECT_LINK", # строка-ссылка на объект (ENTITY)
                 "KEY",         # уникальный ключ в виде строки (используется как идентификатор записи)
                 "FILEPATH",    # строка - путь к файлу
                 "VALUE",       # непосредственное значение
                 "TREE",        # представление в виде дерева
                 "ARRAY",       # массив значений
                 "SEQUENCE";    # массив ссылок на объект определённого типа

# сущности

ENUM ENTITY =  "MESH", "PART", "ASSET", "BTREE", "BTACTION", "SKILL", "DATASET", "INTERFACE", "WEIGHTS", "DEVICE";

ENUM DEVICE = "ROBOT", "SENSOR";

type SCENE = {
    "objects": [ { ref: "1", type: "PART" }; { ref: "2", type: "PART" }; ]
};

type PARAM = {
    "sid": \${KEY:string:""},
    "name": \${NAME:string:""},
    "represent": \${REPRESENT:Enum<REPRESENT>:"VALUE"},
    "link": \${LINK:Enum<LINK>:"topic"}
};

### тип поверхность детали

type MESH/SURFACE = {
  "sid": \${KEY:string:""};
  "path": { "stl": "PATH/*.stl", "brep": "PATH/*.brep", "fbx": "PATH/*.fbx",  }
};

type PART = {
  "sid": \${NAME:string:""},
  "name": \${NAME:string:""},
  "pose6d": { "loc_xyz": \${XYZ:Array<number>3:[0.0,0.0,0.0] }, "rot_xyzw": \${XYZW:Array<number>4:[0.0,0.0,0.0,1.0]} },
  "attributes": [
      "Robossembler_NonSolid": True
  ],
  "surface": { "stl": "PATH/*.stl", "brep": "PATH/*.brep", },
  "material": "/path/to/robossembler/materials/mmm.FCMat",
  "unit_scale": \${UNIT_SCALE:number:1.0},
  "dimensions": \${Array<number>3:[0.0,0.0,0.0]},
  "assets": { "fbx": "PATH/*.fbx", "blend": "PATH/*.blend", }
};

type DATASET = {
  "sid": \${NAME:string:""},
  "name": \${NAME:string:""},
  "objects": \${SEQUENCE:PART},
  "environment": \${SEQUENCE:PART},
  "randomisation": \${FILE}
};

type WEIGHTS = {
  "sid": \${NAME:string:""},
  "name": \${NAME:string:""},
  "file": \${FILE:string:"*.pth"},
  "epoch": \${EPOCH:number:""},
  "dataset": \${OBJECT_LINK:DATASET}
};

type TOPIC = {
  "sid": ...,
  "name": "topic_name",
  "msg": "sensor_msgs/Image",
};

DEVICE = {
  "sid": 1235,
  "name": "dev",
  "topics": \${SEQUENCE:TOPIC},
}

// DEVICE -> TOPIC LIST -> {DEVICE: {TOPIC LIST}}

type POSE_ESTIMATION = {
  "object_name": \${OBJECT_LINK:PART},
  "weights": \${OBJECT_LINK:WEIGHTS},
  "topic_name": \${OBJECT_LINK:TOPIC}
};

type SKILL = {
  "sid": \${NAME:string:""},
  "name": \${NAME:string:""},
  "interface": \${INTERFACE}
};

command_LifeCycle = "run", "stop"

type ASSEMBLY_SEQUENCE = \{SEQUENCE:TASK};

# task1 = { source_state = []; target_state = [ p1 ] }
# task2 = { source_state = [ p1 ]; target_state = [ p1 p2 ] }
# task3 = { source_state = [ p1 p2 ]; target_state = [ p1 p2 p3 ] }

task = { source_state = \${TREE:PART}; target_state = \${TREE:PART} }

type TASK = {
  "sid": ...
  "action": \${BT_TREE}
  "source_state": \${TREE:PART} // PART
  "target_state": \${TREE:PART} // PRODUCT
};

type DOMAIN = {
  "objects": \{SEQUENCE:PART}
  "predicates": \{OBJECT_LINK:CONDITION}
  "actions": \${OBJECT_LINK:BT_ACTION}
};

type BTREE = {
  "sid": \${NAME:string:""},
  "name": \${NAME:string:""},
};

Device Package

Camera

{
  "DevicePackage": { "name": "Robossembler", "version": "1.0", "format": "1" },
  "Module": { "name": "RealSense Dxx", "description": "ROS Wrapper for Intel(R) RealSense(TM) Cameras" },
  "Launch": { "package": "realsense2_camera", "executable": "rs_launch.py" },
  "DTwin": [
    { "interface": {
        "input": [
          { "name": "camera_namespace", "type": "STRING" }, // -- /robot_drawer/455_1/camera_info
          { "name": "camera_name", "type": "STRING" },
          { "name": "serial_port", "type": "STRING" },
        ],
        "output": [
          { "name": "camera_info", "type": "TOPIC", "topic_name": "/${camera_namespace}/${camera_name}/camera_info" },
          { "name": "pose", "type": "TOPIC", "msg": "Pose" }
        ]
      },
    }
  ],
  "Settings": [
    { "name": "camera_config", "description": "Camera Config", "type":"file", "defaultValue": "{ rgb_camera.profile: 1280x720x15 }" }
  ]
}

Robot RBS

{
  "DevicePackage": { "name": "Robossembler", "version": "1.0", "format": "1" },
  "Module": { "name": "RBS", "description": "Main Robot" },
  "Launch": { "package": "rbs_bringup", "executable": "single_robot.launch.py" },
  "DTwin": [
    { "interface": {
        "input": [
          { "name": "robot_namespace", "type": "STRING", "defaultValue": "rbs_arm" },
          { "name": "dof", "type": "INT", "defaultValue": 6 }
        ]
      }
    }
  ],
  "Settings": [
    { "name": "robot_type", "description": "Type of robot by name", "defaultValue": "rbs_arm" }
  ]
}

Robot UR5

{
  "DevicePackage": { "name": "Robossembler", "version": "1.0", "format": "1" },
  "Module": { "name": "UR", "description": "..." },
  "Launch": { "package": "ur_package", "executable": "ur5_single_arm.py" },
  "DTwin": [
    { "interface": {
        "input": [
          { "robot_namespace": "robot1" },
        ],
      },
    }
  ],
  "Settings": [
    { "name": "", "description": "Config", "type":"file", "defaultValue": "{}" }
  ]
}

{
  "SkillPackage": { "name": "Robossembler", "version": "1.0", "format": "1" },
  "Module": { "name": "PoseEstimation", "description": "Pose Estimation skill with DOPE" },
  "Launch": { "package": "rbs_perception", "executable": "pe_dope_lc.py", "name": "lc_dope" },
  "BTAction": [
    { "name": "peConfigure",
      "type": "run",
      "interface": {
        "input": [
          { "name": "image_raw", "type": "TOPIC", "msg": "Image" },
          { "name": "camera_info", "type": "TOPIC", "msg": "CameraInfo" },
          { "name": "object_name", "type": "PART", "msgs": "Part" }, // string
          { "name": "weights", "type": "WEIGHTS", "msgs": "" },
        ],
        "output": [
          { "name": "pose_estimation", "type": "TOPIC" },
        ]
      },
    },
    { "name": "peStop", "type": "stop", "interface": { "input": [], "output": [] } }
  ],
  "Settings": [
    { "name": "publishDelay", "description": "Publish Delay", "type":"float", "defaultValue": 0.5 },
    { "name": "tf2_send_pose", "description": "Transform Pose", "type":"int", "defaultValue": 1 },
    { "name": "mesh_scale", "description": "Part Mesh Scale", "type":"float", "defaultValue": 0.001 }
  ]
}

Атомарные навыки

<TreeNodesModel>
    <SubTree ID="GraspObject">
        <input_port default="false" name="__shared_blackboard"></input_port>
        <input_port name="pose_vec"/>
        <input_port name="robot_name"/>
    </SubTree>
    <Action ID="GripperAction">
        <input_port name="action">Open or close</input_port>
        <input_port name="gripper_name"/>
    </Action>
    <SubTree ID="InspectWorkspace">
        <input_port default="true" name="__shared_blackboard">
        <input_port name="pose_vec"/>
        <input_port name="robot_name"/>
    </SubTree>
    <Condition ID="IsObjectInWorkspace">
        <input_port name="topic_name"/>
    </Condition>
    <Condition ID="IsObjectVisible"/>
    <Action ID="MoveToPose">
        <input_port name="cancel_timeout"/>
        <input_port name="pose"/>
        <input_port name="robot_name"/>
        <input_port name="server_name"/>
        <input_port name="server_timeout"/>
    </Action>
    <Action ID="MoveToPoseArray">
        <input_port name="cancel_timeout"/>
        <input_port name="pose_vec_in"/>
        <output_port name="pose_vec_out"/>
        <input_port name="robot_name"/>
        <input_port name="server_name"/>
        <input_port name="server_timeout"/>
    </Action>
    <SubTree ID="PlaceObject">
        <input_port default="true" name="__shared_blackboard"></input_port>
        <input_port name="pose_vec"/>
        <input_port name="robot_name"/>
    </SubTree>
    <Action ID="PoseEstimationCleanup"/>
    <Action ID="PoseEstimationConfigure">
        <input_port name="object_name"/>
        <output_port name="topic_name"/>
        <input_port name="weight_file"/>
    </Action>
    <Action ID="RbsBtAction">
        <input_port name="command"/>
        <input_port name="do"/>
        <input_port name="server_name"/>
        <input_port name="server_timeout"/>
    </Action>
</TreeNodesModel>

Пример интерфейса в DTDL

{
  "@context": "...;3",
  "@id": "robossember_assembly;1",
  "@type": "Interface",
  "displayName": "Bolt",
  "contents": [
    {
      "@type": "Property",
      "name": "Label",
      "schema": "string"
    },
    {
      "@type": "Property",
      "name": "mesh",
      "writable": true,
      "schema": "string"
    },
    {
      "@type": "Telemetry",
      "name": "Pose6D",
      "writable": true,
      "schema": {
        "@type": "Object",
        "fields": [
          {
            "name": "x",
            "schema": "double"
          },
          {
            "name": "y",
            "schema": "double"
          },
          {
            "name": "z",
            "schema": "double"
          }
        ]
      }
    },
    {
      "@type": "Relationship",
      "name": "contains",
      "target": "dtmi:com:robossember_assembly:Bolt;1"
    },
    {
      "@type": "Component",
      "name": "frontCamera",
      "schema": "dtmi:com:example:Camera;3"
    },
  ]
}

Пример файла описания сцены из Blender

TODO: рассписать потребность в основных элементах, что и для чего

{
  "assets": [
    {
      "name": "robo-arm",
      "id": "629b29d7-fe15-428b-9014-6c3dde045af8",
      "model_path": "../model.urdf"
    }
  ],
  "instances": [
    // измненная URDF модель
    {
      "id": "0e29084f-1190-45d0-bd59-8f4ce2591gb1",
      "name": "robo-arm-1",
      //assetId указывает на родительский ассет
      "assetId": "629b29d7-fe15-428b-9014-6c3dde045af8",
      "pose": {
        "x": 12.0,
        "y": 1.0,
        "z": 4.0,
        "roll": 1.0,
        "pitch": 4.0,
        "yaw": 5.0
      },
      //если изменено внутренее состояние URDF модели
      "tags": [
        "<joint name=\"robotiq_85_right_finger_joint\" type=\"fixed\"><parent link=\"robotiq_85_right_knuckle_link\"/><child link=\"robotiq_85_right_finger_link\"/><origin rpy=\"0 0 0\" xyz=\"-0.03152616 0.0 -0.00376347\"/></joint>"
      ]
    },
    {
      "id": "0e27984f-8890-4d90-bd59-8f4ce29920f9",
      "name": "robo-arm-2",
      //assetId указывает на родительский ассет
      "assetId": "629b29d7-fe15-428b-9014-6c3dde045af8",
      //если дефолтные позиции модели
      "pose": null,
      //если не изменено внутренее состояние URDF модели
      "tags": null
    },
    {
      "type": "LIGHT",
      "light_type": "SPOT",
      "power": 10.0,
      "spot_angle": 45.0,
      "name": null,
      "pose": {
        "x": 0.0,
        "y": 0.0,
        "z": 0.0,
        "roll": 0.0,
        "pitch": 0.0,
        "yaw": 0.0
      }
    }
  ]
}

14 KiB Raw Blame History Unescape Escape

Мета-модель фреймворка

Соответствие Digital Twin Definition Language и мета-модели Robossembler

Сравнение мета-моделей Digital Twin Definition Language и ROS 2

Device Package

Camera

Robot RBS

Robot UR5

Атомарные навыки

Пример интерфейса в DTDL

Пример файла описания сцены из Blender

14 KiB

Raw Blame History