Добавлены модули создания проектов сборки и подготовки датасетов

This commit is contained in:
IDONTSUDO 2024-04-23 10:32:43 +00:00 committed by Igor Brylyov
parent 40b9b116c1
commit f57438b404
173 changed files with 6750 additions and 1857 deletions

1873
server/package-lock.json generated

File diff suppressed because it is too large Load diff

View file

@ -45,13 +45,14 @@
"md5": "^2.3.0",
"mongoose": "^7.6.2",
"mongoose-autopopulate": "^1.1.0",
"pm2": "^5.3.1",
"reflect-metadata": "^0.1.13",
"rimraf": "^5.0.5",
"socket.io": "^4.7.2",
"socket.io-client": "^4.7.2",
"spark-md5": "^3.0.2",
"ts-md5": "^1.3.1",
"tsc-watch": "^6.0.4",
"uuid": "^9.0.1",
"pm2": "^5.3.1"
"uuid": "^9.0.1"
}
}

View file

@ -58,6 +58,8 @@ export class App extends TypedEvent<ServerStatus> {
io.on("connection", (socket) => {
this.socketSubscribers.map((el) => {
el.emitter.on((e) => {
console.log(el.event)
console.log(e)
socket.emit(el.event, e);
});
});
@ -107,7 +109,7 @@ export class App extends TypedEvent<ServerStatus> {
await result.fold(
async (_s) => {
await new CheckAndCreateStaticFilesFolderUseCase().call();
await new SetLastActivePipelineToRealTimeServiceScenario().call();
// await new SetLastActivePipelineToRealTimeServiceScenario().call();
},
async (_e) => {
this.setServerStatus(ServerStatus.error);

View file

@ -26,7 +26,7 @@ export abstract class CallBackStrategyWithQueryPage {
export abstract class CallbackStrategyWithFileUpload {
abstract checkingFileExpression: RegExp;
abstract idValidationExpression: CoreValidation;
abstract call(file: File, id: string, description: string): ResponseBase;
abstract call(file: File, id: string): ResponseBase;
}
interface ISubSetFeatureRouter<T> {
@ -122,12 +122,12 @@ export class CoreHttpController<V> implements ICoreHttpController {
res.status(400).json("need file to form data request");
return;
}
if (req.query.description === undefined) {
res
.status(400)
.json("request query description is null, need query description &description={description:String}");
return;
}
// if (req.query.description === undefined) {
// res
// .status(400)
// .json("request query description is null, need query description &description={description:String}");
// return;
// }
if (req.query.id === undefined) {
res.status(400).json("request query id is null, need query id ?id={id:String}");
return;
@ -142,7 +142,7 @@ export class CoreHttpController<V> implements ICoreHttpController {
return;
}
}
await this.responseHelper(res, el.fn.call(req["files"]["file"], req.query.id, req.query.description));
await this.responseHelper(res, el.fn.call(req["files"]["file"], req.query.id));
}
});
});
@ -197,12 +197,10 @@ export class CoreHttpController<V> implements ICoreHttpController {
}
(await useCase(payload)).fold(
(ok) => {
res.json(ok);
return;
return res.json(ok);
},
(err) => {
res.status(400).json({ error: String(err) });
return;
return res.status(400).json({ error: String(err) });
}
);
}

View file

@ -1,30 +1,9 @@
import { NixStoreManagerPresentation } from "../../features/nix_store_manager/nix_store_manager";
import { PipelinePresentation } from "../../features/pipelines/pipeline_presentation";
import { ProcessPresentation } from "../../features/process/process_presentation";
import {
ProjectInstancePresentation,
RobossemblerAssetsPresentation,
} from "../../features/project_instance/project_instance_presentation";
import { DatasetsPresentation } from "../../features/datasets/datasets_presentation";
import { ProjectsPresentation } from "../../features/projects/projects_presentation";
import { RealTimePresentation } from "../../features/realtime/realtime_presentation";
import { TriggerPresentation } from "../../features/triggers/triggers_presentation";
// import { ProjectsPresentation } from "../../features/_projects/projects_presentation";
import { extensions } from "../extensions/extensions";
import { Routes } from "../interfaces/router";
extensions();
export const routersImplementPureCrud = [
new TriggerPresentation(),
new ProjectsPresentation(),
new ProcessPresentation(),
new PipelinePresentation(),
];
export const httpRoutes: Routes[] = [
new RealTimePresentation(),
new ProjectInstancePresentation(),
new NixStoreManagerPresentation(),
new RobossemblerAssetsPresentation(),
]
.concat(routersImplementPureCrud)
.map((el) => el.call());
export const httpRoutes: Routes[] = [new ProjectsPresentation(), new DatasetsPresentation()].map((el) => el.call());

View file

@ -32,6 +32,3 @@ export const StringExtensions = () => {
};
}
};
// python3 /Users/idontsudo/framework/path.py --path /Users/idontsudo/webservice/server/build/public/0a3422cc-f2e3-4abc-87d8-ae13b8b6d26d/ --env /Users/idontsudo/framework/cad_generation/env.json
// python3 /Users/idontsudo/framework/path.py --path /Users/idontsudo/webservice/server/build/public/0a3422cc-f2e3-4abc-87d8-ae13b8b6d26d/ --env /Users/idontsudo/framework/cad_generation/env.json
// /Users/idontsudo/Desktop/FreeCAD.app/Contents/MacOS/FreeCAD /Users/idontsudo/framework/cad_generation/main.py

View file

@ -17,8 +17,6 @@ export interface WorkerDataExec {
process.on("message", async (message) => {
const workerData = message as WorkerDataExec;
if (workerData.type == WorkerType.SPAWN) {
// Maybe error
// const subprocess = cp.spawn(workerData.command, workerData.cliArgs, {
const subprocess = cp.spawn(workerData.command, {
cwd: workerData.execPath,
});

View file

@ -3,6 +3,19 @@ import { extensions } from "../extensions/extensions";
extensions();
export class ExecError extends Error {
id?:string;
script: string;
unixTime: number;
type = EXEC_TYPE.EXEC;
error: any;
constructor(script: string, ...args: any) {
super(...args);
this.script = script;
this.unixTime = Date.now();
this.error = args.firstElement();
}
static isExecError(e: any): ExecError | void {
try {
if (e) {
@ -14,16 +27,6 @@ export class ExecError extends Error {
console.log(error);
}
}
script: string;
unixTime: number;
type = EXEC_TYPE.EXEC;
error: any;
constructor(script: string, ...args: any) {
super(...args);
this.script = script;
this.unixTime = Date.now();
this.error = args.firstElement();
}
}
export class SpawnError extends Error {

View file

@ -4,6 +4,7 @@ export class ExecutorResult {
type: EXEC_TYPE;
event: EXEC_EVENT;
data: any;
id?:string
constructor(type: EXEC_TYPE, event: EXEC_EVENT, data: any) {
this.type = type;
this.event = event;

View file

@ -1,4 +1,4 @@
import { Trigger } from "../../features/triggers/models/trigger_database_model";
import { Trigger } from "../../features/_triggers/models/trigger_database_model";
import { EXEC_TYPE } from "./exec_error_model";
export interface IPipeline {

View file

@ -1,160 +1,23 @@
import { IsArray, IsEnum, IsNumber, IsOptional, IsString, ValidateNested } from "class-validator";
import { IsArray, IsString } from "class-validator";
import { Type } from "class-transformer";
export class Gravity {
@IsNumber()
x: number;
@IsNumber()
y: number;
@IsNumber()
z: number;
class Asset {
@IsString()
public name: string;
@IsString()
public mesh: string;
@IsString()
public image: string;
}
export class Pose {
@IsNumber()
x: number;
@IsNumber()
y: number;
@IsNumber()
z: number;
@IsNumber()
roll: number;
@IsNumber()
pitch: number;
@IsNumber()
yaw: number;
}
export class Position {
@IsNumber()
x: number;
@IsNumber()
y: number;
@IsNumber()
z: number;
}
export enum InstanceType {
RGB_CAMERA = "rgb_camera",
SCENE_SIMPLE_OBJECT = "scene_simple_object",
}
abstract class CoreInstances {}
export class Instance extends CoreInstances {
@IsEnum(InstanceType)
instanceType: InstanceType;
@Type(() => Position)
position: Position;
@IsArray()
quaternion: number[];
@IsOptional()
@IsString()
instanceAt: null | string = null;
}
export class SceneSimpleObject extends Instance {}
export class InstanceRgbCamera extends Instance {
@IsString()
cameraLink: string;
@IsString()
topicCameraInfo: string;
@IsOptional()
@IsString()
topicDepth: string | null;
@IsString()
topicImage: string;
}
export class Asset {
@IsString()
name: string;
@IsString()
ixx: string;
@IsString()
ixy: string;
@IsString()
ixz: string;
@IsString()
iyy: string;
@IsString()
izz: string;
@IsString()
mass: string;
@IsString()
posX: string;
@IsString()
posY: string;
@IsString()
posZ: string;
@IsString()
eulerX: string;
@IsString()
eulerY: string;
@IsString()
eulerZ: string;
@IsString()
iyz: string;
@IsString()
meshPath: string;
@IsString()
friction: string;
@IsString()
centerMassX: string;
@IsString()
centerMassY: string;
@IsString()
centerMassZ: string;
}
export class Physics {
@IsString()
engine_name: string;
@Type(() => Gravity)
gravity: Gravity;
}
export class RobossemblerAssets {
@ValidateNested()
@IsArray()
@Type(() => Asset)
assets: Asset[];
@IsArray()
@Type(() => Instance, {
discriminator: {
property: "type",
subTypes: [
{ value: InstanceRgbCamera, name: InstanceType.RGB_CAMERA },
{ value: SceneSimpleObject, name: InstanceType.SCENE_SIMPLE_OBJECT },
],
},
keepDiscriminatorProperty: true,
})
instances: Instance[];
@IsOptional()
@ValidateNested()
@Type(() => Physics)
physics: Physics;
convertLocalPathsToServerPaths(server_address: string): RobossemblerAssets {
convertLocalPathsToServerPaths(serverAddress: string): RobossemblerAssets {
this.assets = this.assets.map((el) => {
el.meshPath = server_address + el.meshPath;
el.mesh = `${serverAddress}/${el.mesh.slice(2, el.mesh.length)}`;
el.image = `${serverAddress}/${el.image.slice(2, el.image.length)}`;
return el;
});
return this;
}
getAssetPath(assetName: string): string {
const findElement = this.assets.find((el) => el.name === assetName);
if (findElement === undefined) {
throw new Error("RobossemblerAssets.getAssetPath not found asset by name:" + assetName);
}
return findElement.meshPath;
}
getAssetAtInstance(instanceAt: string): Asset {
return this.assets.filter((el) => el.name === instanceAt)[0];
}
}

View file

@ -1,3 +1,4 @@
export enum StaticFiles {
robossembler_assets = "robossembler_assets.json",
assets = "/assets/assets.json",
}

View file

@ -1,5 +1,6 @@
import * as fs from "fs";
import { promisify } from "node:util";
import { rimraf } from 'rimraf'
export class FileSystemRepository {
public createDir = promisify(fs.mkdir);
@ -9,7 +10,7 @@ export class FileSystemRepository {
public stat = promisify(fs.stat);
public readFileAsync = promisify(fs.readFile);
public readdir = promisify(fs.readdir);
public deleteDirRecursive = rimraf;
async readFileAtBuffer(path: string): Promise<Buffer> {
if ((await this.lsStat(path)).isDirectory()) {
return (
@ -40,4 +41,5 @@ export class FileSystemRepository {
});
return filesToDir;
}
}

View file

@ -1,8 +1,8 @@
import {
IProjectInstanceModel,
ProjectInstanceDbModel,
} from "../../features/project_instance/models/project_instance_database_model";
import { pipelineRealTimeService } from "../../features/realtime/realtime_presentation";
} from "../../features/projects/models/project_instance_database_model";
import { pipelineRealTimeService } from "../../features/_realtime/realtime_presentation";
import { App } from "../controllers/app";
import { CreateFolderUseCase } from "../usecases/create_folder_usecase";
import { SearchDataBaseModelUseCase } from "../usecases/search_database_model_usecase";
@ -15,6 +15,10 @@ export class SetLastActivePipelineToRealTimeServiceScenario {
})
).fold(
async (projectModel) => {
if (projectModel.project === null) {
return;
}
const projectPath = App.staticFilesStoreDir() + projectModel.rootDir + "/";
await new CreateFolderUseCase().call(projectPath);
pipelineRealTimeService.setPipelineDependency(

View file

@ -25,7 +25,12 @@ export class ExecutorProgramService
this.maxTime = maxTime;
}
private async workerExecuted(command: string, workerType: WorkerType, args: Array<string> | undefined = undefined) {
private async workerExecuted(
command: string,
workerType: WorkerType,
args: Array<string> | undefined = undefined,
id: string | undefined = undefined
) {
try {
cluster.setupPrimary({
exec: __dirname + "/../helpers/worker_computed.js",
@ -49,18 +54,23 @@ export class ExecutorProgramService
if (spawnError instanceof SpawnError) {
this.emit(Result.error(spawnError));
this.worker = undefined;
return;
}
const execError = ExecError.isExecError(e);
if (execError instanceof ExecError) {
execError.id = id
this.emit(Result.error(execError));
this.worker = undefined;
return;
}
const executorResult = ExecutorResult.isExecutorResult(e);
if (executorResult instanceof ExecutorResult) {
executorResult.id = id
this.emit(Result.ok(executorResult));
this.worker = undefined;
return;
}
});
@ -78,14 +88,22 @@ export class ExecutorProgramService
console.log(error);
}
}
public async call(type: EXEC_TYPE, command: string, args: Array<string> | undefined = undefined): Promise<void> {
public deleteWorker() {
if (this.worker) this.worker.kill();
this.worker = undefined;
}
public async call(
type: EXEC_TYPE,
command: string,
args: Array<string> | undefined = undefined,
id: string | undefined = undefined
): Promise<void> {
if (type == EXEC_TYPE.EXEC) {
this.workerExecuted(command, WorkerType.EXEC);
this.workerExecuted(command, WorkerType.EXEC, undefined, id);
return;
}
this.workerExecuted(command, WorkerType.SPAWN, args);
this.workerExecuted(command, WorkerType.SPAWN, args, id);
return;
}

View file

@ -7,7 +7,7 @@ import { Result } from "../helpers/result";
import { ExecutorResult } from "../models/executor_result";
import { delay } from "../helpers/delay";
import { TriggerService } from "./trigger_service";
import { Trigger } from "../../features/triggers/models/trigger_database_model";
import { Trigger } from "../../features/_triggers/models/trigger_database_model";
export interface Iteration {
hashes: IHashesCache | null;

View file

@ -3,7 +3,7 @@ import { IHashesCache } from "./files_change_notifier_service";
import { EventsFileChanger } from "../models/meta_data_file_manager_model";
import { Result } from "../helpers/result";
import { TypedEvent } from "../helpers/typed_event";
import { Trigger, TriggerType } from "../../features/triggers/models/trigger_database_model";
import { Trigger, TriggerType } from "../../features/_triggers/models/trigger_database_model";
export class TriggerCallResult {
results: Array<TriggerSuccessResult | TriggerErrorReport>;

View file

@ -0,0 +1,12 @@
import { Result } from "../helpers/result";
import { FileSystemRepository } from "../repository/file_system_repository";
export class DeleteRecursiveFolderUseCase{
repository:FileSystemRepository = new FileSystemRepository()
call = async (path:string):Promise<Result<void,void>> =>{
console.log(path)
await this.repository.deleteDirRecursive(path)
return Result.ok()
}
}

View file

@ -0,0 +1,44 @@
import { CallbackStrategyWithEmpty } from "../controllers/http_controller";
import { Result } from "../helpers/result";
import { TypedEvent } from "../helpers/typed_event";
import { EXEC_TYPE, ExecError, SpawnError } from "../models/exec_error_model";
import { ExecutorResult } from "../models/executor_result";
import { ExecutorProgramService } from "../services/executor_program_service";
export const executorProgramService = new ExecutorProgramService("");
export class KillLastProcessUseCase extends CallbackStrategyWithEmpty {
call = async (): Promise<Result<undefined, string>> => {
executorProgramService.deleteWorker();
return Result.ok("ok");
};
}
export class IsHaveActiveProcessUseCase extends CallbackStrategyWithEmpty {
call = async (): Promise<Result<string, string>> => {
if (executorProgramService.worker === undefined) {
return Result.ok("process not work");
}
return Result.error("process is exists");
};
}
export class ExecProcessUseCase {
call = async (
path: string,
command: string,
id:string,
watcher?: TypedEvent<Result<ExecError | SpawnError, ExecutorResult>>
): Promise<Result<Error, string>> => {
try {
executorProgramService.execPath = path;
executorProgramService.on((event) => {
if (watcher) watcher.emit(event);
});
executorProgramService.call(EXEC_TYPE.EXEC, command, undefined ,id);
return Result.ok("ok");
} catch (error) {
return Result.error(error);
}
};
}

View file

@ -9,7 +9,15 @@ export class ReadByIdDataBaseModelUseCase<D> {
call = async (id: string): Promise<Result<Error, D>> => {
try {
const dbModel = this.databaseModel as any;
return Result.ok(await dbModel.findById(id));
const model = await dbModel.findById(id);
if (model === null) {
return Result.error(
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-expect-error
Error(`the database table ${this.databaseModel.modelName} does not contain an object with this ID`)
);
}
return Result.ok(model);
} catch (error) {
return Result.error(error);
}

View file

@ -7,10 +7,10 @@ export class SearchDataBaseModelUseCase<T> {
this.model = model;
}
call = async (findFilter: Partial<T>): Promise<Result<null, T>> => {
call = async (findFilter: Partial<T>, error: string = "not found database"): Promise<Result<string, T>> => {
const result = await this.model.findOne(findFilter);
if (result === null) {
return Result.error(null);
return Result.error(error);
} else {
return Result.ok(result);
}

View file

@ -4,7 +4,7 @@ import { EXEC_TYPE } from "../../core/models/exec_error_model";
import { ExecutorResult } from "../../core/models/executor_result";
import { IPipeline, IssueType, StackGenerateType } from "../../core/models/process_model";
import { StackService } from "../../core/services/stack_service";
import { TriggerType } from "../triggers/models/trigger_database_model";
import { TriggerType } from "../_triggers/models/trigger_database_model";
class NixStoreModel {}

View file

@ -1,7 +1,7 @@
import { Schema, model } from "mongoose";
import { IPipeline } from "../../../core/models/process_model";
import { schemaProcess } from "../../process/models/process_database_model";
import { triggerSchema } from "../../triggers/models/trigger_database_model";
import { schemaProcess } from "../../_process/models/process_database_model";
import { triggerSchema } from "../../_triggers/models/trigger_database_model";
export const PipelineSchema = new Schema({
process: {

View file

@ -1,8 +1,8 @@
import { IsOptional, ValidateNested } from "class-validator";
import { IPipeline, IProcess, StackGenerateType } from "../../../core/models/process_model";
import { Type } from "class-transformer";
import { ProcessModel } from "../../process/models/process_validation_model";
import { TriggerModelValidationModel } from "../../triggers/models/trigger_validation_model";
import { ProcessModel } from "../../_process/models/process_validation_model";
import { TriggerModelValidationModel } from "../../_triggers/models/trigger_validation_model";
export class PipelineModel implements IPipeline {
@ValidateNested()

View file

@ -1,6 +1,6 @@
import { IsMongoId, IsOptional } from "class-validator";
import { IProcess, StackGenerateType } from "../../../core/models/process_model";
import { TriggerModelValidationModel } from "../../triggers/models/trigger_validation_model";
import { TriggerModelValidationModel } from "../../_triggers/models/trigger_validation_model";
export class PipelineValidationModel {
@IsMongoId()

View file

@ -1,6 +1,5 @@
import { Schema, model } from "mongoose";
import { schemaPipeline } from "../../pipelines/models/pipeline_database_model";
import { PipelineValidationModel } from "../../pipelines/models/pipeline_validation_model";
import { PipelineValidationModel } from "../../_pipelines/models/pipeline_validation_model";
export interface IProjectModel {
_id?: string;
@ -11,15 +10,12 @@ export interface IProjectModel {
}
export const ProjectSchema = new Schema({
pipelines: {
type: Array<Schema.Types.ObjectId>,
ref: schemaPipeline,
autopopulate: true,
default: null,
},
description: {
type: String,
},
rootDir: {
type: String,
},
isActive: {
type: Boolean,
default: false,

View file

@ -0,0 +1,13 @@
import { CrudController } from "../../core/controllers/crud_controller";
import { ProjectDBModel } from "./models/project_database_model";
import { ProjectValidationModel } from "./models/project_validation_model";
export class ProjectsPresentation extends CrudController<ProjectValidationModel, typeof ProjectDBModel> {
constructor() {
super({
url: "project",
validationModel: ProjectValidationModel,
databaseModel: ProjectDBModel,
});
}
}

View file

@ -3,11 +3,8 @@ import { CallbackStrategyWithEmpty } from "../../../core/controllers/http_contro
import { Result } from "../../../core/helpers/result";
import { IPipeline } from "../../../core/models/process_model";
import { ReadByIdDataBaseModelUseCase } from "../../../core/usecases/read_by_id_database_model_usecase";
import { PipelineValidationModel } from "../../pipelines/models/pipeline_validation_model";
import {
IProjectInstanceModel,
ProjectInstanceDbModel,
} from "../../project_instance/models/project_instance_database_model";
import { PipelineValidationModel } from "../../_pipelines/models/pipeline_validation_model";
import { IProjectInstanceModel, ProjectInstanceDbModel } from "../../projects/models/project_instance_database_model";
import { pipelineRealTimeService } from "../realtime_presentation";
import { PipelineStatusUseCase } from "./pipeline_status_usecase";

View file

@ -11,4 +11,3 @@ export class TriggerPresentation extends CrudController<TriggerModelValidationMo
});
}
}
"".isEmpty();

View file

@ -0,0 +1,42 @@
import { CrudController } from "../../core/controllers/crud_controller";
import { IsHaveActiveProcessUseCase, KillLastProcessUseCase } from "../../core/usecases/exec_process_usecase";
import { CreateDataSetScenario } from "./domain/create_dataset_scenario";
import { DeleteDatasetUseCase } from "./domain/delete_dataset_use_case";
import { ExecDatasetProcessScenario } from "./domain/exec_process_scenario";
import { GetDatasetActiveProjectScenario } from "./domain/get_dataset_active_project_scenario";
import { DatasetDBModel } from "./models/dataset_database_model";
import { DatasetValidationModel } from "./models/dataset_validation_model";
export class DatasetsPresentation extends CrudController<DatasetValidationModel, typeof DatasetDBModel> {
constructor() {
super({
url: "datasets",
validationModel: DatasetValidationModel,
databaseModel: DatasetDBModel,
});
super.post(new CreateDataSetScenario().call);
super.get(new GetDatasetActiveProjectScenario().call);
super.delete(null)
this.subRoutes.push({
method: "POST",
subUrl: "exec",
fn: new ExecDatasetProcessScenario(),
});
this.subRoutes.push({
method: "GET",
subUrl: "is/running",
fn: new IsHaveActiveProcessUseCase(),
});
this.subRoutes.push({
method: "GET",
subUrl: "delete/process",
fn: new KillLastProcessUseCase(),
});
this.subRoutes.push({
method: "DELETE",
subUrl: "dataset",
fn: new DeleteDatasetUseCase()
})
}
}

View file

@ -0,0 +1,60 @@
import { ObjectId } from "mongoose";
import { CallbackStrategyWithValidationModel, ResponseBase } from "../../../core/controllers/http_controller";
import { Result } from "../../../core/helpers/result";
import { TypedEvent } from "../../../core/helpers/typed_event";
import { EXEC_EVENT, ExecError, SpawnError } from "../../../core/models/exec_error_model";
import { ExecutorResult } from "../../../core/models/executor_result";
import { SearchDataBaseModelUseCase } from "../../../core/usecases/search_database_model_usecase";
import { IProjectModel, ProjectDBModel } from "../../_projects/models/project_database_model";
import { DatasetDBModel } from "../models/dataset_database_model";
import { DatasetValidationModel, ProcessStatus } from "../models/dataset_validation_model";
export class ProcessWatcherAndDatabaseUpdateService extends TypedEvent<Result<ExecError | SpawnError, ExecutorResult>> {
databaseId: ObjectId;
constructor(databaseId: ObjectId) {
super();
this.databaseId = databaseId;
this.on((event) => this.lister(event));
}
lister(event: Result<ExecError | SpawnError, ExecutorResult>) {
event.fold(
async (success) => {
if (success.event == EXEC_EVENT.END) {
const dbModel = await DatasetDBModel.findById(this.databaseId);
if (dbModel !== null) {
dbModel.local_path;
dbModel.processStatus = ProcessStatus.END;
dbModel.processLogs = success.data;
await dbModel.save();
}
}
},
async (error) => {
const dbModel = await DatasetDBModel.findById(this.databaseId);
if (dbModel !== null) {
dbModel.processStatus = ProcessStatus.ERROR;
dbModel.processLogs = error.message;
await dbModel.save();
}
}
);
}
}
export class CreateDataSetScenario extends CallbackStrategyWithValidationModel<DatasetValidationModel> {
validationModel: DatasetValidationModel;
call = async (model: DatasetValidationModel): ResponseBase => {
return (
await new SearchDataBaseModelUseCase<IProjectModel>(ProjectDBModel).call({ isActive: true }, "no active projects")
).map(async (project) => {
model.processStatus = ProcessStatus.NEW;
model.local_path = project.rootDir;
model.projectId = project._id;
const d = new DatasetDBModel();
Object.assign(d, model);
await d.save();
return Result.ok("create dataset ok");
});
};
}

View file

@ -0,0 +1,19 @@
import { CallbackStrategyWithIdQuery, ResponseBase } from "../../../core/controllers/http_controller";
import { Result } from "../../../core/helpers/result";
import { DeleteDataBaseModelUseCase } from "../../../core/usecases/delete_database_model_usecase";
import { DeleteRecursiveFolderUseCase } from "../../../core/usecases/delete_recursive_folder_usecase";
import { ReadByIdDataBaseModelUseCase } from "../../../core/usecases/read_by_id_database_model_usecase";
import { MongoIdValidation } from "../../../core/validations/mongo_id_validation";
import { DatasetDBModel } from "../models/dataset_database_model";
import { IDatasetModel } from "../models/dataset_validation_model";
export class DeleteDatasetUseCase extends CallbackStrategyWithIdQuery {
idValidationExpression = new MongoIdValidation();
call = async (id: string): ResponseBase =>
(await new ReadByIdDataBaseModelUseCase<IDatasetModel>(DatasetDBModel).call(id)).map(async (model) =>
(await new DeleteRecursiveFolderUseCase().call(`${model.local_path}/${model.name}/`)).map(async () =>
(await new DeleteDataBaseModelUseCase(model).call(model._id)).map(() => Result.ok({ status: "delete dataset" }))
)
);
}

View file

@ -0,0 +1,27 @@
import { ObjectId } from "mongoose";
import { CallbackStrategyWithIdQuery, ResponseBase } from "../../../core/controllers/http_controller";
import { ExecProcessUseCase, IsHaveActiveProcessUseCase } from "../../../core/usecases/exec_process_usecase";
import { ReadByIdDataBaseModelUseCase } from "../../../core/usecases/read_by_id_database_model_usecase";
import { MongoIdValidation } from "../../../core/validations/mongo_id_validation";
import { DatasetDBModel } from "../models/dataset_database_model";
import { IDatasetModel } from "../models/dataset_validation_model";
import { ProcessWatcherAndDatabaseUpdateService } from "./create_dataset_scenario";
export class ExecDatasetProcessScenario extends CallbackStrategyWithIdQuery {
idValidationExpression = new MongoIdValidation();
call = async (id: string): ResponseBase => {
return (await new ReadByIdDataBaseModelUseCase<IDatasetModel>(DatasetDBModel).call(id)).map(async (model) => {
return (await new IsHaveActiveProcessUseCase().call()).map(async () => {
await DatasetDBModel.findById(id).updateOne({ processStatus: "RUN" });
console.log(`blenderproc run $PYTHON_BLENDER_PROC --cfg '${JSON.stringify(model)}'`);
return new ExecProcessUseCase().call(
`${model.project.rootDir}/`,
`blenderproc run $PYTHON_BLENDER_PROC --cfg '${JSON.stringify(model)}'`,
id,
new ProcessWatcherAndDatabaseUpdateService(id as unknown as ObjectId)
);
});
});
};
}

View file

@ -0,0 +1,15 @@
import { CallbackStrategyWithEmpty, ResponseBase } from "../../../core/controllers/http_controller";
import { Result } from "../../../core/helpers/result";
import { SearchDataBaseModelUseCase } from "../../../core/usecases/search_database_model_usecase";
import { IProjectModel, ProjectDBModel } from "../../_projects/models/project_database_model";
import { DatasetDBModel } from "../models/dataset_database_model";
export class GetDatasetActiveProjectScenario extends CallbackStrategyWithEmpty {
call = async (): ResponseBase => {
return (
await new SearchDataBaseModelUseCase<IProjectModel>(ProjectDBModel).call({ isActive: true }, "no active projects")
).map(async (project) => {
return Result.ok(await DatasetDBModel.find({ project: project._id }));
});
};
}

View file

@ -0,0 +1,50 @@
import { Mongoose, Schema, model } from "mongoose";
import { IDatasetModel } from "./dataset_validation_model";
import { projectSchema } from "../../_projects/models/project_database_model";
export const DatasetSchema = new Schema({
name: {
type: String,
},
local_path: {
type: String,
},
dataSetObjects: {
type: Array,
of: String,
},
formBuilder: {
type: Schema.Types.Mixed,
of: String,
},
unixTime: {
type: Number,
default: Date.now(),
},
neuralNetworkAction: {
type: String,
},
neuralNetworkName: {
type: String,
},
processStatus: {
type: String,
default: "none",
},
project: {
type: Schema.Types.ObjectId,
ref: projectSchema,
autopopulate: true,
require: true,
},
processLogs: {
type: String,
},
datasetType: {
type: String,
},
}).plugin(require("mongoose-autopopulate"));
export const datasetSchema = "Dataset";
export const DatasetDBModel = model<IDatasetModel>(datasetSchema, DatasetSchema);

View file

@ -0,0 +1,43 @@
import { Type } from "class-transformer";
import { IsArray, IsOptional, IsString, ValidateNested } from "class-validator";
import { IProjectModel } from "../../_projects/models/project_database_model";
export class FormBuilderValidationModel {
@IsString()
public result: string;
@IsString()
public context: string;
@IsArray()
public form: [];
}
export enum ProcessStatus {
END = "END",
ERROR = "ERROR",
NEW = "NEW",
}
export interface IDatasetModel {
_id?:string;
name: string;
local_path: string;
dataSetObjects: string[];
formBuilder: FormBuilderValidationModel;
processLogs: string;
processStatus: ProcessStatus;
project?: IProjectModel;
}
export class DatasetValidationModel implements IDatasetModel {
@IsString()
public name: string;
@IsArray()
public dataSetObjects: string[];
@ValidateNested()
@Type(() => FormBuilderValidationModel)
public formBuilder: FormBuilderValidationModel;
public local_path: string;
@IsOptional()
@IsString()
public processStatus: ProcessStatus;
public projectId: string;
public processLogs: string;
}

View file

@ -1,22 +0,0 @@
import { App } from "../../../core/controllers/app";
import { Result } from "../../../core/helpers/result";
import { CreateDataBaseModelUseCase } from "../../../core/usecases/create_database_model_usecase";
import { CreateFolderUseCase } from "../../../core/usecases/create_folder_usecase";
import { pipelineRealTimeService } from "../../realtime/realtime_presentation";
import { ProjectInstanceDbModel } from "../models/project_instance_database_model";
import { ProjectInstanceValidationModel } from "../models/project_instance_validation_model";
import { v4 as uuidv4 } from "uuid";
import { SetActiveProjectScenario } from "./set_active_project_use_scenario";
export class CreateNewProjectInstanceScenario {
call = async (): Promise<Result<Error, any>> => {
try {
// (await new SetActiveProjectScenario().call(id)).map(() => {
// return Result.ok({ status: "ok" });
// });
} catch (error) {
console.log(error);
return Result.error(error as Error);
}
};
}

View file

@ -1,33 +0,0 @@
import { CallbackStrategyWithEmpty, ResponseBase } from "../../../core/controllers/http_controller";
import { Result } from "../../../core/helpers/result";
import { RobossemblerAssets } from "../../../core/models/robossembler_assets";
import { StaticFiles } from "../../../core/models/static_files";
import { ReadingJsonFileAndConvertingToInstanceClassScenario } from "../../../core/scenarios/read_file_and_json_to_plain_instance_class_scenario";
import { GetServerAddressUseCase } from "../../../core/usecases/get_server_address_usecase";
import { PipelineStatusUseCase } from "../../realtime/domain/pipeline_status_usecase";
export class RobossemblerAssetsNetworkMapperScenario extends CallbackStrategyWithEmpty {
async call(): ResponseBase {
try {
const result = await new PipelineStatusUseCase().call();
return await result.map(async (activeInstanceModel) => {
return (
await new ReadingJsonFileAndConvertingToInstanceClassScenario(RobossemblerAssets).call(
`${activeInstanceModel.path}${StaticFiles.robossembler_assets}`
)
).map((robossemblerAssets) => {
return new GetServerAddressUseCase().call().map((address) => {
return Result.ok(
robossemblerAssets.convertLocalPathsToServerPaths(
`${address}/${activeInstanceModel.rootDir.pathNormalize()}`
)
);
});
});
});
} catch (error) {
return Result.error(error);
}
}
}

View file

@ -1,37 +0,0 @@
import { App } from "../../../core/controllers/app";
import { CallbackStrategyWithFileUpload, ResponseBase } from "../../../core/controllers/http_controller";
import { Result } from "../../../core/helpers/result";
import { IFile } from "../../../core/interfaces/file";
import { CreateDataBaseModelUseCase } from "../../../core/usecases/create_database_model_usecase";
import { CreateFileUseCase } from "../../../core/usecases/create_file_usecase";
import { CreateFolderUseCase } from "../../../core/usecases/create_folder_usecase";
import { MongoIdValidation } from "../../../core/validations/mongo_id_validation";
import { ProjectInstanceDbModel } from "../models/project_instance_database_model";
import { ProjectInstanceValidationModel } from "../models/project_instance_validation_model";
import { v4 as uuidv4 } from "uuid";
import { SetActiveProjectScenario } from "./set_active_project_use_scenario";
export class UploadCadFileToProjectScenario extends CallbackStrategyWithFileUpload {
checkingFileExpression: RegExp = RegExp(".FCStd");
idValidationExpression = new MongoIdValidation();
async call(file: IFile, id: string, description: string): ResponseBase {
const folderName = uuidv4() + "/";
const model = new ProjectInstanceValidationModel();
model["project"] = id;
model["description"] = description;
model["rootDir"] = folderName;
model["isActive"] = true;
return (await new CreateFolderUseCase().call(App.staticFilesStoreDir() + folderName)).map(async () =>
(await new CreateDataBaseModelUseCase(ProjectInstanceDbModel).call(model)).map(async (databaseModel) =>
(await new SetActiveProjectScenario().call(databaseModel.id)).map(async () =>
(await new CreateFileUseCase().call(App.staticFilesStoreDir() + folderName + file.name, file.data)).map(
() => {
return Result.ok("ok");
}
)
)
)
);
}
}

View file

@ -1,49 +0,0 @@
import { CrudController } from "../../core/controllers/crud_controller";
import { CoreHttpController } from "../../core/controllers/http_controller";
import { RobossemblerAssets } from "../../core/models/robossembler_assets";
import { CreateNewProjectInstanceScenario } from "./domain/create_new_project_scenario";
import { RobossemblerAssetsNetworkMapperScenario } from "./domain/robossembler_assets_network_mapper_scenario";
import { SaveActiveSceneScenario } from "./domain/save_active_scene_scenario";
import { SetActiveProjectScenario } from "./domain/set_active_project_use_scenario";
import { UploadCadFileToProjectScenario } from "./domain/upload_file_to_to_project_scenario";
import { ProjectInstanceDbModel } from "./models/project_instance_database_model";
import { ProjectInstanceValidationModel } from "./models/project_instance_validation_model";
export class ProjectInstancePresentation extends CrudController<
ProjectInstanceValidationModel,
typeof ProjectInstanceDbModel
> {
constructor() {
super({
validationModel: ProjectInstanceValidationModel,
url: "project_instance",
databaseModel: ProjectInstanceDbModel,
});
super.post(new CreateNewProjectInstanceScenario().call);
this.subRoutes.push({
method: "POST",
subUrl: "set/active/project",
fn: new SetActiveProjectScenario(),
});
this.subRoutes.push({
method: "POST",
subUrl: "upload",
fn: new UploadCadFileToProjectScenario(),
});
}
}
export class RobossemblerAssetsPresentation extends CoreHttpController<RobossemblerAssets> {
constructor() {
super({
url: "robossembler_assets",
validationModel: RobossemblerAssets,
});
super.get(new RobossemblerAssetsNetworkMapperScenario().call);
super.post(new SaveActiveSceneScenario().call);
}
}

View file

@ -0,0 +1,34 @@
import { App } from "../../../core/controllers/app";
import { Result } from "../../../core/helpers/result";
import { v4 as uuidv4 } from "uuid";
import { IsString } from "class-validator";
import { ProjectDBModel } from "../../_projects/models/project_database_model";
import { CreateDataBaseModelUseCase } from "../../../core/usecases/create_database_model_usecase";
import { CreateFolderUseCase } from "../../../core/usecases/create_folder_usecase";
export class ProjectValidationModel {
@IsString()
description: string;
}
export class CreateNewProjectInstanceScenario {
call = async (model: ProjectValidationModel): Promise<Result<any, any>> => {
try {
const projectFolder = uuidv4();
return (await new CreateFolderUseCase().call(App.staticFilesStoreDir() + projectFolder)).map(async (_) => {
for await (const el of await ProjectDBModel.find({ isActive: true })) {
el.isActive = false;
await el.save();
}
const projectDbModel = await new ProjectDBModel({
isActive: true,
rootDir: App.staticFilesStoreDir() + projectFolder,
description: model.description,
}).save();
return Result.ok({ id: projectDbModel._id });
});
} catch (error) {
return Result.error(error as Error);
}
};
}

View file

@ -0,0 +1,12 @@
import { CallbackStrategyWithEmpty, ResponseBase } from "../../../core/controllers/http_controller";
import { Result } from "../../../core/helpers/result";
import { SearchDataBaseModelUseCase } from "../../../core/usecases/search_database_model_usecase";
import { IProjectModel, ProjectDBModel } from "../../_projects/models/project_database_model";
export class GetActiveProjectScenario extends CallbackStrategyWithEmpty {
async call(): ResponseBase {
return (
await new SearchDataBaseModelUseCase<IProjectModel>(ProjectDBModel).call({ isActive: true }, "no active projects")
).map((model) => Result.ok({ id: model._id }));
}
}

View file

@ -0,0 +1,33 @@
import { CallbackStrategyWithEmpty, ResponseBase } from "../../../core/controllers/http_controller";
import { Result } from "../../../core/helpers/result";
import { RobossemblerAssets } from "../../../core/models/robossembler_assets";
import { StaticFiles } from "../../../core/models/static_files";
import { ReadingJsonFileAndConvertingToInstanceClassScenario } from "../../../core/scenarios/read_file_and_json_to_plain_instance_class_scenario";
import { GetServerAddressUseCase } from "../../../core/usecases/get_server_address_usecase";
import { ProjectDBModel } from "../../_projects/models/project_database_model";
export class RobossemblerAssetsNetworkMapperScenario extends CallbackStrategyWithEmpty {
async call(): ResponseBase {
const projectDbModel = await ProjectDBModel.findOne({ isActive: true });
if (projectDbModel === null) {
return Result.error("is dont active projects");
}
const { rootDir } = projectDbModel;
return new GetServerAddressUseCase().call().map(async (address) =>
(
await new ReadingJsonFileAndConvertingToInstanceClassScenario<RobossemblerAssets>(RobossemblerAssets).call(
rootDir + StaticFiles.assets
)
).map((model) => {
return Result.ok(
model.convertLocalPathsToServerPaths(
`${address}/${
rootDir.match(new RegExp(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gm))[0]
}/assets`
)
);
})
);
}
}

View file

@ -4,7 +4,7 @@ import { RobossemblerAssets } from "../../../core/models/robossembler_assets";
import { StaticFiles } from "../../../core/models/static_files";
import { ReadingJsonFileAndConvertingToInstanceClassScenario } from "../../../core/scenarios/read_file_and_json_to_plain_instance_class_scenario";
import { WriteFileSystemFileUseCase } from "../../../core/usecases/write_file_system_file_usecase";
import { PipelineStatusUseCase } from "../../realtime/domain/pipeline_status_usecase";
import { PipelineStatusUseCase } from "../../_realtime/domain/pipeline_status_usecase";
export class SaveActiveSceneScenario extends CallbackStrategyWithValidationModel<RobossemblerAssets> {
validationModel: RobossemblerAssets = new RobossemblerAssets();

View file

@ -12,24 +12,31 @@ export class SetActiveProjectScenario extends CallbackStrategyWithIdQuery {
idValidationExpression = new MongoIdValidation();
async call(id: string): ResponseBase {
const result = await new ReadByIdDataBaseModelUseCase<IProjectInstanceModel>(ProjectInstanceDbModel).call(id);
// id
try {
const result = await new ReadByIdDataBaseModelUseCase<IProjectInstanceModel>(ProjectInstanceDbModel).call(id);
// id
if (result.isFailure()) {
return result.forward();
}
const model = result.value;
if (result.isFailure()) {
return result.forward();
}
const model = result.value;
return await (
await new CreateFolderUseCase().call(App.staticFilesStoreDir() + model.rootDir)
).map(async () => {
model.isActive = true;
return (await new UpdateDataBaseModelUseCase(ProjectInstanceDbModel).call(model)).map(async (el) => {
// TODO(IDONTSUDO): move it to a separate UseCase
await ProjectInstanceDbModel.updateMany({ _id: { $ne: el._id }, isActive: { $eq: true } }, { isActive: false });
await new SetLastActivePipelineToRealTimeServiceScenario().call();
return Result.ok(`project ${id} is active`);
return await (
await new CreateFolderUseCase().call(App.staticFilesStoreDir() + model.rootDir)
).map(async () => {
model.isActive = true;
return (await new UpdateDataBaseModelUseCase(ProjectInstanceDbModel).call(model)).map(async (el) => {
// TODO(IDONTSUDO): move it to a separate UseCase
await ProjectInstanceDbModel.updateMany(
{ _id: { $ne: el._id }, isActive: { $eq: true } },
{ isActive: false }
);
await new SetLastActivePipelineToRealTimeServiceScenario().call();
return Result.ok(`project ${id} is active`);
});
});
});
} catch (error) {
return Result.error("SetActiveProjectScenario error:" + String(error));
}
}
}

View file

@ -0,0 +1,27 @@
import { CallbackStrategyWithFileUpload, ResponseBase } from "../../../core/controllers/http_controller";
import { IFile } from "../../../core/interfaces/file";
import { CreateFileUseCase } from "../../../core/usecases/create_file_usecase";
import { MongoIdValidation } from "../../../core/validations/mongo_id_validation";
import { IProjectInstanceModel } from "../models/project_instance_database_model";
import { ReadByIdDataBaseModelUseCase } from "../../../core/usecases/read_by_id_database_model_usecase";
import { ProjectDBModel } from "../../_projects/models/project_database_model";
import { ExecProcessUseCase } from "../../../core/usecases/exec_process_usecase";
export class UploadCadFileToProjectScenario extends CallbackStrategyWithFileUpload {
checkingFileExpression: RegExp = new RegExp(".FCStd");
idValidationExpression = new MongoIdValidation();
async call(file: IFile, id: string): ResponseBase {
return (await new ReadByIdDataBaseModelUseCase<IProjectInstanceModel>(ProjectDBModel).call(id)).map(
async (databaseModel) =>
(await new CreateFileUseCase().call(`${databaseModel.rootDir}/${file.name}`, file.data)).map(
async () =>
await new ExecProcessUseCase().call(
`${databaseModel.rootDir}/`,
'',
`python3 $PYTHON_BLENDER --path '${databaseModel.rootDir}/assets/'`
)
)
);
}
}

View file

@ -1,5 +1,5 @@
import { Schema, model } from "mongoose";
import { IProjectModel, projectSchema } from "../../projects/models/project_database_model";
import { IProjectModel, projectSchema } from "../../_projects/models/project_database_model";
export interface IProjectInstanceModel {
_id: string;
@ -10,12 +10,6 @@ export interface IProjectInstanceModel {
}
export const ProjectInstanceSchema = new Schema({
project: {
type: Schema.Types.ObjectId,
ref: projectSchema,
autopopulate: true,
default: null,
},
description: {
type: String,
},

View file

@ -1,9 +1,6 @@
import { IsMongoId, IsOptional, IsString } from "class-validator";
export class ProjectInstanceValidationModel {
@IsMongoId()
public project: string;
@IsString()
public description: string;

View file

@ -1,13 +1,41 @@
import { CrudController } from "../../core/controllers/crud_controller";
import { ProjectDBModel } from "./models/project_database_model";
import { ProjectValidationModel } from "./models/project_validation_model";
import { ProjectDBModel } from "../_projects/models/project_database_model";
import { CreateNewProjectInstanceScenario } from "./domain/create_new_project_scenario";
import { GetActiveProjectScenario } from "./domain/get_active_project_scenario";
import { RobossemblerAssetsNetworkMapperScenario } from "./domain/robossembler_assets_network_mapper_scenario";
import { SetActiveProjectScenario } from "./domain/set_active_project_use_scenario";
import { UploadCadFileToProjectScenario } from "./domain/upload_file_to_to_project_scenario";
import { ProjectInstanceValidationModel as ProjectsValidationModel } from "./models/project_instance_validation_model";
export class ProjectsPresentation extends CrudController<ProjectValidationModel, typeof ProjectDBModel> {
export class ProjectsPresentation extends CrudController<ProjectsValidationModel, typeof ProjectDBModel> {
constructor() {
super({
url: "project",
validationModel: ProjectValidationModel,
validationModel: ProjectsValidationModel,
url: "projects",
databaseModel: ProjectDBModel,
});
super.post(new CreateNewProjectInstanceScenario().call);
this.subRoutes.push({
method: "POST",
subUrl: "set/active/project",
fn: new SetActiveProjectScenario(),
});
this.subRoutes.push({
method: "GET",
subUrl: "get/active/project/id",
fn: new GetActiveProjectScenario(),
});
this.subRoutes.push({
method: "POST",
subUrl: "upload",
fn: new UploadCadFileToProjectScenario(),
});
this.subRoutes.push({
method: "GET",
subUrl: "assets",
fn: new RobossemblerAssetsNetworkMapperScenario(),
});
}
}

View file

@ -3,12 +3,10 @@ import { App } from "./core/controllers/app";
import { SocketSubscriber } from "./core/controllers/socket_controller";
import { extensions } from "./core/extensions/extensions";
import { httpRoutes } from "./core/controllers/routes";
import { pipelineRealTimeService } from "./features/realtime/realtime_presentation";
import { main } from "./p";
import { executorProgramService } from "./core/usecases/exec_process_usecase";
extensions();
const socketSubscribers = [new SocketSubscriber(pipelineRealTimeService, "realtime")];
const socketSubscribers = [new SocketSubscriber(executorProgramService, "realtime")];
new App(httpRoutes, socketSubscribers).listen();
main();

View file

@ -1,6 +1,6 @@
import { EXEC_TYPE } from "../../src/core/models/exec_error_model";
import { IPipeline, IssueType, StackGenerateType } from "../../src/core/models/process_model";
import { TriggerType } from "../../src/features/triggers/models/trigger_database_model";
import { TriggerType } from "../../src/features/_triggers/models/trigger_database_model";
export const mockSimplePipeline: IPipeline[] = [
{

View file

@ -1,7 +1,7 @@
import { EventsFileChanger, MetaDataFileManagerModel } from "../../src/core/models/meta_data_file_manager_model";
import { TriggerService } from "../../src/core/services/trigger_service";
import { TriggerType } from "../../src/features/triggers/models/trigger_database_model";
import { TriggerType } from "../../src/features/_triggers/models/trigger_database_model";
import { assert } from "../test";
abstract class TriggerTest {
abstract test(): Promise<boolean>;

View file

@ -14,7 +14,7 @@ import { UpdateDataBaseModelUseCaseTest } from "./usecases/update_database_model
import { PaginationDataBaseModelUseCaseTest } from "./usecases/pagination_database_model_usecase_test";
import { extensions } from "../src/core/extensions/extensions";
import { CrudControllerTest } from "./controllers/crud_controller_test";
import { TriggerPresentation } from "../src/features/triggers/triggers_presentation";
import { TriggerPresentation } from "../src/features/_triggers/triggers_presentation";
import { App, Environment, ServerStatus } from "../src/core/controllers/app";
import { httpRoutes } from "../src/core/controllers/routes";
import { DataBaseConnectUseCase } from "../src/core/usecases/database_connect_usecase";