a lot of refactos

2026-06-21 21:08:46 +00:00
parent 3066a54a4c
commit 4c9e989bda
16 changed files with 472 additions and 60 deletions
@@ -24,6 +24,11 @@ export class maestroServer {
        this.orchestrationState = MaestroState.IDLE
        this.simulationId = null
        this.agentIds = []
+        this.keyframeId = null
+        this.infraId = null
+        this.bigBangEpoch = null
+        this.resumeEpoch = null
+        this.pausedAt = null
    }

    getMaestroSettings() {
@@ -54,6 +59,18 @@ export class maestroServer {
        return(this.orchestrationState === MaestroState.LIVE)
    }

+    isPaused() {
+        return(this.orchestrationState === MaestroState.PAUSED)
+    }
+
+    simNow() {
+        if(this.bigBangEpoch === null) return(null)
+        if(this.resumeEpoch !== null) {
+            return(this.pausedAt + (performance.now() - this.resumeEpoch) / 1000)
+        }
+        return((performance.now() - this.bigBangEpoch) / 1000)
+    }
+
    async init() {
        const mysqlCfg = this.maestroConfig.mysql
        if(!mysqlCfg) {
@@ -111,6 +128,11 @@ export class maestroServer {
        this.orchestrationState = MaestroState.IDLE
        this.simulationId = null
        this.agentIds = []
+        this.keyframeId = null
+        this.infraId = null
+        this.bigBangEpoch = null
+        this.resumeEpoch = null
+        this.pausedAt = null
        this.prepareQuorum?.cancel()
    }

@@ -120,24 +142,58 @@ export class maestroServer {
        return(this.prepareQuorum.handleMessage(msg, chan))
    }

-    async publishLifecycle(eventType, payload) {
+    async publishLifecycle(eventType, payload, state = null) {
        if(!this.arenaCnx) throw(new Error('No arena Redis connection'))
        const { arenaChannel, senderId } = this.getMaestroSettings().lifecycle
+        const resolvedState = state ?? this.orchestrationStateFor(payload?.simulationId ?? this.simulationId)
        await this.arenaCnx.redisPublish(arenaChannel, {
            eventType,
            sender: senderId,
            payload,
        })
+        await this.publishSystemLifecycle(eventType, payload, resolvedState)
        if(this.debug) console.log(`[Maestro] Published ${eventType} simulationId=${payload.simulationId}`)
    }

+    async publishSystemLifecycle(eventType, payload, state) {
+        if(!this.systemCnx || !this.simRepo) return
+
+        const simulationId = payload?.simulationId
+        if(!simulationId) return
+
+        const ownersResult = await this.simRepo.listSimulationOwnerUuids(simulationId)
+        if(!ownersResult.ok || !ownersResult.ownerUuids.length) return
+
+        const maestro = this.maestroConfig.maestro ?? {}
+        const channelTemplate = maestro.systemLifecycleChannel ?? 'system:maestro:lifecycle:[UID]'
+        const senderId = maestro.senderId ?? 'maestro'
+        const msg = {
+            eventType,
+            sender: senderId,
+            payload: { ...payload, state },
+        }
+
+        for(const uid of ownersResult.ownerUuids) {
+            const chan = channelTemplate.replace(/\[UID\]/g, uid)
+            await this.systemCnx.redisPublish(chan, msg)
+        }
+        if(this.debug) {
+            console.log(`[Maestro] Published system ${eventType} state=${state} simulationId=${simulationId}`)
+        }
+    }
+
    async startSimulation(userUuid, payload) {
        if(!this.simRepo) return({ ok: false, err: 'Database not initialized' })
        if(!this.arenaGroom) return({ ok: false, err: 'No arena Redis connection' })
+
+        const simulationUuid = payload?.simulationUuid
+        if(this.isPaused()) {
+            return(this.resumeSimulation(userUuid, simulationUuid))
+        }
+
        if(!this.prepareQuorum) return({ ok: false, err: 'No prepare quorum (arena Redis not wired)' })
        if(!this.isIdle()) return({ ok: false, err: 'A simulation is already in progress' })

-        const simulationUuid = payload?.simulationUuid
        const infraId = payload?.infraId ?? null

        const access = await this.simRepo.validateSimulationAccess(userUuid, simulationUuid)
@@ -152,6 +208,8 @@ export class maestroServer {

        this.simulationId = simulationUuid
        this.agentIds = agentsResult.agents.map(a => a.id)
+        this.keyframeId = keyframeId
+        this.infraId = infraId
        this.orchestrationState = MaestroState.PREPARING

        const lifecyclePayload = {
@@ -166,30 +224,131 @@ export class maestroServer {

        await this.publishLifecycle('onYourMarks', lifecyclePayload)

-        const quorum = await readyWait
-        if(!quorum.ok) {
-            await this.arenaGroom.clearArena()
-            this.resetOrchestration()
-            return(quorum)
-        }
-
-        await this.publishLifecycle('bigBang', {
-            simulationId: this.simulationId,
-            agentIds: this.agentIds,
+        this.continueStartSimulation(simulationUuid, readyWait).catch(err => {
+            console.error('[Maestro] continueStartSimulation failed:', err)
        })

-        this.orchestrationState = MaestroState.LIVE
-        if(this.debug) console.log(`[Maestro] LIVE simulationId=${this.simulationId}`)
-
        return({
            ok: true,
            simulationId: this.simulationId,
            keyframeId,
            infraId,
            agentIds: this.agentIds,
+            resumed: false,
+            state: MaestroState.PREPARING,
        })
    }

+    async continueStartSimulation(simulationUuid, readyWait) {
+        try {
+            const quorum = await readyWait
+            if(!quorum.ok) {
+                if(this.orchestrationState === MaestroState.PREPARING && this.simulationId === simulationUuid) {
+                    await this.publishSystemLifecycle('simulationPrepareFailed', {
+                        simulationId: simulationUuid,
+                        err: quorum.err,
+                    }, MaestroState.IDLE)
+                    await this.arenaGroom.clearArena()
+                    this.resetOrchestration()
+                }
+                if(this.debug) {
+                    console.warn(`[Maestro] Prepare failed simulationId=${simulationUuid}: ${quorum.err}`)
+                }
+                return
+            }
+
+            if(this.orchestrationState !== MaestroState.PREPARING || this.simulationId !== simulationUuid) {
+                return
+            }
+
+            this.bigBangEpoch = performance.now()
+            this.resumeEpoch = null
+            this.pausedAt = null
+            this.orchestrationState = MaestroState.LIVE
+
+            await this.publishLifecycle('bigBang', {
+                simulationId: this.simulationId,
+                agentIds: this.agentIds,
+            })
+
+            if(this.debug) console.log(`[Maestro] LIVE simulationId=${this.simulationId}`)
+        } catch(err) {
+            console.error(`[Maestro] continueStartSimulation error simulationId=${simulationUuid}:`, err)
+            if(this.simulationId === simulationUuid && this.orchestrationState === MaestroState.PREPARING) {
+                await this.publishSystemLifecycle('simulationPrepareFailed', {
+                    simulationId: simulationUuid,
+                    err: err.message ?? 'Prepare failed',
+                }, MaestroState.IDLE)
+                await this.arenaGroom?.clearArena()
+                this.resetOrchestration()
+            }
+        }
+    }
+
+    async resumeSimulation(userUuid, simulationUuid) {
+        if(this.simulationId !== simulationUuid) {
+            return({ ok: false, err: 'Another simulation is paused' })
+        }
+
+        const access = await this.simRepo.validateSimulationAccess(userUuid, simulationUuid)
+        if(!access.ok) return(access)
+
+        await this.publishLifecycle('simulationResumed', {
+            simulationId: simulationUuid,
+            agentIds: [...this.agentIds],
+            t: this.pausedAt,
+        })
+
+        this.resumeEpoch = performance.now()
+        this.orchestrationState = MaestroState.LIVE
+        if(this.debug) console.log(`[Maestro] RESUMED at t=${this.pausedAt}, simulationId=${simulationUuid}`)
+
+        return({
+            ok: true,
+            simulationId: this.simulationId,
+            keyframeId: this.keyframeId,
+            infraId: this.infraId,
+            agentIds: [...this.agentIds],
+            resumed: true,
+            state: MaestroState.LIVE,
+        })
+    }
+
+    async pauseSimulation(userUuid, payload) {
+        if(!this.simRepo) return({ ok: false, err: 'Database not initialized' })
+
+        const simulationUuid = payload?.simulationUuid
+        const access = await this.simRepo.validateSimulationAccess(userUuid, simulationUuid)
+        if(!access.ok) return(access)
+
+        if(this.simulationId !== simulationUuid) {
+            if(this.simulationId) return({ ok: false, err: 'Another simulation is active' })
+            return({ ok: false, err: 'Simulation is not running' })
+        }
+
+        if(this.orchestrationState === MaestroState.PAUSED) {
+            return({ ok: true, simulationId: simulationUuid, t: this.pausedAt })
+        }
+
+        if(this.orchestrationState !== MaestroState.LIVE && this.orchestrationState !== MaestroState.PREPARING) {
+            return({ ok: false, err: 'Simulation is not running' })
+        }
+
+        const t = this.orchestrationState === MaestroState.LIVE ? this.simNow() : 0
+
+        await this.publishLifecycle('simulationPaused', {
+            simulationId: simulationUuid,
+            agentIds: [...this.agentIds],
+            t,
+        })
+
+        this.pausedAt = t
+        this.orchestrationState = MaestroState.PAUSED
+        if(this.debug) console.log(`[Maestro] PAUSED at t=${t}, simulationId=${simulationUuid}`)
+
+        return({ ok: true, simulationId: simulationUuid, t })
+    }
+
    async stopSimulation(userUuid, payload) {
        if(!this.simRepo) return({ ok: false, err: 'Database not initialized' })
        if(!this.arenaGroom) return({ ok: false, err: 'No arena Redis connection' })
@@ -202,6 +361,26 @@ export class maestroServer {
            return({ ok: false, err: 'Another simulation is active' })
        }

+        if(this.simulationId === simulationUuid) {
+            let t = null
+            if(this.orchestrationState === MaestroState.LIVE) {
+                t = this.simNow()
+            } else if(this.orchestrationState === MaestroState.PREPARING) {
+                t = 0
+            } else if(this.orchestrationState === MaestroState.PAUSED) {
+                t = this.pausedAt
+            }
+            await this.publishLifecycle('simulationStopped', {
+                simulationId: simulationUuid,
+                agentIds: [...this.agentIds],
+                t,
+            }, MaestroState.IDLE)
+        }
+
+        if(this.simulationId === simulationUuid && this.orchestrationState === MaestroState.PREPARING) {
+            this.prepareQuorum?.abortPending({ ok: false, err: 'Simulation stopped' })
+        }
+
        await this.arenaGroom.clearArena()
        this.resetOrchestration()

@@ -210,6 +389,25 @@ export class maestroServer {
        return({ ok: true, simulationId: simulationUuid })
    }

+    orchestrationStateFor(simulationId) {
+        if(this.simulationId !== simulationId) return(MaestroState.IDLE)
+        return(this.orchestrationState)
+    }
+
+    async getSimulationsStatus(userUuid) {
+        if(!this.simRepo) return({ ok: false, err: 'Database not initialized' })
+
+        const listResult = await this.simRepo.listOwnerSimulations(userUuid)
+        if(!listResult.ok) return(listResult)
+
+        const simulations = listResult.simulationIds.map(simulationId => ({
+            simulationId,
+            state: this.orchestrationStateFor(simulationId),
+        }))
+
+        return({ ok: true, simulations })
+    }
+
    async reloadAccessRights() {
        await this.configHelper.refreshAccessRights()
        this.maestroConfig.accessRights = this.configHelper.config.accessRights