aboutsummaryrefslogtreecommitdiff
path: root/modules/core/module-orchestrator.js
diff options
context:
space:
mode:
Diffstat (limited to 'modules/core/module-orchestrator.js')
-rw-r--r--modules/core/module-orchestrator.js474
1 files changed, 440 insertions, 34 deletions
diff --git a/modules/core/module-orchestrator.js b/modules/core/module-orchestrator.js
index de88424..ec385bb 100644
--- a/modules/core/module-orchestrator.js
+++ b/modules/core/module-orchestrator.js
@@ -1,12 +1,22 @@
-// module-orchestrator.js — Multi-agent orchestration system
-// Commands: orchestrate, orchestrator-status, orchestrator-spawn, orchestrator-kill, orchestrator-purge
+// module-orchestrator.js — Multi-agent orchestration system with A2A mailbox transport
+// Commands: orchestrate, orchestrator-status, orchestrator-spawn, orchestrator-kill,
+// orchestrator-purge, orchestrator-config, orchestrator-list, orchestrator-sleep,
+// orchestrator-wake, orchestrator-restart-policy, a2a-send, a2a-list
class OrchestratorModule {
constructor() {
this.commands = {};
this.terminal = null;
this._agents = {};
- this._SPAWN_LIMIT = 3;
+ this._SPAWN_LIMIT = 5;
+ this._SPAWN_STRATEGY = 'eager';
+ this._HEARTBEAT_TIMEOUT = 15000;
+ this._IDLE_TIMEOUT = 120000;
+ this._RESTART_POLICY = 'never';
+ this._A2A_POLLING = true;
+ this._a2aPollTimer = null;
this._heartbeatInterval = null;
+ this._a2aMessages = {};
+ this._restartBackoffs = {};
}
registerCommands() {
this.commands['orchestrate'] = (args) => this.delegateTask(args.join(' '));
@@ -14,6 +24,29 @@ class OrchestratorModule {
this.commands['orchestrator-spawn'] = (args) => this.spawnAgent(args[0], args.slice(1));
this.commands['orchestrator-kill'] = (args) => this.killAgent(args[0]);
this.commands['orchestrator-purge'] = () => this.purgeDeadAgents();
+ this.commands['orchestrator-config'] = (args) => this.setConfig(args);
+ this.commands['orchestrator-list'] = () => this.listAgents();
+ this.commands['orchestrator-sleep'] = (args) => this.sleepAgent(args[0]);
+ this.commands['orchestrator-wake'] = (args) => this.wakeAgent(args[0]);
+ this.commands['orchestrator-restart-policy'] = (args) => this.setRestartPolicy(args);
+ this.commands['a2a-send'] = (args) => this.sendA2AMessage(args);
+ this.commands['a2a-list'] = () => this.listA2AMessages();
+ }
+ getHelpInfo() {
+ return {
+ 'orchestrate': 'orchestrate <task> — Delegate a task to an available agent (auto-spawns if needed)',
+ 'orchestrator-status': 'orchestrator-status — Show agent pool status and config',
+ 'orchestrator-spawn': 'orchestrator-spawn [role] [capabilities...] — Manually spawn a new agent',
+ 'orchestrator-kill': 'orchestrator-kill <agent-id> — Terminate an agent and revoke leases',
+ 'orchestrator-purge': 'orchestrator-purge — Remove dead agents (>60s since heartbeat)',
+ 'orchestrator-config': 'orchestrator-config <key> <value> — Tune: spawn-limit, spawn-strategy, heartbeat-timeout, idle-timeout, restart-policy, a2a-polling',
+ 'orchestrator-list': 'orchestrator-list — List all spawned agents with full details',
+ 'orchestrator-sleep': 'orchestrator-sleep <agent-id> — Suspend agent to VFS',
+ 'orchestrator-wake': 'orchestrator-wake <agent-id> — Restore agent from VFS sleep',
+ 'orchestrator-restart-policy': 'orchestrator-restart-policy <agent-id> <on-failure|always|never> — Set per-agent restart policy',
+ 'a2a-send': 'a2a-send <agent-id> <json-message> — Send an A2A message to an agent\'s inbox',
+ 'a2a-list': 'a2a-list — Show pending A2A messages across all agents',
+ };
}
init(terminal) {
this.terminal = terminal;
@@ -22,10 +55,129 @@ class OrchestratorModule {
if (registry) {
registry.registerAgent(['register', 'orchestrator-0', 'delegate', 'spawn', 'monitor', 'purge']);
}
+ // Reanimate sleeping agents from VFS
+ this._reanimateFromVFS();
+ // A2A mailbox polling loop (500ms)
+ if (this._A2A_POLLING) {
+ this._a2aPollTimer = setInterval(() => this._pollAgentMailboxes(), 500);
+ }
// Heartbeat watchdog every 3 seconds
this._heartbeatInterval = setInterval(() => this._checkHeartbeats(), 3000);
- this.terminal.print('🧠 Orchestrator loaded — ready to delegate', 'success');
+ // Idle timeout sweep every 30s
+ setInterval(() => this._sweepIdle(), 30000);
+ this.terminal.print('🧠 Orchestrator loaded — A2A mailbox transport active', 'success');
+ }
+
+ // ─── A2A Mailbox Transport ───
+
+ sendA2AMessage(args) {
+ if (args.length < 2) return this.terminal.print('Usage: a2a-send <agent-id> <json-message>', 'error');
+ const agentId = args[0];
+ let payload;
+ try {
+ payload = JSON.parse(args.slice(1).join(' '));
+ } catch (e) {
+ // Treat as plain text message
+ payload = { type: 'request', payload: { text: args.slice(1).join(' ') } };
+ }
+ const agent = this._agents[agentId];
+ if (!agent) return this.terminal.print(`❌ Agent '${agentId}' not found`, 'error');
+
+ const msg = {
+ a2a_version: '0.1',
+ msg_id: `msg-${Date.now()}-${Math.random().toString(36).substr(2, 4)}`,
+ type: payload.type || 'request',
+ from: 'orchestrator-0',
+ to: agentId,
+ timestamp: new Date().toISOString(),
+ ttl_seconds: 30,
+ contract: agent.capabilities ? { capabilities: agent.capabilities } : {},
+ payload: payload.payload || payload
+ };
+
+ // Write to agent's VFS inbox
+ if (!this.terminal.fs.readFile(`/agents/${agentId}/`)) this.terminal.fs.mkdir(`agents/${agentId}`);
+ this.terminal.fs.writeFile(`agents/${agentId}/inbox.json`, JSON.stringify(msg));
+
+ // Track in memory
+ if (!this._a2aMessages[agentId]) this._a2aMessages[agentId] = [];
+ this._a2aMessages[agentId].push({ ...msg, _processed: false });
+
+ this.terminal.print(`📨 A2A message sent to ${agentId} (${msg.type}, id: ${msg.msg_id})`, 'success');
+ }
+
+ listA2AMessages() {
+ const ids = Object.keys(this._a2aMessages);
+ if (ids.length === 0) return this.terminal.print('No A2A messages', 'info');
+ this.terminal.print('📨 A2A Messages:', 'header');
+ for (const agentId of ids) {
+ const msgs = this._a2aMessages[agentId];
+ const pending = msgs.filter(m => !m._processed).length;
+ this.terminal.print(` ${agentId}: ${msgs.length} total (${pending} pending)`, 'info');
+ for (const m of msgs.slice(-3)) {
+ const status = m._processed ? '✅' : '⏳';
+ this.terminal.print(` ${status} ${m.msg_id} ${m.type} → ${m.to} (${(m.payload?.text || JSON.stringify(m.payload)).substring(0, 40)})`, 'text-muted');
+ }
+ }
+ }
+
+ _pollAgentMailboxes() {
+ const fs = this.terminal.fs;
+ if (!fs) return;
+ for (const [id, agent] of Object.entries(this._agents)) {
+ try {
+ const inbox = fs.fsReadJSON(`/agents/${id}/inbox.json`);
+ if (!inbox) continue;
+ // Check if this message is already processed
+ const existing = this._a2aMessages[id] || [];
+ if (existing.find(m => m.msg_id === inbox.msg_id && m._processed)) continue;
+
+ // Route based on message type
+ if (inbox.a2a_version && inbox.type) {
+ inbox._processed = true;
+ if (!this._a2aMessages[id]) this._a2aMessages[id] = [];
+ this._a2aMessages[id].push(inbox);
+
+ if (inbox.type === 'request') {
+ this.terminal.print(`📨 A2A: ${inbox.from} → ${id} (${inbox.msg_id})`, 'info');
+ // Check outbox for response
+ this._checkAgentOutbox(id);
+ } else if (inbox.type === 'broadcast') {
+ this.terminal.print(`📢 A2A broadcast from ${inbox.from}: ${(inbox.payload?.text || JSON.stringify(inbox.payload)).substring(0, 60)}`, 'info');
+ } else if (inbox.type === 'heartbeat') {
+ agent.lastHeartbeat = Date.now();
+ } else if (inbox.type === 'response') {
+ this.terminal.print(`📨 A2A response from ${id}: ${(inbox.payload?.result || JSON.stringify(inbox.payload)).substring(0, 60)}`, 'success');
+ agent.status = 'idle';
+ agent.currentTask = null;
+ }
+ }
+ } catch (e) { /* skip agent with bad inbox */ }
+ }
+ }
+
+ _checkAgentOutbox(agentId) {
+ try {
+ const outbox = this.terminal.fs.fsReadJSON(`/agents/${agentId}/outbox.json`);
+ if (outbox && outbox.status === 'completed') {
+ const msg = {
+ a2a_version: '0.1',
+ msg_id: `resp-${Date.now()}`,
+ type: 'response',
+ from: agentId,
+ to: 'orchestrator-0',
+ timestamp: new Date().toISOString(),
+ payload: outbox
+ };
+ if (!this._a2aMessages['orchestrator-0']) this._a2aMessages['orchestrator-0'] = [];
+ this._a2aMessages['orchestrator-0'].push(msg);
+ this.terminal.print(`📨 A2A: ${agentId} completed task (outbox)`, 'success');
+ }
+ } catch (e) {}
}
+
+ // ─── Agent Lifecycle ───
+
spawnAgent(role, capabilities) {
const id = `agent-${Object.keys(this._agents).length + 1}-${role || 'worker'}`;
if (Object.keys(this._agents).length >= this._SPAWN_LIMIT) {
@@ -39,15 +191,24 @@ class OrchestratorModule {
createdAt: new Date().toISOString(),
lastHeartbeat: Date.now(),
status: 'idle',
- currentTask: null
+ currentTask: null,
+ leaseId: null,
+ restartPolicy: this._RESTART_POLICY,
+ failureCount: 0,
+ restartCount: 0,
+ nextRestartAt: null
};
this._agents[id] = agent;
+ // Create VFS agent directory with status
+ if (!this.terminal.fs.readFile(`/agents/${id}/`)) this.terminal.fs.mkdir(`agents/${id}`);
+ this.terminal.fs.writeFile(`agents/${id}/status.json`, JSON.stringify(agent));
// Register in agent registry
const registry = this.terminal.moduleManager.modules['AgentRegistryModule'];
if (registry) registry.registerAgent(['register', id, ...agent.capabilities]);
this.terminal.print(`🤖 Agent '${id}' spawned (role: ${role || 'worker'}, ${agent.capabilities.length} caps)`, 'success');
return agent;
}
+
killAgent(id) {
if (!this._agents[id]) return this.terminal.print(`❌ Agent '${id}' not found`, 'error');
// Revoke any active lease
@@ -60,31 +221,212 @@ class OrchestratorModule {
// Unregister from registry
const registry = this.terminal.moduleManager.modules['AgentRegistryModule'];
if (registry) registry.unregisterAgent(id);
+ // Clean up VFS
+ this.terminal.fs.deleteFile('agents/' + id);
delete this._agents[id];
+ delete this._restartBackoffs[id];
this.terminal.print(`💀 Agent '${id}' killed`, 'warning');
}
+
+ sleepAgent(id) {
+ if (!this._agents[id]) return this.terminal.print(`❌ Agent '${id}' not found`, 'error');
+ const agent = this._agents[id];
+ // Serialize to VFS
+ const state = {
+ ...agent,
+ _sleepTime: new Date().toISOString()
+ };
+ delete state.leaseId; // leases don't survive sleep
+ this.terminal.fs.writeFile(`agents/${id}/sleep.json`, JSON.stringify(state));
+ agent.status = 'sleeping';
+ this.terminal.print(`💤 Agent '${id}' sleeping (state saved to VFS)`, 'warning');
+ }
+
+ wakeAgent(id) {
+ const data = this.terminal.fs.fsReadJSON(`/agents/${id}/sleep.json`);
+ if (!data) return this.terminal.print(`❌ No saved state for '${id}' — use orchestrator-spawn instead`, 'error');
+ // Restore agent from VFS
+ const agent = {
+ ...data,
+ status: 'idle',
+ currentTask: null,
+ leaseId: null,
+ lastHeartbeat: Date.now()
+ };
+ delete agent._sleepTime;
+ this._agents[id] = agent;
+ this.terminal.fs.writeFile(`agents/${id}/status.json`, JSON.stringify({ ...agent, status: 'idle' }));
+ this.terminal.fs.deleteFile(`agents/${id}/sleep.json`);
+ this.terminal.print(`🔄 Agent '${id}' reanimated from VFS sleep`, 'success');
+ }
+
+ _reanimateFromVFS() {
+ const fs = this.terminal.fs;
+ const agentsDir = fs.readDir('/agents/');
+ if (!agentsDir) return;
+ let reanimated = 0;
+ for (const entry of agentsDir) {
+ if (entry.type !== 'dir') continue;
+ const sleepData = fs.fsReadJSON(`/agents/${entry.name}/sleep.json`);
+ if (sleepData) {
+ const agent = {
+ ...sleepData,
+ status: 'idle',
+ currentTask: null,
+ leaseId: null,
+ lastHeartbeat: Date.now()
+ };
+ delete agent._sleepTime;
+ this._agents[entry.name] = agent;
+ fs.writeFile(`agents/${entry.name}/status.json`, JSON.stringify({ ...agent, status: 'idle' }));
+ reanimated++;
+ }
+ }
+ if (reanimated > 0) {
+ this.terminal.print(`🔄 Reanimated ${reanimated} agent(s) from VFS sleep`, 'info');
+ }
+ }
+
+ // ─── Restart Policies ───
+
+ setRestartPolicy(args) {
+ if (args.length < 2) return this.terminal.print('Usage: orchestrator-restart-policy <agent-id> <on-failure|always|never>', 'error');
+ const id = args[0];
+ const policy = args[1];
+ if (!['on-failure', 'always', 'never'].includes(policy)) {
+ return this.terminal.print('Policy must be: on-failure, always, or never', 'error');
+ }
+ if (id === 'default') {
+ this._RESTART_POLICY = policy;
+ return this.terminal.print(`✅ Default restart policy set to '${policy}'`, 'success');
+ }
+ const agent = this._agents[id];
+ if (!agent) return this.terminal.print(`❌ Agent '${id}' not found`, 'error');
+ agent.restartPolicy = policy;
+ this.terminal.print(`✅ Restart policy for '${id}' set to '${policy}'`, 'success');
+ }
+
+ _handleAgentFailure(id, error) {
+ const agent = this._agents[id];
+ if (!agent) return;
+ agent.failureCount = (agent.failureCount || 0) + 1;
+ agent.lastError = error;
+ agent.lastErrorAt = new Date().toISOString();
+
+ let shouldRestart = false;
+ if (agent.restartPolicy === 'always') {
+ shouldRestart = true;
+ } else if (agent.restartPolicy === 'on-failure' && agent.failureCount <= 1) {
+ shouldRestart = true;
+ }
+
+ if (shouldRestart) {
+ const backoff = Math.min(Math.pow(2, agent.failureCount), 60); // 2^N capped at 60s
+ agent.nextRestartAt = Date.now() + (backoff * 1000);
+ agent.restartCount = (agent.restartCount || 0) + 1;
+ this._restartBackoffs[id] = setTimeout(() => {
+ if (this._agents[id]) {
+ this._agents[id].status = 'idle';
+ this._agents[id].currentTask = null;
+ this.terminal.print(`🔄 Agent '${id}' auto-restarted (backoff: ${backoff}s, attempt: ${agent.restartCount})`, 'info');
+ }
+ }, backoff * 1000);
+ this.terminal.print(`⚠️ Agent '${id}' failed — restart in ${backoff}s (policy: ${agent.restartPolicy})`, 'warning');
+ } else {
+ agent.status = 'failed';
+ this.terminal.print(`❌ Agent '${id}' failed — no restart (policy: ${agent.restartPolicy})`, 'error');
+ }
+ }
+
+ // ─── Autonomous Spawning ───
+
+ _deriveCapabilities(task) {
+ const caps = ['list_files', 'read_file'];
+ const lower = task.toLowerCase();
+ if (lower.includes('search') || lower.includes('find') || lower.includes('look up') || lower.includes('web')) caps.push('web_search');
+ if (lower.includes('write') || lower.includes('create') || lower.includes('save') || lower.includes('edit')) caps.push('write_file');
+ if (lower.includes('delete') || lower.includes('remove') || lower.includes('rm ')) caps.push('delete_file');
+ if (lower.includes('install') || lower.includes('package') || lower.includes('wpm')) caps.push('wpm_install');
+ if (lower.includes('module') || lower.includes('view') || lower.includes('edit')) caps.push('view_module', 'edit_module');
+ if (lower.includes('list') || lower.includes('status') || lower.includes('health')) caps.push('get_status');
+ if (lower.includes('fetch') || lower.includes('url') || lower.includes('http')) caps.push('web_fetch');
+ return [...new Set(caps)];
+ }
+
+ _deriveRole(task) {
+ const lower = task.toLowerCase();
+ if (lower.includes('search') || lower.includes('find')) return 'searcher';
+ if (lower.includes('write') || lower.includes('create') || lower.includes('save')) return 'writer';
+ if (lower.includes('delete') || lower.includes('remove') || lower.includes('clean')) return 'janitor';
+ if (lower.includes('install') || lower.includes('package')) return 'packager';
+ if (lower.includes('module') || lower.includes('code') || lower.includes('fix')) return 'coder';
+ if (lower.includes('health') || lower.includes('check') || lower.includes('audit')) return 'auditor';
+ if (lower.includes('fetch') || lower.includes('url') || lower.includes('http')) return 'fetcher';
+ return 'worker';
+ }
+
+ // ─── Config ───
+
+ setConfig(args) {
+ if (args.length < 2) return this.terminal.print('Usage: orchestrator-config <key> <value>', 'error');
+ const key = args[0];
+ const val = args[1];
+
+ const configMap = {
+ 'spawn-limit': () => { this._SPAWN_LIMIT = parseInt(val); return `spawn-limit set to ${val}`; },
+ 'spawn-strategy': () => { if (!['eager', 'lazy'].includes(val)) return 'Must be eager or lazy'; this._SPAWN_STRATEGY = val; return `spawn-strategy set to ${val}`; },
+ 'heartbeat-timeout': () => { this._HEARTBEAT_TIMEOUT = parseInt(val); return `heartbeat-timeout set to ${val}ms`; },
+ 'idle-timeout': () => { this._IDLE_TIMEOUT = parseInt(val); return `idle-timeout set to ${val}ms`; },
+ 'restart-policy': () => { if (!['on-failure', 'always', 'never'].includes(val)) return 'Must be on-failure, always, or never'; this._RESTART_POLICY = val; return `default restart-policy set to ${val}`; },
+ 'a2a-polling': () => {
+ this._A2A_POLLING = val === 'on' || val === 'true';
+ if (this._A2A_POLLING && !this._a2aPollTimer) {
+ this._a2aPollTimer = setInterval(() => this._pollAgentMailboxes(), 500);
+ } else if (!this._A2A_POLLING && this._a2aPollTimer) {
+ clearInterval(this._a2aPollTimer);
+ this._a2aPollTimer = null;
+ }
+ return `a2a-polling ${this._A2A_POLLING ? 'on' : 'off'}`;
+ }
+ };
+
+ if (!configMap[key]) {
+ return this.terminal.print(`Unknown config key: ${key}. Valid: ${Object.keys(configMap).join(', ')}`, 'error');
+ }
+ const result = configMap[key]();
+ this.terminal.print(`✅ ${result}`, 'success');
+ }
+
+ // ─── Task Delegation ───
+
async delegateTask(task) {
if (!task) return this.terminal.print('Usage: orchestrate <task description>', 'error');
- // Find an idle agent or auto-spawn
+
+ // Find an idle agent
let agent = Object.values(this._agents).find(a => a.status === 'idle');
+
+ // Auto-spawn with role-derived capabilities
if (!agent) {
if (Object.keys(this._agents).length >= this._SPAWN_LIMIT) {
- return this.terminal.print('❌ All agents busy and spawn limit reached', 'error');
+ // Try finding any non-failed agent to reuse
+ agent = Object.values(this._agents).find(a => a.status !== 'failed' && a.status !== 'sleeping');
+ }
+ if (!agent) {
+ const role = this._deriveRole(task);
+ const caps = this._deriveCapabilities(task);
+ this.spawnAgent(role, caps);
+ agent = Object.values(this._agents).find(a => a.status === 'idle');
}
- this.spawnAgent('worker', ['list_files', 'read_file', 'write_file', 'web_search']);
- agent = Object.values(this._agents).find(a => a.status === 'idle');
}
if (!agent) return this.terminal.print('❌ No available agents', 'error');
- // A2A Contract handshake — validate task against agent's contract
+ // A2A Contract handshake
const registry = this.terminal.moduleManager.modules['AgentRegistryModule'];
let contractValid = true;
if (registry) {
const result = registry.validateTask(agent.role, task);
if (!result.valid) {
this.terminal.print(`⚠️ A2A validation: ${result.reason}`, 'warning');
- this.terminal.print(` Contract for '${agent.role}' requires: ${(registry.getContract(agent.role) || {}).input_schema || 'free-form'}`, 'warning');
- // Offer to proceed anyway (permissive) — user can create a proper contract
contractValid = false;
} else if (result.reason === 'matched') {
this.terminal.print(`✅ A2A contract matched for role '${agent.role}'`, 'success');
@@ -94,52 +436,88 @@ class OrchestratorModule {
// Create a lease
const leaseModule = this.terminal.moduleManager.modules['LeaseModule'];
const lease = leaseModule ? leaseModule.createLease(['lease-create', agent.id, '60', task]) : null;
+
// Mark agent as busy
agent.status = 'busy';
agent.currentTask = task;
agent.leaseId = lease?.id;
- // Write task to agent's inbox
- if (!this.terminal.fs.readFile(`/agents/${agent.id}/`)) this.terminal.fs.mkdir(`agents/${agent.id}`);
- this.terminal.fs.writeFile(`agents/${agent.id}/inbox.json`, JSON.stringify({
- task,
- from: 'orchestrator-0',
- leaseId: lease?.id,
- timestamp: new Date().toISOString(),
- ttl_seconds: 60
- }));
- this.terminal.print(`🎯 Delegated "${task.substring(0, 60)}" to ${agent.id} (lease: ${lease?.id || 'none'})`, 'info');
+ agent.lastHeartbeat = Date.now();
+
+ // Send A2A request via inbox
+ this.sendA2AMessage([agent.id, JSON.stringify({
+ type: 'request',
+ payload: { task, from: 'orchestrator-0', leaseId: lease?.id }
+ })]);
+
+ this.terminal.print(`🎯 Delegated \"${task.substring(0, 60)}\" to ${agent.id} (lease: ${lease?.id || 'none'})`, 'info');
+
// Execute via AI assistant
try {
const aiModule = this.terminal.moduleManager.modules['WebX11AIAssistantModule'];
if (aiModule && typeof aiModule.chat === 'function') {
const result = await aiModule.chat([`[DELEGATED] ${task}`]);
agent.status = 'completed';
+
+ // Write result to outbox
this.terminal.fs.writeFile(`agents/${agent.id}/outbox.json`, JSON.stringify({
- result: result || 'completed',
+ result: typeof result === 'string' ? result : (result?.content || 'completed'),
status: 'completed',
timestamp: new Date().toISOString()
}));
+
+ // Record in memory graph
+ const memgraph = this.terminal.moduleManager.modules['MemoryGraphModule'];
+ if (memgraph) {
+ memgraph.recordTask(task, typeof result === 'string' ? result : (result?.content || ''), agent.id);
+ memgraph.recordFileAccess(`/agents/${agent.id}/outbox.json`);
+ }
+
this.terminal.print(`✅ ${agent.id} completed task`, 'success');
+
+ // Check restart policy — if always, set back to idle for next task
+ if (agent.restartPolicy === 'always') {
+ agent.status = 'idle';
+ agent.currentTask = null;
+ agent.lastHeartbeat = Date.now();
+ }
} else {
this.terminal.print(`ℹ️ AI module not available — task queued for ${agent.id}`, 'info');
}
} catch (e) {
- agent.status = 'failed';
- this.terminal.print(`❌ ${agent.id} failed: ${e.message}`, 'error');
+ this._handleAgentFailure(agent.id, e.message);
+ }
+
+ // Return to idle pool (unless failed/restarting)
+ if (agent.status === 'failed') {
+ // handled by _handleAgentFailure
+ } else if (agent.status !== 'busy') {
+ agent.status = 'idle';
+ agent.currentTask = null;
+ agent.lastHeartbeat = Date.now();
}
- // Return to idle pool
- agent.status = 'idle';
- agent.currentTask = null;
- agent.lastHeartbeat = Date.now();
}
+
+ // ─── Monitoring ───
+
_checkHeartbeats() {
const now = Date.now();
Object.entries(this._agents).forEach(([id, agent]) => {
- if (agent.status === 'busy' && now - agent.lastHeartbeat > 15000) {
+ if (agent.status === 'busy' && now - agent.lastHeartbeat > this._HEARTBEAT_TIMEOUT) {
this.terminal.print(`⚠️ Agent '${id}' unresponsive (${Math.round((now - agent.lastHeartbeat) / 1000)}s since heartbeat)`, 'warning');
}
});
}
+
+ _sweepIdle() {
+ const now = Date.now();
+ Object.entries(this._agents).forEach(([id, agent]) => {
+ if (agent.status === 'idle' && now - agent.lastHeartbeat > this._IDLE_TIMEOUT) {
+ this.terminal.print(`💤 Agent '${id}' idle for >${this._IDLE_TIMEOUT/1000}s — auto-sleeping`, 'info');
+ this.sleepAgent(id);
+ }
+ });
+ }
+
purgeDeadAgents() {
const now = Date.now();
let count = 0;
@@ -151,17 +529,45 @@ class OrchestratorModule {
});
this.terminal.print(`🧹 Purged ${count} dead agents`, count > 0 ? 'warning' : 'info');
}
+
+ // ─── Status & List ───
+
showStatus() {
const total = Object.keys(this._agents).length;
const busy = Object.values(this._agents).filter(a => a.status === 'busy').length;
const idle = Object.values(this._agents).filter(a => a.status === 'idle').length;
- const failed = Object.values(this._agents).filter(a => a.status === 'failed').length;
+ const sleeping = Object.values(this._agents).filter(a => a.status === 'sleeping').length;
+ const failed = Object.values(this._agents).filter(a => a.status === 'failed' || a.status === 'restarting').length;
+ const a2aTotal = Object.values(this._a2aMessages).reduce((s, msgs) => s + msgs.length, 0);
+ const a2aPending = Object.values(this._a2aMessages).reduce((s, msgs) => s + msgs.filter(m => !m._processed).length, 0);
+
this.terminal.print('🧠 Orchestrator Status', 'header');
- this.terminal.print(` Agents: ${total} (${busy} busy, ${idle} idle, ${failed} failed)`, 'info');
- this.terminal.print(` Spawn limit: ${this._SPAWN_LIMIT}`, 'info');
+ this.terminal.print(` Agents: ${total} (${busy} busy, ${idle} idle, ${sleeping} sleeping, ${failed} failed)`, 'info');
+ this.terminal.print(` Spawn limit: ${this._SPAWN_LIMIT} (strategy: ${this._SPAWN_STRATEGY})`, 'info');
+ this.terminal.print(` Default restart: ${this._RESTART_POLICY}`, 'info');
+ this.terminal.print(` A2A Messages: ${a2aTotal} (${a2aPending} pending)`, 'info');
Object.values(this._agents).forEach(a => {
const task = a.currentTask ? ` (task: ${a.currentTask.substring(0, 40)})` : '';
- this.terminal.print(` ${a.id} — ${a.status}${task}`, 'info');
+ const backoff = a.nextRestartAt && a.nextRestartAt > Date.now() ? ` [restart in ${Math.round((a.nextRestartAt - Date.now()) / 1000)}s]` : '';
+ this.terminal.print(` ${a.id} [${a.role}] — ${a.status}${task}${backoff}`, 'info');
+ });
+ }
+
+ listAgents() {
+ const all = Object.values(this._agents);
+ if (all.length === 0) return this.terminal.print('No agents spawned', 'info');
+ this.terminal.print(`📋 Agents (${all.length}):`, 'header');
+ all.forEach(a => {
+ const caps = (a.capabilities || []).join(', ');
+ this.terminal.print(` ${a.id} [${a.role}] — ${a.status}`, 'info');
+ this.terminal.print(` Caps: ${caps}`, 'text-muted');
+ this.terminal.print(` Policy: ${a.restartPolicy || this._RESTART_POLICY} | Failures: ${a.failureCount || 0} | Restarts: ${a.restartCount || 0}`, 'text-muted');
+ if (a.currentTask) this.terminal.print(` Task: ${a.currentTask.substring(0, 60)}`, 'text-muted');
});
}
+}
+
+// Required for module system
+if (typeof window !== 'undefined') {
+ window.OrchestratorModule = OrchestratorModule;
} \ No newline at end of file