using System; using System.IO; using System.Linq; using System.Reflection; using System.Timers; using log4net; using Mono.Addins; using Nini.Config; using OpenSim.Framework; using OpenSim.Framework.Console; using OpenSim.Region.Framework.Interfaces; using OpenSim.Region.Framework.Scenes; [assembly: Addin("RegionHealthMonitor", "1.0.0")] [assembly: AddinDependency("OpenSim.Region.Framework", OpenSim.VersionInfo.VersionNumber)] [assembly: AddinDescription("Region health polling + threshold warnings + CSV export + health event bus.")] [assembly: AddinAuthor("Christopher Händler")] namespace OpenSimRegionWarmupHealthGuard.Modules { // Simple health sample DTO public class HealthSample { public DateTime TimestampUtc; public string Region; public int Agents; public int Prims; public float ScriptTimeMs; public float PhysicsTimeMs; public float NetTimeMs; public int ScriptErrors; public TimeSpan Uptime; } public enum HealthSeverity { Trace, Info, Warn, Error } public class HealthEvent { public DateTime TimestampUtc; public string Region; public HealthSeverity Severity; public string Message; public HealthSample Sample; // optional snapshot of metrics } // Event-bus interface for other modules (alerts/export/auto-heal/policy) public interface IRegionHealthBus { event Action OnSample; // every poll event Action OnIncident; // only on rule/threshold violation void PublishSample(HealthSample s); void PublishIncident(HealthEvent e); } public class RegionHealthBus : IRegionHealthBus { public event Action OnSample; public event Action OnIncident; public void PublishSample(HealthSample s) => OnSample?.Invoke(s); public void PublishIncident(HealthEvent e) => OnIncident?.Invoke(e); } [Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "RegionHealthMonitor")] public class RegionHealthMonitorModule : INonSharedRegionModule { private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); private Scene _scene; private bool _enabled; // Config private int _healthIntervalSec = 30; private float _warnScriptTimeMs = 12.0f; private float _warnPhysicsTimeMs = 6.0f; private float _warnNetTimeMs = 6.0f; private int _warnScriptErrors = 10; private string _metricsExportFile = ""; // optional CSV export private Timer _healthTimer; private volatile int _opGuard; // Expose bus for other modules private readonly RegionHealthBus _bus = new RegionHealthBus(); public IRegionHealthBus Bus => _bus; public string Name => "RegionHealthMonitor"; public Type ReplaceableInterface => null; public void Initialise(IConfigSource source) { var modules = source.Configs["Modules"]; if (modules == null) return; var status = modules.GetString("RegionHealthMonitor", string.Empty); if (!string.Equals(status, "enabled", StringComparison.OrdinalIgnoreCase)) return; var cfg = source.Configs["RegionHealthMonitor"] ?? source.AddConfig("RegionHealthMonitor"); _healthIntervalSec = Math.Max(5, cfg.GetInt("HealthIntervalSec", _healthIntervalSec)); _warnScriptTimeMs = (float)cfg.GetDouble("WarnScriptTimeMs", _warnScriptTimeMs); _warnPhysicsTimeMs = (float)cfg.GetDouble("WarnPhysicsTimeMs", _warnPhysicsTimeMs); _warnNetTimeMs = (float)cfg.GetDouble("WarnNetTimeMs", _warnNetTimeMs); _warnScriptErrors = Math.Max(0, cfg.GetInt("WarnScriptErrors", _warnScriptErrors)); _metricsExportFile = cfg.GetString("MetricsExportFile", _metricsExportFile); _enabled = true; Log.Info("[HEALTH] RegionHealthMonitor enabled"); } public void AddRegion(Scene scene) { if (!_enabled) return; _scene = scene; // Make bus discoverable to other modules _scene.RegisterModuleInterface(_bus); _scene.RegisterModuleInterface(this); } public void RegionLoaded(Scene scene) { if (!_enabled) return; MainConsole.Instance.Commands.AddCommand("Region", false, "health", "health", "Health monitor commands. Usage: health status|export", HandleConsole); _healthTimer = new Timer(_healthIntervalSec * 1000) { AutoReset = true }; _healthTimer.Elapsed += (_, __) => SafeRun("HealthPoll", PollHealth); _healthTimer.Start(); Log.Info($"[HEALTH] Ready for region '{_scene.RegionInfo.RegionName}'. Interval={_healthIntervalSec}s."); } public void RemoveRegion(Scene scene) { if (!_enabled) return; _healthTimer?.Stop(); _healthTimer?.Dispose(); _healthTimer = null; _scene.UnregisterModuleInterface(_bus); _scene.UnregisterModuleInterface(this); _scene = null; } public void Close() { _healthTimer?.Stop(); _healthTimer?.Dispose(); _healthTimer = null; } private void HandleConsole(string module, string[] args) { if (_scene == null) { MainConsole.Instance.Output("[HEALTH] No active scene."); return; } if (args.Length < 2) { MainConsole.Instance.Output("health status - show current health"); MainConsole.Instance.Output("health export - export one CSV line (if MetricsExportFile configured)"); return; } switch (args[1].ToLowerInvariant()) { case "status": ReportHealth(toConsole: true); break; case "export": var path = ExportMetricsOnce(); MainConsole.Instance.Output(string.IsNullOrEmpty(path) ? "[HEALTH] Export disabled or failed." : $"[HEALTH] Exported metrics to: {path}"); break; default: MainConsole.Instance.Output("health status|export"); break; } } private void SafeRun(string tag, Action action) { if (System.Threading.Interlocked.Exchange(ref _opGuard, 1) == 1) return; try { action(); } catch (Exception e) { Log.Warn($"[HEALTH] {tag} failed: {e.Message}"); } finally { System.Threading.Interlocked.Exchange(ref _opGuard, 0); } } private void PollHealth() { var s = CollectStats(); // Publish every sample _bus.PublishSample(s); // Threshold checks -> incidents if (s.ScriptTimeMs >= _warnScriptTimeMs) PublishIncident(HealthSeverity.Warn, $"High script time: {s.ScriptTimeMs:0.00} ms", s); if (s.PhysicsTimeMs >= _warnPhysicsTimeMs) PublishIncident(HealthSeverity.Warn, $"High physics time: {s.PhysicsTimeMs:0.00} ms", s); if (s.NetTimeMs >= _warnNetTimeMs) PublishIncident(HealthSeverity.Warn, $"High network time: {s.NetTimeMs:0.00} ms", s); if (s.ScriptErrors >= _warnScriptErrors) PublishIncident(HealthSeverity.Warn, $"High script errors: {s.ScriptErrors}", s); ExportMetrics(s); } private void PublishIncident(HealthSeverity sev, string msg, HealthSample s) { var e = new HealthEvent { TimestampUtc = DateTime.UtcNow, Region = _scene.RegionInfo.RegionName, Severity = sev, Message = msg, Sample = s }; _bus.PublishIncident(e); if (sev >= HealthSeverity.Warn) Log.Warn($"[HEALTH] {msg}"); else Log.Info($"[HEALTH] {msg}"); } private void ReportHealth(bool toConsole) { var s = CollectStats(); var line = $"[Health] Agents={s.Agents} Prims={s.Prims} " + $"ScriptMs={s.ScriptTimeMs:0.00} PhysMs={s.PhysicsTimeMs:0.00} NetMs={s.NetTimeMs:0.00} " + $"ScriptErrors={s.ScriptErrors} Uptime={s.Uptime:hh\\:mm\\:ss}"; if (toConsole) MainConsole.Instance.Output(line); Log.Info(line); } private HealthSample CollectStats() { float scriptMs = 0f, physMs = 0f, netMs = 0f; try { var stats = _scene.StatsReporter; // 0.9.3.x provides this if (stats != null) { scriptMs = stats.ScriptMs; physMs = stats.PhysicsMs; netMs = stats.NetMs; } } catch { } int agents = 0, prims = 0, scriptErrors = 0; try { agents = _scene.GetRootAgentCount() + _scene.GetChildAgentCount(); prims = _scene.GetTotalObjectsCount(); } catch { } try { var engine = _scene.RequestModuleInterface(); if (engine != null) scriptErrors = engine.GetScriptErrorCount(); } catch { } TimeSpan uptime = TimeSpan.Zero; try { uptime = DateTime.UtcNow - _scene.RegionInfo.StartupTime; } catch { } return new HealthSample { TimestampUtc = DateTime.UtcNow, Region = _scene.RegionInfo.RegionName, Agents = agents, Prims = prims, ScriptTimeMs = scriptMs, PhysicsTimeMs = physMs, NetTimeMs = netMs, ScriptErrors = scriptErrors, Uptime = uptime }; } private void ExportMetrics(HealthSample s) { try { if (string.IsNullOrWhiteSpace(_metricsExportFile)) return; bool header = !File.Exists(_metricsExportFile); using (var sw = new StreamWriter(_metricsExportFile, append: true)) { if (header) sw.WriteLine("ts,region,agents,prims,script_ms,phys_ms,net_ms,script_errors,uptime_s"); sw.WriteLine($"{DateTime.UtcNow:O},{Csv(_scene.RegionInfo.RegionName)},{s.Agents},{s.Prims}," + $"{s.ScriptTimeMs:0.###},{s.PhysicsTimeMs:0.###},{s.NetTimeMs:0.###}," + $"{s.ScriptErrors},{(long)s.Uptime.TotalSeconds}"); } } catch (Exception e) { Log.Warn($"[HEALTH] CSV export failed: {e.Message}"); } } private string ExportMetricsOnce() { try { if (string.IsNullOrWhiteSpace(_metricsExportFile)) return null; var s = CollectStats(); ExportMetrics(s); return _metricsExportFile; } catch { return null; } } private string Csv(string s) => "\"" + s.Replace("\"", "\"\"") + "\""; } }