mirror of
https://github.com/jamenai/opensim-addon-modules-exp.git
synced 2026-07-28 05:12:50 +00:00
332 lines
12 KiB
C#
332 lines
12 KiB
C#
using System;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Reflection;
|
|
using System.Timers;
|
|
using log4net;
|
|
using Mono.Addins;
|
|
using Nini.Config;
|
|
using OpenSim.Framework;
|
|
using OpenSim.Framework.Console;
|
|
using OpenSim.Region.Framework.Interfaces;
|
|
using OpenSim.Region.Framework.Scenes;
|
|
|
|
[assembly: Addin("RegionHealthMonitor", "1.0.0")]
|
|
[assembly: AddinDependency("OpenSim.Region.Framework", OpenSim.VersionInfo.VersionNumber)]
|
|
[assembly: AddinDescription("Region health polling + threshold warnings + CSV export + health event bus.")]
|
|
[assembly: AddinAuthor("Christopher Händler")]
|
|
|
|
namespace OpenSimRegionWarmupHealthGuard.Modules
|
|
{
|
|
// Simple health sample DTO
|
|
public class HealthSample
|
|
{
|
|
public DateTime TimestampUtc;
|
|
public string Region;
|
|
public int Agents;
|
|
public int Prims;
|
|
public float ScriptTimeMs;
|
|
public float PhysicsTimeMs;
|
|
public float NetTimeMs;
|
|
public int ScriptErrors;
|
|
public TimeSpan Uptime;
|
|
}
|
|
|
|
public enum HealthSeverity { Trace, Info, Warn, Error }
|
|
|
|
public class HealthEvent
|
|
{
|
|
public DateTime TimestampUtc;
|
|
public string Region;
|
|
public HealthSeverity Severity;
|
|
public string Message;
|
|
public HealthSample Sample; // optional snapshot of metrics
|
|
}
|
|
|
|
// Event-bus interface for other modules (alerts/export/auto-heal/policy)
|
|
public interface IRegionHealthBus
|
|
{
|
|
event Action<HealthSample> OnSample; // every poll
|
|
event Action<HealthEvent> OnIncident; // only on rule/threshold violation
|
|
|
|
void PublishSample(HealthSample s);
|
|
void PublishIncident(HealthEvent e);
|
|
}
|
|
|
|
public class RegionHealthBus : IRegionHealthBus
|
|
{
|
|
public event Action<HealthSample> OnSample;
|
|
public event Action<HealthEvent> OnIncident;
|
|
|
|
public void PublishSample(HealthSample s) => OnSample?.Invoke(s);
|
|
public void PublishIncident(HealthEvent e) => OnIncident?.Invoke(e);
|
|
}
|
|
|
|
[Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "RegionHealthMonitor")]
|
|
public class RegionHealthMonitorModule : INonSharedRegionModule
|
|
{
|
|
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
|
|
|
|
private Scene _scene;
|
|
private bool _enabled;
|
|
|
|
// Config
|
|
private int _healthIntervalSec = 30;
|
|
private float _warnScriptTimeMs = 12.0f;
|
|
private float _warnPhysicsTimeMs = 6.0f;
|
|
private float _warnNetTimeMs = 6.0f;
|
|
private int _warnScriptErrors = 10;
|
|
private string _metricsExportFile = ""; // optional CSV export
|
|
|
|
private Timer _healthTimer;
|
|
private volatile int _opGuard;
|
|
|
|
// Expose bus for other modules
|
|
private readonly RegionHealthBus _bus = new RegionHealthBus();
|
|
public IRegionHealthBus Bus => _bus;
|
|
|
|
public string Name => "RegionHealthMonitor";
|
|
public Type ReplaceableInterface => null;
|
|
|
|
public void Initialise(IConfigSource source)
|
|
{
|
|
var modules = source.Configs["Modules"];
|
|
if (modules == null) return;
|
|
|
|
var status = modules.GetString("RegionHealthMonitor", string.Empty);
|
|
if (!string.Equals(status, "enabled", StringComparison.OrdinalIgnoreCase)) return;
|
|
|
|
var cfg = source.Configs["RegionHealthMonitor"] ?? source.AddConfig("RegionHealthMonitor");
|
|
|
|
_healthIntervalSec = Math.Max(5, cfg.GetInt("HealthIntervalSec", _healthIntervalSec));
|
|
_warnScriptTimeMs = (float)cfg.GetDouble("WarnScriptTimeMs", _warnScriptTimeMs);
|
|
_warnPhysicsTimeMs = (float)cfg.GetDouble("WarnPhysicsTimeMs", _warnPhysicsTimeMs);
|
|
_warnNetTimeMs = (float)cfg.GetDouble("WarnNetTimeMs", _warnNetTimeMs);
|
|
_warnScriptErrors = Math.Max(0, cfg.GetInt("WarnScriptErrors", _warnScriptErrors));
|
|
_metricsExportFile = cfg.GetString("MetricsExportFile", _metricsExportFile);
|
|
|
|
_enabled = true;
|
|
Log.Info("[HEALTH] RegionHealthMonitor enabled");
|
|
}
|
|
|
|
public void AddRegion(Scene scene)
|
|
{
|
|
if (!_enabled) return;
|
|
_scene = scene;
|
|
|
|
// Make bus discoverable to other modules
|
|
_scene.RegisterModuleInterface<IRegionHealthBus>(_bus);
|
|
_scene.RegisterModuleInterface(this);
|
|
}
|
|
|
|
public void RegionLoaded(Scene scene)
|
|
{
|
|
if (!_enabled) return;
|
|
|
|
MainConsole.Instance.Commands.AddCommand("Region", false, "health",
|
|
"health",
|
|
"Health monitor commands. Usage: health status|export",
|
|
HandleConsole);
|
|
|
|
_healthTimer = new Timer(_healthIntervalSec * 1000) { AutoReset = true };
|
|
_healthTimer.Elapsed += (_, __) => SafeRun("HealthPoll", PollHealth);
|
|
_healthTimer.Start();
|
|
|
|
Log.Info($"[HEALTH] Ready for region '{_scene.RegionInfo.RegionName}'. Interval={_healthIntervalSec}s.");
|
|
}
|
|
|
|
public void RemoveRegion(Scene scene)
|
|
{
|
|
if (!_enabled) return;
|
|
_healthTimer?.Stop();
|
|
_healthTimer?.Dispose();
|
|
_healthTimer = null;
|
|
|
|
_scene.UnregisterModuleInterface<IRegionHealthBus>(_bus);
|
|
_scene.UnregisterModuleInterface(this);
|
|
_scene = null;
|
|
}
|
|
|
|
public void Close()
|
|
{
|
|
_healthTimer?.Stop();
|
|
_healthTimer?.Dispose();
|
|
_healthTimer = null;
|
|
}
|
|
|
|
private void HandleConsole(string module, string[] args)
|
|
{
|
|
if (_scene == null) { MainConsole.Instance.Output("[HEALTH] No active scene."); return; }
|
|
if (args.Length < 2)
|
|
{
|
|
MainConsole.Instance.Output("health status - show current health");
|
|
MainConsole.Instance.Output("health export - export one CSV line (if MetricsExportFile configured)");
|
|
return;
|
|
}
|
|
|
|
switch (args[1].ToLowerInvariant())
|
|
{
|
|
case "status":
|
|
ReportHealth(toConsole: true);
|
|
break;
|
|
case "export":
|
|
var path = ExportMetricsOnce();
|
|
MainConsole.Instance.Output(string.IsNullOrEmpty(path)
|
|
? "[HEALTH] Export disabled or failed."
|
|
: $"[HEALTH] Exported metrics to: {path}");
|
|
break;
|
|
default:
|
|
MainConsole.Instance.Output("health status|export");
|
|
break;
|
|
}
|
|
}
|
|
|
|
private void SafeRun(string tag, Action action)
|
|
{
|
|
if (System.Threading.Interlocked.Exchange(ref _opGuard, 1) == 1)
|
|
return;
|
|
|
|
try { action(); }
|
|
catch (Exception e) { Log.Warn($"[HEALTH] {tag} failed: {e.Message}"); }
|
|
finally { System.Threading.Interlocked.Exchange(ref _opGuard, 0); }
|
|
}
|
|
|
|
private void PollHealth()
|
|
{
|
|
var s = CollectStats();
|
|
|
|
// Publish every sample
|
|
_bus.PublishSample(s);
|
|
|
|
// Threshold checks -> incidents
|
|
if (s.ScriptTimeMs >= _warnScriptTimeMs)
|
|
PublishIncident(HealthSeverity.Warn, $"High script time: {s.ScriptTimeMs:0.00} ms", s);
|
|
|
|
if (s.PhysicsTimeMs >= _warnPhysicsTimeMs)
|
|
PublishIncident(HealthSeverity.Warn, $"High physics time: {s.PhysicsTimeMs:0.00} ms", s);
|
|
|
|
if (s.NetTimeMs >= _warnNetTimeMs)
|
|
PublishIncident(HealthSeverity.Warn, $"High network time: {s.NetTimeMs:0.00} ms", s);
|
|
|
|
if (s.ScriptErrors >= _warnScriptErrors)
|
|
PublishIncident(HealthSeverity.Warn, $"High script errors: {s.ScriptErrors}", s);
|
|
|
|
ExportMetrics(s);
|
|
}
|
|
|
|
private void PublishIncident(HealthSeverity sev, string msg, HealthSample s)
|
|
{
|
|
var e = new HealthEvent
|
|
{
|
|
TimestampUtc = DateTime.UtcNow,
|
|
Region = _scene.RegionInfo.RegionName,
|
|
Severity = sev,
|
|
Message = msg,
|
|
Sample = s
|
|
};
|
|
_bus.PublishIncident(e);
|
|
if (sev >= HealthSeverity.Warn)
|
|
Log.Warn($"[HEALTH] {msg}");
|
|
else
|
|
Log.Info($"[HEALTH] {msg}");
|
|
}
|
|
|
|
private void ReportHealth(bool toConsole)
|
|
{
|
|
var s = CollectStats();
|
|
var line = $"[Health] Agents={s.Agents} Prims={s.Prims} " +
|
|
$"ScriptMs={s.ScriptTimeMs:0.00} PhysMs={s.PhysicsTimeMs:0.00} NetMs={s.NetTimeMs:0.00} " +
|
|
$"ScriptErrors={s.ScriptErrors} Uptime={s.Uptime:hh\\:mm\\:ss}";
|
|
if (toConsole) MainConsole.Instance.Output(line);
|
|
Log.Info(line);
|
|
}
|
|
|
|
private HealthSample CollectStats()
|
|
{
|
|
float scriptMs = 0f, physMs = 0f, netMs = 0f;
|
|
try
|
|
{
|
|
var stats = _scene.StatsReporter; // 0.9.3.x provides this
|
|
if (stats != null)
|
|
{
|
|
scriptMs = stats.ScriptMs;
|
|
physMs = stats.PhysicsMs;
|
|
netMs = stats.NetMs;
|
|
}
|
|
}
|
|
catch { }
|
|
|
|
int agents = 0, prims = 0, scriptErrors = 0;
|
|
try
|
|
{
|
|
agents = _scene.GetRootAgentCount() + _scene.GetChildAgentCount();
|
|
prims = _scene.GetTotalObjectsCount();
|
|
}
|
|
catch { }
|
|
|
|
try
|
|
{
|
|
var engine = _scene.RequestModuleInterface<IScriptModule>();
|
|
if (engine != null)
|
|
scriptErrors = engine.GetScriptErrorCount();
|
|
}
|
|
catch { }
|
|
|
|
TimeSpan uptime = TimeSpan.Zero;
|
|
try { uptime = DateTime.UtcNow - _scene.RegionInfo.StartupTime; } catch { }
|
|
|
|
return new HealthSample
|
|
{
|
|
TimestampUtc = DateTime.UtcNow,
|
|
Region = _scene.RegionInfo.RegionName,
|
|
Agents = agents,
|
|
Prims = prims,
|
|
ScriptTimeMs = scriptMs,
|
|
PhysicsTimeMs = physMs,
|
|
NetTimeMs = netMs,
|
|
ScriptErrors = scriptErrors,
|
|
Uptime = uptime
|
|
};
|
|
}
|
|
|
|
private void ExportMetrics(HealthSample s)
|
|
{
|
|
try
|
|
{
|
|
if (string.IsNullOrWhiteSpace(_metricsExportFile))
|
|
return;
|
|
|
|
bool header = !File.Exists(_metricsExportFile);
|
|
using (var sw = new StreamWriter(_metricsExportFile, append: true))
|
|
{
|
|
if (header)
|
|
sw.WriteLine("ts,region,agents,prims,script_ms,phys_ms,net_ms,script_errors,uptime_s");
|
|
|
|
sw.WriteLine($"{DateTime.UtcNow:O},{Csv(_scene.RegionInfo.RegionName)},{s.Agents},{s.Prims}," +
|
|
$"{s.ScriptTimeMs:0.###},{s.PhysicsTimeMs:0.###},{s.NetTimeMs:0.###}," +
|
|
$"{s.ScriptErrors},{(long)s.Uptime.TotalSeconds}");
|
|
}
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
Log.Warn($"[HEALTH] CSV export failed: {e.Message}");
|
|
}
|
|
}
|
|
|
|
private string ExportMetricsOnce()
|
|
{
|
|
try
|
|
{
|
|
if (string.IsNullOrWhiteSpace(_metricsExportFile))
|
|
return null;
|
|
|
|
var s = CollectStats();
|
|
ExportMetrics(s);
|
|
return _metricsExportFile;
|
|
}
|
|
catch { return null; }
|
|
}
|
|
|
|
private string Csv(string s) => "\"" + s.Replace("\"", "\"\"") + "\"";
|
|
}
|
|
} |