added OpenSimRegionWarmupHealthGuard Modules

This commit is contained in:
Lordfox
2025-10-05 16:51:26 +02:00
parent ea6403400a
commit 4ecca4aea7
14 changed files with 2094 additions and 0 deletions
@@ -0,0 +1,355 @@
using System;
using System.Collections.Generic;
using System.Reflection;
using System.Threading;
using log4net;
using Mono.Addins;
using Nini.Config;
using OpenMetaverse;
using OpenSim.Framework;
using OpenSim.Framework.Console;
using OpenSim.Region.Framework.Interfaces;
using OpenSim.Region.Framework.Scenes;
[assembly: Addin("RegionAutoHeal", "1.0.0")]
[assembly: AddinDependency("OpenSim.Region.Framework", OpenSim.VersionInfo.VersionNumber)]
[assembly: AddinDescription("Auto-heal (dry-run by default): targeted script reset and best-effort throttling of heavy updaters.")]
[assembly: AddinAuthor("Christopher Händler")]
namespace OpenSimRegionWarmupHealthGuard.Modules
{
[Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "RegionAutoHeal")]
public class RegionAutoHealModule : INonSharedRegionModule
{
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
private Scene _scene;
private bool _enabled;
// Config
private bool _dryRun = true; // safe default
private bool _enableScriptReset = false; // needs explicit opt-in
private bool _throttleHeavyUpdaters = true;
private int _throttleThresholdUpdatesPerSec = 30;
private int _scriptErrorBurstThreshold = 25; // incident filter
private int _cooldownSec = 60; // avoid repeated actions
// State
private IRegionHealthBus _bus;
private DateTime _lastActionUtc = DateTime.MinValue;
private volatile int _opGuard;
public string Name => "RegionAutoHeal";
public Type ReplaceableInterface => null;
public void Initialise(IConfigSource source)
{
var modules = source.Configs["Modules"];
if (modules == null) return;
var status = modules.GetString("RegionAutoHeal", string.Empty);
if (!string.Equals(status, "enabled", StringComparison.OrdinalIgnoreCase)) return;
var cfg = source.Configs["RegionAutoHeal"] ?? source.AddConfig("RegionAutoHeal");
_dryRun = cfg.GetBoolean("DryRun", _dryRun);
_enableScriptReset = cfg.GetBoolean("EnableScriptReset", _enableScriptReset);
_throttleHeavyUpdaters = cfg.GetBoolean("ThrottleHeavyUpdaters", _throttleHeavyUpdaters);
_throttleThresholdUpdatesPerSec = Math.Max(1, cfg.GetInt("ThrottleThresholdUpdatesPerSec", _throttleThresholdUpdatesPerSec));
_scriptErrorBurstThreshold = Math.Max(1, cfg.GetInt("ScriptErrorBurstThreshold", _scriptErrorBurstThreshold));
_cooldownSec = Math.Max(10, cfg.GetInt("CooldownSec", _cooldownSec));
_enabled = true;
Log.Info($"[AUTOHEAL] Enabled (DryRun={_dryRun}, Reset={_enableScriptReset}, Throttle={_throttleHeavyUpdaters})");
}
public void AddRegion(Scene scene)
{
if (!_enabled) return;
_scene = scene;
_bus = _scene.RequestModuleInterface<IRegionHealthBus>();
if (_bus == null)
{
Log.Warn("[AUTOHEAL] No IRegionHealthBus found. Auto-heal will remain idle until RegionHealthMonitor is enabled.");
}
else
{
_bus.OnIncident += OnIncident;
}
_scene.RegisterModuleInterface(this);
}
public void RegionLoaded(Scene scene)
{
if (!_enabled) return;
MainConsole.Instance.Commands.AddCommand("Region", false, "autoheal",
"autoheal",
"Auto-heal commands. Usage: autoheal status|dryrun on|off|reset <objectId>",
HandleConsole);
Log.Info($"[AUTOHEAL] Ready for region '{_scene.RegionInfo.RegionName}'.");
}
public void RemoveRegion(Scene scene)
{
if (!_enabled) return;
if (_bus != null)
_bus.OnIncident -= OnIncident;
_scene.UnregisterModuleInterface(this);
_scene = null;
}
public void Close()
{
}
private void HandleConsole(string module, string[] args)
{
if (_scene == null) { MainConsole.Instance.Output("[AUTOHEAL] No active scene."); return; }
if (args.Length < 2)
{
PrintHelp();
return;
}
switch (args[1].ToLowerInvariant())
{
case "status":
MainConsole.Instance.Output($"[AUTOHEAL] DryRun={_dryRun} EnableScriptReset={_enableScriptReset} Throttle={_throttleHeavyUpdaters} ThresholdUpd/s={_throttleThresholdUpdatesPerSec} CooldownSec={_cooldownSec}");
break;
case "dryrun":
if (args.Length >= 3)
{
bool on = args[2].Equals("on", StringComparison.OrdinalIgnoreCase);
bool off = args[2].Equals("off", StringComparison.OrdinalIgnoreCase);
if (on || off)
{
_dryRun = on;
MainConsole.Instance.Output($"[AUTOHEAL] DryRun={_dryRun}");
break;
}
}
PrintHelp();
break;
case "reset":
if (args.Length >= 3)
{
if (UUID.TryParse(args[2], out var id))
{
RunOnceSafe(() => TryResetObjectScripts(id, reason: "manual"));
break;
}
MainConsole.Instance.Output("[AUTOHEAL] invalid UUID");
break;
}
PrintHelp();
break;
default:
PrintHelp();
break;
}
}
private void PrintHelp()
{
MainConsole.Instance.Output("autoheal status - show status");
MainConsole.Instance.Output("autoheal dryrun on|off - toggle dry-run mode");
MainConsole.Instance.Output("autoheal reset <objectId> - targeted script reset (requires EnableScriptReset=true)");
}
private void OnIncident(HealthEvent e)
{
// Only react to WARN/ERROR and when thresholds suggest action
if (e == null || e.Sample == null) return;
if (e.Severity < HealthSeverity.Warn) return;
// basic cooldown
if ((DateTime.UtcNow - _lastActionUtc).TotalSeconds < _cooldownSec)
return;
// Heuristics: large script error burst
if (e.Sample.ScriptErrors >= _scriptErrorBurstThreshold)
{
RunOnceSafe(() =>
{
// 1) find suspicious objects (heuristic, best-effort)
var suspects = FindSuspectObjects(max: 10);
// 2) reset scripts (if enabled) or log suggestion
foreach (var so in suspects)
{
TryResetObjectScripts(so.UUID, reason: $"auto: errorBurst({e.Sample.ScriptErrors})");
}
// 3) throttle heavy updaters
if (_throttleHeavyUpdaters)
BestEffortThrottleHeavyUpdaters(max: 10);
});
}
}
private void RunOnceSafe(Action action)
{
if (Interlocked.Exchange(ref _opGuard, 1) == 1)
return;
try
{
action();
_lastActionUtc = DateTime.UtcNow;
}
catch (Exception ex)
{
Log.Warn($"[AUTOHEAL] action failed: {ex.Message}");
}
finally
{
Interlocked.Exchange(ref _opGuard, 0);
}
}
// Heuristic: find objects with many scripts or high part count as suspects
private IEnumerable<SceneObjectGroup> FindSuspectObjects(int max)
{
var result = new List<SceneObjectGroup>(max);
try
{
var all = _scene.GetSceneObjectGroups();
// simple score: script count + part count
foreach (var so in all)
{
int scripts = 0;
try
{
foreach (var p in so.Parts)
scripts += p.Inventory?.GetInventoryItems()?.Count ?? 0;
}
catch { }
int score = scripts + so.Parts?.Length ?? 0;
// crude threshold
if (score >= 20)
{
result.Add(so);
if (result.Count >= max) break;
}
}
}
catch { }
Log.Info($"[AUTOHEAL] suspects found: {result.Count}");
return result;
}
private void TryResetObjectScripts(UUID objectId, string reason)
{
var sog = _scene.GetSceneObjectGroup(objectId);
if (sog == null)
{
Log.Info($"[AUTOHEAL] object not found for reset: {objectId}");
return;
}
if (!_enableScriptReset)
{
Log.Warn($"[AUTOHEAL] (dry) would reset scripts of '{sog.Name}' ({objectId}) reason={reason}");
return;
}
if (_dryRun)
{
Log.Warn($"[AUTOHEAL] (dry) reset scripts of '{sog.Name}' ({objectId}) reason={reason}");
return;
}
try
{
var engine = _scene.RequestModuleInterface<IScriptModule>();
if (engine == null)
{
Log.Warn("[AUTOHEAL] no script engine, cannot reset scripts.");
return;
}
int count = 0;
foreach (var part in sog.Parts)
{
try
{
// @todo: ResetScript????
engine.ResetScript(part.UUID, UUID.Zero); // best-effort API call
count++;
}
catch { }
}
Log.Warn($"[AUTOHEAL] reset scripts of '{sog.Name}' parts={count} reason={reason}");
}
catch (Exception ex)
{
Log.Warn($"[AUTOHEAL] reset failed: {ex.Message}");
}
}
private void BestEffortThrottleHeavyUpdaters(int max)
{
try
{
int throttled = 0;
foreach (var so in _scene.GetSceneObjectGroups())
{
if (throttled >= max) break;
// crude detection: objects with many parts and recent updates
var parts = so.Parts;
if (parts == null || parts.Length < 1) continue;
int localUpdates = 0;
foreach (var p in parts)
{
// Heuristic placeholder: if a part's last update was very recent, count it.
// OpenSim does not provide a standard per-part updates/sec API; so we log suggestions.
// If your fork exposes update rates, plug them here and compare to _throttleThresholdUpdatesPerSec.
// For now we treat "big & busy" objects as candidates.
if (p.UpdateFlag != 0) localUpdates++;
}
if (parts.Length >= 10 || localUpdates >= 5)
{
if (_dryRun)
{
Log.Warn($"[AUTOHEAL] (dry) would throttle '{so.Name}' parts={parts.Length} updates~{localUpdates}/tick");
}
else
{
// Best-effort: disable constant updates we control (e.g., temporary turning off dynamics/particles where sensible)
foreach (var p in parts)
{
try
{
// Example: reduce alpha mode/particle sources if available
// Without stable generic API, we only log
}
catch { }
}
Log.Warn($"[AUTOHEAL] throttled '{so.Name}' (best-effort)");
}
throttled++;
}
}
if (throttled > 0)
Log.Warn($"[AUTOHEAL] throttled objects: {throttled}");
}
catch (Exception ex)
{
Log.Warn($"[AUTOHEAL] throttle failed: {ex.Message}");
}
}
}
}
@@ -0,0 +1,332 @@
using System;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Timers;
using log4net;
using Mono.Addins;
using Nini.Config;
using OpenSim.Framework;
using OpenSim.Framework.Console;
using OpenSim.Region.Framework.Interfaces;
using OpenSim.Region.Framework.Scenes;
[assembly: Addin("RegionHealthMonitor", "1.0.0")]
[assembly: AddinDependency("OpenSim.Region.Framework", OpenSim.VersionInfo.VersionNumber)]
[assembly: AddinDescription("Region health polling + threshold warnings + CSV export + health event bus.")]
[assembly: AddinAuthor("Christopher Händler")]
namespace OpenSimRegionWarmupHealthGuard.Modules
{
// Simple health sample DTO
public class HealthSample
{
public DateTime TimestampUtc;
public string Region;
public int Agents;
public int Prims;
public float ScriptTimeMs;
public float PhysicsTimeMs;
public float NetTimeMs;
public int ScriptErrors;
public TimeSpan Uptime;
}
public enum HealthSeverity { Trace, Info, Warn, Error }
public class HealthEvent
{
public DateTime TimestampUtc;
public string Region;
public HealthSeverity Severity;
public string Message;
public HealthSample Sample; // optional snapshot of metrics
}
// Event-bus interface for other modules (alerts/export/auto-heal/policy)
public interface IRegionHealthBus
{
event Action<HealthSample> OnSample; // every poll
event Action<HealthEvent> OnIncident; // only on rule/threshold violation
void PublishSample(HealthSample s);
void PublishIncident(HealthEvent e);
}
public class RegionHealthBus : IRegionHealthBus
{
public event Action<HealthSample> OnSample;
public event Action<HealthEvent> OnIncident;
public void PublishSample(HealthSample s) => OnSample?.Invoke(s);
public void PublishIncident(HealthEvent e) => OnIncident?.Invoke(e);
}
[Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "RegionHealthMonitor")]
public class RegionHealthMonitorModule : INonSharedRegionModule
{
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
private Scene _scene;
private bool _enabled;
// Config
private int _healthIntervalSec = 30;
private float _warnScriptTimeMs = 12.0f;
private float _warnPhysicsTimeMs = 6.0f;
private float _warnNetTimeMs = 6.0f;
private int _warnScriptErrors = 10;
private string _metricsExportFile = ""; // optional CSV export
private Timer _healthTimer;
private volatile int _opGuard;
// Expose bus for other modules
private readonly RegionHealthBus _bus = new RegionHealthBus();
public IRegionHealthBus Bus => _bus;
public string Name => "RegionHealthMonitor";
public Type ReplaceableInterface => null;
public void Initialise(IConfigSource source)
{
var modules = source.Configs["Modules"];
if (modules == null) return;
var status = modules.GetString("RegionHealthMonitor", string.Empty);
if (!string.Equals(status, "enabled", StringComparison.OrdinalIgnoreCase)) return;
var cfg = source.Configs["RegionHealthMonitor"] ?? source.AddConfig("RegionHealthMonitor");
_healthIntervalSec = Math.Max(5, cfg.GetInt("HealthIntervalSec", _healthIntervalSec));
_warnScriptTimeMs = (float)cfg.GetDouble("WarnScriptTimeMs", _warnScriptTimeMs);
_warnPhysicsTimeMs = (float)cfg.GetDouble("WarnPhysicsTimeMs", _warnPhysicsTimeMs);
_warnNetTimeMs = (float)cfg.GetDouble("WarnNetTimeMs", _warnNetTimeMs);
_warnScriptErrors = Math.Max(0, cfg.GetInt("WarnScriptErrors", _warnScriptErrors));
_metricsExportFile = cfg.GetString("MetricsExportFile", _metricsExportFile);
_enabled = true;
Log.Info("[HEALTH] RegionHealthMonitor enabled");
}
public void AddRegion(Scene scene)
{
if (!_enabled) return;
_scene = scene;
// Make bus discoverable to other modules
_scene.RegisterModuleInterface<IRegionHealthBus>(_bus);
_scene.RegisterModuleInterface(this);
}
public void RegionLoaded(Scene scene)
{
if (!_enabled) return;
MainConsole.Instance.Commands.AddCommand("Region", false, "health",
"health",
"Health monitor commands. Usage: health status|export",
HandleConsole);
_healthTimer = new Timer(_healthIntervalSec * 1000) { AutoReset = true };
_healthTimer.Elapsed += (_, __) => SafeRun("HealthPoll", PollHealth);
_healthTimer.Start();
Log.Info($"[HEALTH] Ready for region '{_scene.RegionInfo.RegionName}'. Interval={_healthIntervalSec}s.");
}
public void RemoveRegion(Scene scene)
{
if (!_enabled) return;
_healthTimer?.Stop();
_healthTimer?.Dispose();
_healthTimer = null;
_scene.UnregisterModuleInterface<IRegionHealthBus>(_bus);
_scene.UnregisterModuleInterface(this);
_scene = null;
}
public void Close()
{
_healthTimer?.Stop();
_healthTimer?.Dispose();
_healthTimer = null;
}
private void HandleConsole(string module, string[] args)
{
if (_scene == null) { MainConsole.Instance.Output("[HEALTH] No active scene."); return; }
if (args.Length < 2)
{
MainConsole.Instance.Output("health status - show current health");
MainConsole.Instance.Output("health export - export one CSV line (if MetricsExportFile configured)");
return;
}
switch (args[1].ToLowerInvariant())
{
case "status":
ReportHealth(toConsole: true);
break;
case "export":
var path = ExportMetricsOnce();
MainConsole.Instance.Output(string.IsNullOrEmpty(path)
? "[HEALTH] Export disabled or failed."
: $"[HEALTH] Exported metrics to: {path}");
break;
default:
MainConsole.Instance.Output("health status|export");
break;
}
}
private void SafeRun(string tag, Action action)
{
if (System.Threading.Interlocked.Exchange(ref _opGuard, 1) == 1)
return;
try { action(); }
catch (Exception e) { Log.Warn($"[HEALTH] {tag} failed: {e.Message}"); }
finally { System.Threading.Interlocked.Exchange(ref _opGuard, 0); }
}
private void PollHealth()
{
var s = CollectStats();
// Publish every sample
_bus.PublishSample(s);
// Threshold checks -> incidents
if (s.ScriptTimeMs >= _warnScriptTimeMs)
PublishIncident(HealthSeverity.Warn, $"High script time: {s.ScriptTimeMs:0.00} ms", s);
if (s.PhysicsTimeMs >= _warnPhysicsTimeMs)
PublishIncident(HealthSeverity.Warn, $"High physics time: {s.PhysicsTimeMs:0.00} ms", s);
if (s.NetTimeMs >= _warnNetTimeMs)
PublishIncident(HealthSeverity.Warn, $"High network time: {s.NetTimeMs:0.00} ms", s);
if (s.ScriptErrors >= _warnScriptErrors)
PublishIncident(HealthSeverity.Warn, $"High script errors: {s.ScriptErrors}", s);
ExportMetrics(s);
}
private void PublishIncident(HealthSeverity sev, string msg, HealthSample s)
{
var e = new HealthEvent
{
TimestampUtc = DateTime.UtcNow,
Region = _scene.RegionInfo.RegionName,
Severity = sev,
Message = msg,
Sample = s
};
_bus.PublishIncident(e);
if (sev >= HealthSeverity.Warn)
Log.Warn($"[HEALTH] {msg}");
else
Log.Info($"[HEALTH] {msg}");
}
private void ReportHealth(bool toConsole)
{
var s = CollectStats();
var line = $"[Health] Agents={s.Agents} Prims={s.Prims} " +
$"ScriptMs={s.ScriptTimeMs:0.00} PhysMs={s.PhysicsTimeMs:0.00} NetMs={s.NetTimeMs:0.00} " +
$"ScriptErrors={s.ScriptErrors} Uptime={s.Uptime:hh\\:mm\\:ss}";
if (toConsole) MainConsole.Instance.Output(line);
Log.Info(line);
}
private HealthSample CollectStats()
{
float scriptMs = 0f, physMs = 0f, netMs = 0f;
try
{
var stats = _scene.StatsReporter; // 0.9.3.x provides this
if (stats != null)
{
scriptMs = stats.ScriptMs;
physMs = stats.PhysicsMs;
netMs = stats.NetMs;
}
}
catch { }
int agents = 0, prims = 0, scriptErrors = 0;
try
{
agents = _scene.GetRootAgentCount() + _scene.GetChildAgentCount();
prims = _scene.GetTotalObjectsCount();
}
catch { }
try
{
var engine = _scene.RequestModuleInterface<IScriptModule>();
if (engine != null)
scriptErrors = engine.GetScriptErrorCount();
}
catch { }
TimeSpan uptime = TimeSpan.Zero;
try { uptime = DateTime.UtcNow - _scene.RegionInfo.StartupTime; } catch { }
return new HealthSample
{
TimestampUtc = DateTime.UtcNow,
Region = _scene.RegionInfo.RegionName,
Agents = agents,
Prims = prims,
ScriptTimeMs = scriptMs,
PhysicsTimeMs = physMs,
NetTimeMs = netMs,
ScriptErrors = scriptErrors,
Uptime = uptime
};
}
private void ExportMetrics(HealthSample s)
{
try
{
if (string.IsNullOrWhiteSpace(_metricsExportFile))
return;
bool header = !File.Exists(_metricsExportFile);
using (var sw = new StreamWriter(_metricsExportFile, append: true))
{
if (header)
sw.WriteLine("ts,region,agents,prims,script_ms,phys_ms,net_ms,script_errors,uptime_s");
sw.WriteLine($"{DateTime.UtcNow:O},{Csv(_scene.RegionInfo.RegionName)},{s.Agents},{s.Prims}," +
$"{s.ScriptTimeMs:0.###},{s.PhysicsTimeMs:0.###},{s.NetTimeMs:0.###}," +
$"{s.ScriptErrors},{(long)s.Uptime.TotalSeconds}");
}
}
catch (Exception e)
{
Log.Warn($"[HEALTH] CSV export failed: {e.Message}");
}
}
private string ExportMetricsOnce()
{
try
{
if (string.IsNullOrWhiteSpace(_metricsExportFile))
return null;
var s = CollectStats();
ExportMetrics(s);
return _metricsExportFile;
}
catch { return null; }
}
private string Csv(string s) => "\"" + s.Replace("\"", "\"\"") + "\"";
}
}
@@ -0,0 +1,312 @@
using System;
using System.Globalization;
using System.Net;
using System.Reflection;
using System.Text;
using System.Threading;
using log4net;
using Mono.Addins;
using Nini.Config;
using OpenSim.Framework;
using OpenSim.Framework.Console;
using OpenSim.Region.Framework.Interfaces;
using OpenSim.Region.Framework.Scenes;
[assembly: Addin("RegionMetricsExporter", "1.0.0")]
[assembly: AddinDependency("OpenSim.Region.Framework", OpenSim.VersionInfo.VersionNumber)]
[assembly: AddinDescription("Prometheus (HTTP pull) metrics exporter for RegionHealthMonitor.")]
[assembly: AddinAuthor("Christopher Händler")]
namespace OpenSimRegionWarmupHealthGuard.Modules
{
[Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "RegionMetricsExporter")]
public class RegionMetricsExporterModule : INonSharedRegionModule
{
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
private Scene _scene;
private bool _enabled;
// Config
private int _httpPort = 9109;
private string _metricsPrefix = "opensim_region_";
private string _bindAddress = "127.0.0.1"; // default loopback for safety
private bool _includeRegionLabel = true;
// State
private HttpListener _listener;
private Thread _httpThread;
private volatile bool _httpRunning;
private readonly object _lastLock = new object();
private HealthSample _lastSample; // last health sample for this region
// Bus
private IRegionHealthBus _bus;
public string Name => "RegionMetricsExporter";
public Type ReplaceableInterface => null;
public void Initialise(IConfigSource source)
{
var modules = source.Configs["Modules"];
if (modules == null) return;
var status = modules.GetString("RegionMetricsExporter", string.Empty);
if (!string.Equals(status, "enabled", StringComparison.OrdinalIgnoreCase)) return;
var cfg = source.Configs["RegionMetricsExporter"] ?? source.AddConfig("RegionMetricsExporter");
_httpPort = Math.Max(1, cfg.GetInt("HttpPort", _httpPort));
_metricsPrefix = cfg.GetString("MetricsPrefix", _metricsPrefix);
_bindAddress = cfg.GetString("BindAddress", _bindAddress);
_includeRegionLabel = cfg.GetBoolean("IncludeRegionLabel", _includeRegionLabel);
_enabled = true;
Log.Info($"[METRICS] RegionMetricsExporter enabled (port={_httpPort}, bind={_bindAddress})");
}
public void AddRegion(Scene scene)
{
if (!_enabled) return;
_scene = scene;
// Discover health bus (provided by RegionHealthMonitor)
_bus = _scene.RequestModuleInterface<IRegionHealthBus>();
if (_bus == null)
{
Log.Warn("[METRICS] No IRegionHealthBus found. Metrics will remain empty until RegionHealthMonitor is enabled.");
}
else
{
_bus.OnSample += OnHealthSample;
}
_scene.RegisterModuleInterface(this);
}
public void RegionLoaded(Scene scene)
{
if (!_enabled) return;
// Start HTTP listener per region module instance (separate port or one shared port per process recommended)
// Here: single-port per region instance. In multi-region processes, use different ports per region.
StartHttp();
MainConsole.Instance.Commands.AddCommand("Region", false, "metrics",
"metrics",
"Metrics exporter commands. Usage: metrics status",
HandleConsole);
Log.Info($"[METRICS] Ready for region '{_scene.RegionInfo.RegionName}'. Endpoint: http://{_bindAddress}:{_httpPort}/metrics");
}
public void RemoveRegion(Scene scene)
{
if (!_enabled) return;
StopHttp();
if (_bus != null)
_bus.OnSample -= OnHealthSample;
_scene.UnregisterModuleInterface(this);
_scene = null;
}
public void Close()
{
StopHttp();
}
private void HandleConsole(string module, string[] args)
{
if (_scene == null) { MainConsole.Instance.Output("[METRICS] No active scene."); return; }
if (args.Length < 2)
{
MainConsole.Instance.Output("metrics status - show exporter status");
return;
}
switch (args[1].ToLowerInvariant())
{
case "status":
MainConsole.Instance.Output($"[METRICS] Endpoint: http://{_bindAddress}:{_httpPort}/metrics Running={_httpRunning}");
break;
default:
MainConsole.Instance.Output("metrics status");
break;
}
}
private void OnHealthSample(HealthSample s)
{
lock (_lastLock)
{
_lastSample = s;
}
}
private void StartHttp()
{
try
{
if (_httpRunning) return;
_listener = new HttpListener();
_listener.Prefixes.Add($"http://{_bindAddress}:{_httpPort}/metrics/");
// Allow also without trailing slash
_listener.Prefixes.Add($"http://{_bindAddress}:{_httpPort}/metrics");
_listener.Start();
_httpRunning = true;
_httpThread = new Thread(HttpLoop) { IsBackground = true, Name = "RegionMetricsExporter-HTTP" };
_httpThread.Start();
Log.Info($"[METRICS] HTTP metrics endpoint started on http://{_bindAddress}:{_httpPort}/metrics");
}
catch (Exception e)
{
Log.Error($"[METRICS] Failed to start HTTP listener: {e.Message}");
_httpRunning = false;
try { _listener?.Stop(); } catch { }
_listener = null;
}
}
private void StopHttp()
{
try
{
_httpRunning = false;
try { _listener?.Stop(); } catch { }
try { _listener?.Close(); } catch { }
_listener = null;
}
catch { }
if (_httpThread != null)
{
try { _httpThread.Join(1000); } catch { }
_httpThread = null;
}
}
private void HttpLoop()
{
while (_httpRunning && _listener != null)
{
HttpListenerContext ctx = null;
try
{
ctx = _listener.GetContext();
}
catch
{
if (!_httpRunning) break;
continue;
}
ThreadPool.QueueUserWorkItem(_ => HandleRequest(ctx));
}
}
private void HandleRequest(HttpListenerContext ctx)
{
try
{
if (ctx.Request.HttpMethod != "GET")
{
ctx.Response.StatusCode = 405;
ctx.Response.Close();
return;
}
if (!ctx.Request.Url.AbsolutePath.StartsWith("/metrics", StringComparison.OrdinalIgnoreCase))
{
ctx.Response.StatusCode = 404;
ctx.Response.Close();
return;
}
var text = BuildPrometheusText();
var buffer = Encoding.UTF8.GetBytes(text);
ctx.Response.StatusCode = 200;
ctx.Response.ContentType = "text/plain; version=0.0.4; charset=utf-8";
ctx.Response.ContentEncoding = Encoding.UTF8;
ctx.Response.ContentLength64 = buffer.Length;
ctx.Response.OutputStream.Write(buffer, 0, buffer.Length);
ctx.Response.OutputStream.Close();
}
catch
{
try { ctx.Response.StatusCode = 500; ctx.Response.Close(); } catch { }
}
}
private string BuildPrometheusText()
{
HealthSample s;
lock (_lastLock)
{
s = _lastSample;
}
var region = _scene?.RegionInfo?.RegionName ?? "unknown";
var sb = new StringBuilder(512);
var prefix = _metricsPrefix;
// HELP/TYPE
AppendHelpType(sb, prefix + "agents", "Number of agents (root + child) in region", "gauge");
AppendHelpType(sb, prefix + "prims", "Total number of prims/objects in region", "gauge");
AppendHelpType(sb, prefix + "script_time_ms", "Script time (ms)", "gauge");
AppendHelpType(sb, prefix + "physics_time_ms", "Physics time (ms)", "gauge");
AppendHelpType(sb, prefix + "net_time_ms", "Network time (ms)", "gauge");
AppendHelpType(sb, prefix + "script_errors", "Script error count", "gauge");
AppendHelpType(sb, prefix + "uptime_seconds", "Region uptime (seconds)", "gauge");
AppendHelpType(sb, prefix + "sample_timestamp_seconds", "Last sample timestamp (unix seconds)", "gauge");
var labels = _includeRegionLabel ? $"{{region=\"{EscapeLabel(region)}\"}}" : "";
if (s == null)
{
// no sample yet: export zeros
sb.AppendLine($"{prefix}agents{labels} 0");
sb.AppendLine($"{prefix}prims{labels} 0");
sb.AppendLine($"{prefix}script_time_ms{labels} 0");
sb.AppendLine($"{prefix}physics_time_ms{labels} 0");
sb.AppendLine($"{prefix}net_time_ms{labels} 0");
sb.AppendLine($"{prefix}script_errors{labels} 0");
sb.AppendLine($"{prefix}uptime_seconds{labels} 0");
sb.AppendLine($"{prefix}sample_timestamp_seconds{labels} 0");
return sb.ToString();
}
// Invariant culture for dots
var ci = CultureInfo.InvariantCulture;
sb.AppendLine($"{prefix}agents{labels} {s.Agents.ToString(ci)}");
sb.AppendLine($"{prefix}prims{labels} {s.Prims.ToString(ci)}");
sb.AppendLine($"{prefix}script_time_ms{labels} {s.ScriptTimeMs.ToString(ci)}");
sb.AppendLine($"{prefix}physics_time_ms{labels} {s.PhysicsTimeMs.ToString(ci)}");
sb.AppendLine($"{prefix}net_time_ms{labels} {s.NetTimeMs.ToString(ci)}");
sb.AppendLine($"{prefix}script_errors{labels} {s.ScriptErrors.ToString(ci)}");
sb.AppendLine($"{prefix}uptime_seconds{labels} {((long)s.Uptime.TotalSeconds).ToString(ci)}");
var unix = new DateTimeOffset(s.TimestampUtc).ToUnixTimeSeconds();
sb.AppendLine($"{prefix}sample_timestamp_seconds{labels} {unix.ToString(ci)}");
return sb.ToString();
}
private static void AppendHelpType(StringBuilder sb, string name, string help, string type)
{
sb.AppendLine($"# HELP {name} {help}");
sb.AppendLine($"# TYPE {name} {type}");
}
private static string EscapeLabel(string value)
{
if (string.IsNullOrEmpty(value)) return "";
return value.Replace("\\", "\\\\").Replace("\"", "\\\"");
}
}
}
@@ -0,0 +1,263 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Timers;
using log4net;
using Mono.Addins;
using Nini.Config;
using OpenSim.Framework;
using OpenSim.Framework.Console;
using OpenSim.Region.Framework.Interfaces;
using OpenSim.Region.Framework.Scenes;
[assembly: Addin("RegionPolicyEngine", "1.0.0")]
[assembly: AddinDependency("OpenSim.Region.Framework", OpenSim.VersionInfo.VersionNumber)]
[assembly: AddinDescription("Applies time-based/profile overlays to other modules' configs (cooperative).")]
[assembly: AddinAuthor("Christopher Händler")]
namespace OpenSimRegionWarmupHealthGuard.Modules
{
public interface IPolicyOverlayConsumer
{
// Modules can implement this to accept overlays at runtime.
// Keys/values are simple strings; module maps to local settings.
void ApplyPolicyOverlay(IDictionary<string, string> overrides, string sourceProfile);
}
[Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "RegionPolicyEngine")]
public class RegionPolicyEngineModule : INonSharedRegionModule
{
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
private Scene _scene;
private bool _enabled;
private bool _engineEnabled = true;
private string[] _profiles = Array.Empty<string>();
private readonly Dictionary<string, string> _profileCron = new();
private readonly Dictionary<string, Dictionary<string, string>> _profileOverrides = new();
private Timer _tick;
private int _intervalSec = 60; // check every minute
public string Name => "RegionPolicyEngine";
public Type ReplaceableInterface => null;
public void Initialise(IConfigSource source)
{
var modules = source.Configs["Modules"];
if (modules == null) return;
var status = modules.GetString("RegionPolicyEngine", string.Empty);
if (!string.Equals(status, "enabled", StringComparison.OrdinalIgnoreCase)) return;
var cfg = source.Configs["RegionPolicyEngine"] ?? source.AddConfig("RegionPolicyEngine");
_engineEnabled = cfg.GetBoolean("Enabled", _engineEnabled);
_intervalSec = Math.Max(15, cfg.GetInt("CheckIntervalSec", _intervalSec));
var profilesCsv = cfg.GetString("Profiles", "");
_profiles = profilesCsv.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries)
.Select(s => s.Trim())
.ToArray();
foreach (var p in _profiles)
{
var cron = cfg.GetString($"{p}.Cron", "");
var overridesLine = cfg.GetString($"{p}.Overrides", "");
if (!string.IsNullOrWhiteSpace(cron))
_profileCron[p] = cron;
var dict = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var kv in overridesLine.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries))
{
var i = kv.IndexOf('=');
if (i > 0)
{
var k = kv.Substring(0, i).Trim();
var v = kv.Substring(i + 1).Trim();
if (k.Length > 0) dict[k] = v;
}
}
_profileOverrides[p] = dict;
}
_enabled = _engineEnabled;
Log.Info($"[POLICY] RegionPolicyEngine enabled={_engineEnabled} profiles={string.Join(",", _profiles)}");
}
public void AddRegion(Scene scene)
{
if (!_enabled) return;
_scene = scene;
_scene.RegisterModuleInterface(this);
}
public void RegionLoaded(Scene scene)
{
if (!_enabled) return;
MainConsole.Instance.Commands.AddCommand("Region", false, "policy",
"policy",
"Policy engine commands. Usage: policy status|apply <Profile>|dryrun <Profile>",
HandleConsole);
_tick = new Timer(_intervalSec * 1000) { AutoReset = true };
_tick.Elapsed += (_, __) => EvaluateAndApply();
_tick.Start();
Log.Info($"[POLICY] Ready for region '{_scene.RegionInfo.RegionName}'. check={_intervalSec}s");
}
public void RemoveRegion(Scene scene)
{
if (!_enabled) return;
try { _tick?.Stop(); _tick?.Dispose(); } catch { }
_tick = null;
_scene.UnregisterModuleInterface(this);
_scene = null;
}
public void Close()
{
try { _tick?.Stop(); _tick?.Dispose(); } catch { }
_tick = null;
}
private void HandleConsole(string module, string[] args)
{
if (_scene == null) { MainConsole.Instance.Output("[POLICY] No active scene."); return; }
if (args.Length < 2)
{
PrintHelp();
return;
}
var cmd = args[1].ToLowerInvariant();
if (cmd == "status")
{
MainConsole.Instance.Output($"[POLICY] Enabled={_engineEnabled} Profiles={string.Join(",", _profiles)}");
foreach (var p in _profiles)
{
_profileCron.TryGetValue(p, out var cron);
_profileOverrides.TryGetValue(p, out var dict);
MainConsole.Instance.Output($" - {p}: Cron='{cron}' Overrides='{string.Join(";", dict.Select(kv => kv.Key + \"=\" + kv.Value))}'");
}
return;
}
if ((cmd == "apply" || cmd == "dryrun") && args.Length >= 3)
{
var profile = args[2];
if (!_profileOverrides.TryGetValue(profile, out var ov))
{
MainConsole.Instance.Output($"[POLICY] Profile not found: {profile}");
return;
}
if (cmd == "dryrun")
{
MainConsole.Instance.Output($"[POLICY] DryRun profile={profile} -> {string.Join(";", ov.Select(kv => kv.Key + \"=\" + kv.Value))}");
return;
}
ApplyOverlay(ov, profile);
MainConsole.Instance.Output($"[POLICY] Applied profile={profile}");
return;
}
PrintHelp();
}
private void PrintHelp()
{
MainConsole.Instance.Output("policy status - show profiles and config");
MainConsole.Instance.Output("policy apply <Profile> - apply overrides now");
MainConsole.Instance.Output("policy dryrun <Profile> - show what would be applied");
}
private void EvaluateAndApply()
{
// Minimalistic time matcher:
// Cron format (simplified): mm HH hhRange? (e.g., "0 0 20-6" -> every minute between 20 and 6)
// For real cron parsing you could add a tiny parser; here we implement a simple hours window.
var now = DateTime.Now;
foreach (var p in _profiles)
{
if (!_profileCron.TryGetValue(p, out var cron) || string.IsNullOrWhiteSpace(cron))
continue;
// Expect format: "0 0 20-6" OR "* * 20-6", we only check hours window
var parts = cron.Split(new[] { ' ', '\t' }, StringSplitOptions.RemoveEmptyEntries);
string hours = parts.Length >= 3 ? parts[2] : parts.LastOrDefault();
if (string.IsNullOrWhiteSpace(hours)) continue;
bool inWindow = IsHourInWindow(now.Hour, hours);
if (!inWindow) continue;
if (_profileOverrides.TryGetValue(p, out var ov))
{
ApplyOverlay(ov, p);
}
}
}
private static bool IsHourInWindow(int hour, string expr)
{
// Accept "20-6" or "8-18" or single "22"
if (expr.Contains("-"))
{
var sp = expr.Split('-');
if (sp.Length != 2) return false;
if (!int.TryParse(sp[0], out var a)) return false;
if (!int.TryParse(sp[1], out var b)) return false;
a = (a % 24 + 24) % 24;
b = (b % 24 + 24) % 24;
if (a <= b) return hour >= a && hour <= b; // e.g., 8-18
else return hour >= a || hour <= b; // e.g., 20-6 (wrap)
}
else
{
return int.TryParse(expr, out var h) && h == hour;
}
}
private void ApplyOverlay(IDictionary<string, string> ov, string profile)
{
try
{
// Discover consumers and deliver overlay
var consumers = new List<IPolicyOverlayConsumer>();
var health = _scene.RequestModuleInterface<RegionHealthMonitorModule>();
if (health is IPolicyOverlayConsumer c1) consumers.Add(c1);
var warmup = _scene.RequestModuleInterface<RegionWarmupModule>();
if (warmup is IPolicyOverlayConsumer c2) consumers.Add(c2);
var alerts = _scene.RequestModuleInterface<RegionWebhookAlertsModule>();
if (alerts is IPolicyOverlayConsumer c3) consumers.Add(c3);
var auto = _scene.RequestModuleInterface<RegionAutoHealModule>();
if (auto is IPolicyOverlayConsumer c4) consumers.Add(c4);
foreach (var c in consumers.Distinct())
{
try { c.ApplyPolicyOverlay(ov, profile); }
catch (Exception e) { Log.Debug($"[POLICY] overlay error: {e.Message}"); }
}
Log.Info($"[POLICY] overlay applied profile={profile} targets={consumers.Count}");
}
catch (Exception e)
{
Log.Debug($"[POLICY] apply failed: {e.Message}");
}
}
}
}
@@ -0,0 +1,216 @@
using System;
using System.Linq;
using System.Reflection;
using System.Timers;
using log4net;
using Mono.Addins;
using Nini.Config;
using OpenSim.Framework;
using OpenSim.Framework.Console;
using OpenSim.Region.Framework.Interfaces;
using OpenSim.Region.Framework.Scenes;
[assembly: Addin("RegionWarmup", "1.0.0")]
[assembly: AddinDependency("OpenSim.Region.Framework", OpenSim.VersionInfo.VersionNumber)]
[assembly: AddinDescription("Region warmup: terrain touch, limited asset pre-touch, prime script VM, optional deep warmup tick.")]
[assembly: AddinAuthor("Christopher Händler")]
namespace OpenSimRegionWarmupHealthGuard.Modules
{
[Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "RegionWarmup")]
public class RegionWarmupModule : INonSharedRegionModule
{
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
private Scene _scene;
private bool _enabled;
// Config
private bool _warmupOnRegionLoaded = true;
private bool _preloadAssets = true;
private bool _primeScriptVm = true;
private bool _touchTerrain = true;
private int _deepWarmupLimit = 200;
private int _deepWarmupDelaySec = 30;
private Timer _deepTimer;
public string Name => "RegionWarmup";
public Type ReplaceableInterface => null;
public void Initialise(IConfigSource source)
{
var modules = source.Configs["Modules"];
if (modules == null) return;
var status = modules.GetString("RegionWarmup", string.Empty);
if (!string.Equals(status, "enabled", StringComparison.OrdinalIgnoreCase)) return;
var cfg = source.Configs["RegionWarmup"] ?? source.AddConfig("RegionWarmup");
_warmupOnRegionLoaded = cfg.GetBoolean("WarmupOnRegionLoaded", _warmupOnRegionLoaded);
_preloadAssets = cfg.GetBoolean("PreloadAssets", _preloadAssets);
_primeScriptVm = cfg.GetBoolean("PrimeScriptVM", _primeScriptVm);
_touchTerrain = cfg.GetBoolean("TouchTerrain", _touchTerrain);
_deepWarmupLimit = Math.Max(0, cfg.GetInt("DeepWarmupLimit", _deepWarmupLimit));
_deepWarmupDelaySec = Math.Max(0, cfg.GetInt("DeepWarmupDelaySec", _deepWarmupDelaySec));
_enabled = true;
Log.Info("[WARMUP] RegionWarmup enabled");
}
public void AddRegion(Scene scene)
{
if (!_enabled) return;
_scene = scene;
_scene.RegisterModuleInterface(this);
}
public void RegionLoaded(Scene scene)
{
if (!_enabled) return;
MainConsole.Instance.Commands.AddCommand("Region", false, "warmup",
"warmup",
"Warmup commands. Usage: warmup run|status",
HandleConsole);
if (_warmupOnRegionLoaded)
{
RunWarmup(tag: "startup");
}
if (_deepWarmupLimit > 0 && _deepWarmupDelaySec > 0)
{
_deepTimer = new Timer(_deepWarmupDelaySec * 1000) { AutoReset = false };
_deepTimer.Elapsed += (_, __) =>
{
try
{
DeepWarmup(_deepWarmupLimit);
}
catch (Exception e)
{
Log.Debug($"[WARMUP] deep warmup failed: {e.Message}");
}
};
_deepTimer.Start();
}
Log.Info($"[WARMUP] Ready for region '{_scene.RegionInfo.RegionName}'.");
}
public void RemoveRegion(Scene scene)
{
if (!_enabled) return;
try { _deepTimer?.Stop(); _deepTimer?.Dispose(); } catch { }
_deepTimer = null;
_scene.UnregisterModuleInterface(this);
_scene = null;
}
public void Close()
{
try { _deepTimer?.Stop(); _deepTimer?.Dispose(); } catch { }
_deepTimer = null;
}
private void HandleConsole(string module, string[] args)
{
if (_scene == null) { MainConsole.Instance.Output("[WARMUP] No active scene."); return; }
if (args.Length < 2)
{
MainConsole.Instance.Output("warmup status - show config");
MainConsole.Instance.Output("warmup run - run warmup now");
return;
}
switch (args[1].ToLowerInvariant())
{
case "status":
MainConsole.Instance.Output($"[WARMUP] OnLoad={_warmupOnRegionLoaded} Terrain={_touchTerrain} PreloadAssets={_preloadAssets} PrimeVM={_primeScriptVm} DeepLimit={_deepWarmupLimit} DeepDelay={_deepWarmupDelaySec}s");
break;
case "run":
RunWarmup(tag: "manual");
MainConsole.Instance.Output("[WARMUP] Warmup completed.");
break;
default:
MainConsole.Instance.Output("warmup status|run");
break;
}
}
private void RunWarmup(string tag)
{
if (_touchTerrain)
{
try
{
var t = _scene.TerrainChannel;
var x = _scene.RegionInfo.RegionSizeX / 2;
var y = _scene.RegionInfo.RegionSizeY / 2;
var h = t.GetNormalizedGroundHeight(x, y);
Log.Debug($"[WARMUP] terrain touch ok (h={h:0.00})");
}
catch (Exception e) { Log.Debug($"[WARMUP] terrain touch skipped: {e.Message}"); }
}
if (_preloadAssets)
{
try
{
foreach (var so in _scene.GetSceneObjectGroups().Take(50))
{
var _ = so.Parts?.Length;
foreach (var p in so.Parts)
_ = p.Inventory?.GetInventoryItems()?.Count;
}
Log.Debug("[WARMUP] limited asset pre-touch ok");
}
catch (Exception e) { Log.Debug($"[WARMUP] pre-touch skipped: {e.Message}"); }
}
if (_primeScriptVm)
{
try
{
var engine = _scene.RequestModuleInterface<IScriptModule>();
if (engine != null)
{
int cnt = engine.GetScriptCount();
Log.Debug($"[WARMUP] script VM primed (scripts={cnt})");
}
}
catch (Exception e) { Log.Debug($"[WARMUP] script VM prime skipped: {e.Message}"); }
}
Log.Info($"[WARMUP] Warmup done ({tag})");
}
private void DeepWarmup(int limit)
{
try
{
int scanned = 0;
foreach (var so in _scene.GetSceneObjectGroups())
{
var parts = so.Parts;
if (parts == null) continue;
foreach (var p in parts)
{
var _ = p.Inventory?.GetInventoryItems()?.Count;
}
scanned++;
if (scanned >= limit) break;
}
Log.Info($"[WARMUP] deep warmup scanned objects={scanned}/{limit}");
}
catch (Exception e)
{
Log.Debug($"[WARMUP] deep warmup error: {e.Message}");
}
}
}
}
@@ -0,0 +1,393 @@
using System;
using System.Collections.Generic;
using System.Net;
using System.Reflection;
using System.Text;
using System.Threading;
using log4net;
using Mono.Addins;
using Nini.Config;
using OpenSim.Framework;
using OpenSim.Framework.Console;
using OpenSim.Region.Framework.Interfaces;
using OpenSim.Region.Framework.Scenes;
using System.IO;
[assembly: Addin("RegionWebhookAlerts", "1.0.0")]
[assembly: AddinDependency("OpenSim.Region.Framework", OpenSim.VersionInfo.VersionNumber)]
[assembly: AddinDescription("Generic JSON webhook alerts (batching + rate-limit) for RegionHealthMonitor.")]
[assembly: AddinAuthor("Christopher Händler")]
namespace OpenSimRegionWarmupHealthGuard.Modules
{
public enum AlertSeverity { Trace = 0, Info = 1, Warn = 2, Error = 3 }
[Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "RegionWebhookAlerts")]
public class RegionWebhookAlertsModule : INonSharedRegionModule
{
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
private Scene _scene;
private bool _enabled;
// Config
private string _url = ""; // webhook endpoint (e.g., n8n)
private AlertSeverity _minSeverity = AlertSeverity.Warn;
private int _batchWindowSec = 10; // collect events for N seconds before sending
private int _rateLimitPerMin = 20; // max POSTs per minute
private int _connectTimeoutMs = 5000;
private int _sendTimeoutMs = 5000;
private string _payloadFields = "region,agents,prims,scriptMs,physMs,netMs,scriptErrors,uptime,severity,message,ts"; // selectable
// State
private IRegionHealthBus _bus;
private readonly object _queueLock = new object();
private readonly List<HealthEvent> _queue = new List<HealthEvent>();
private Timer _batchTimer;
private DateTime _windowStartUtc;
private int _sentThisMinute;
private DateTime _minuteWindowUtc;
public string Name => "RegionWebhookAlerts";
public Type ReplaceableInterface => null;
public void Initialise(IConfigSource source)
{
var modules = source.Configs["Modules"];
if (modules == null) return;
var status = modules.GetString("RegionWebhookAlerts", string.Empty);
if (!string.Equals(status, "enabled", StringComparison.OrdinalIgnoreCase)) return;
var cfg = source.Configs["RegionWebhookAlerts"] ?? source.AddConfig("RegionWebhookAlerts");
_url = cfg.GetString("Url", _url);
_minSeverity = ParseSeverity(cfg.GetString("MinSeverity", _minSeverity.ToString())) ?? _minSeverity;
_batchWindowSec = Math.Max(1, cfg.GetInt("BatchWindowSec", _batchWindowSec));
_rateLimitPerMin = Math.Max(1, cfg.GetInt("RateLimitPerMin", _rateLimitPerMin));
_connectTimeoutMs = Math.Max(1000, cfg.GetInt("ConnectTimeoutMs", _connectTimeoutMs));
_sendTimeoutMs = Math.Max(1000, cfg.GetInt("SendTimeoutMs", _sendTimeoutMs));
_payloadFields = cfg.GetString("PayloadFields", _payloadFields);
if (string.IsNullOrWhiteSpace(_url))
{
Log.Warn("[ALERTS] Url not configured. Module will remain idle.");
}
_enabled = true;
Log.Info($"[ALERTS] RegionWebhookAlerts enabled (min={_minSeverity}, batch={_batchWindowSec}s, rate={_rateLimitPerMin}/min)");
}
public void AddRegion(Scene scene)
{
if (!_enabled) return;
_scene = scene;
_bus = _scene.RequestModuleInterface<IRegionHealthBus>();
if (_bus == null)
{
Log.Warn("[ALERTS] No IRegionHealthBus found. Alerts will remain idle until RegionHealthMonitor is enabled.");
}
else
{
_bus.OnIncident += OnIncident;
}
_scene.RegisterModuleInterface(this);
}
public void RegionLoaded(Scene scene)
{
if (!_enabled) return;
MainConsole.Instance.Commands.AddCommand("Region", false, "alerts",
"alerts",
"Webhook alerts commands. Usage: alerts status|test \"Message...\"",
HandleConsole);
_windowStartUtc = DateTime.UtcNow;
_minuteWindowUtc = _windowStartUtc;
_batchTimer = new Timer(_ => FlushIfWindowElapsed(), null, _batchWindowSec * 1000, _batchWindowSec * 1000);
Log.Info($"[ALERTS] Ready for region '{_scene.RegionInfo.RegionName}'.");
}
public void RemoveRegion(Scene scene)
{
if (!_enabled) return;
try { _batchTimer?.Dispose(); } catch { }
_batchTimer = null;
if (_bus != null)
_bus.OnIncident -= OnIncident;
_scene.UnregisterModuleInterface(this);
_scene = null;
}
public void Close()
{
try { _batchTimer?.Dispose(); } catch { }
_batchTimer = null;
}
private void HandleConsole(string module, string[] args)
{
if (_scene == null) { MainConsole.Instance.Output("[ALERTS] No active scene."); return; }
if (args.Length < 2)
{
MainConsole.Instance.Output("alerts status - show current status");
MainConsole.Instance.Output("alerts test \"Message\" - enqueue a test incident with severity=Warn");
return;
}
switch (args[1].ToLowerInvariant())
{
case "status":
MainConsole.Instance.Output($"[ALERTS] Url={(string.IsNullOrWhiteSpace(_url) ? "(not set)" : _url)} " +
$"MinSeverity={_minSeverity} Batch={_batchWindowSec}s Rate={_rateLimitPerMin}/min");
break;
case "test":
{
string msg = args.Length >= 3 ? string.Join(" ", args, 2, args.Length - 2) : "Test alert";
var ev = new HealthEvent
{
TimestampUtc = DateTime.UtcNow,
Region = _scene.RegionInfo.RegionName,
Severity = HealthSeverity.Warn,
Message = msg,
Sample = null
};
OnIncident(ev);
MainConsole.Instance.Output("[ALERTS] Test event queued.");
break;
}
default:
MainConsole.Instance.Output("alerts status|test \"Message\"");
break;
}
}
private void OnIncident(HealthEvent e)
{
if (string.IsNullOrWhiteSpace(_url))
return;
var sev = MapSeverity(e.Severity);
if (sev < _minSeverity)
return;
lock (_queueLock)
{
_queue.Add(e);
}
}
private void FlushIfWindowElapsed()
{
try
{
var now = DateTime.UtcNow;
// rate-limit window
if ((now - _minuteWindowUtc).TotalSeconds >= 60)
{
_minuteWindowUtc = now;
_sentThisMinute = 0;
}
// batch window
if ((now - _windowStartUtc).TotalSeconds < _batchWindowSec)
return;
List<HealthEvent> batch;
lock (_queueLock)
{
if (_queue.Count == 0)
{
_windowStartUtc = now;
return;
}
batch = new List<HealthEvent>(_queue);
_queue.Clear();
_windowStartUtc = now;
}
if (_sentThisMinute >= _rateLimitPerMin)
{
Log.Warn("[ALERTS] Rate limit reached; dropping batch.");
return;
}
// Build JSON and POST
var payload = BuildJson(batch);
if (PostJson(_url, payload))
{
_sentThisMinute++;
}
}
catch (Exception ex)
{
Log.Warn($"[ALERTS] Flush failed: {ex.Message}");
}
}
private string BuildJson(List<HealthEvent> events)
{
// Minimal custom JSON builder (avoid external deps)
// PayloadFields controls which top-level fields/metrics are included.
var include = new HashSet<string>(SplitCsv(_payloadFields), StringComparer.OrdinalIgnoreCase);
var sb = new StringBuilder(events.Count * 256);
sb.Append('[');
for (int i = 0; i < events.Count; i++)
{
var e = events[i];
sb.Append('{');
bool first = true;
void add(string key, string value, bool raw = false)
{
if (!first) sb.Append(',');
first = false;
sb.Append('"').Append(E(key)).Append("\":");
if (raw) sb.Append(value);
else sb.Append('"').Append(E(value)).Append('"');
}
if (include.Contains("ts")) add("ts", e.TimestampUtc.ToString("o"));
if (include.Contains("region")) add("region", e.Region ?? _scene?.RegionInfo?.RegionName ?? "unknown");
if (include.Contains("severity")) add("severity", e.Severity.ToString());
if (include.Contains("message")) add("message", e.Message ?? "");
if (include.Contains("metrics"))
{
// combined object "metrics"
if (!first) sb.Append(',');
first = false;
sb.Append("\"metrics\":");
sb.Append('{');
bool fm = true;
void addm(string k, string v, bool raw = false)
{
if (!fm) sb.Append(',');
fm = false;
sb.Append('"').Append(E(k)).Append("\":");
if (raw) sb.Append(v);
else sb.Append('"').Append(E(v)).Append('"');
}
var s = e.Sample;
if (s != null)
{
if (include.Contains("agents")) addm("agents", s.Agents.ToString(), true);
if (include.Contains("prims")) addm("prims", s.Prims.ToString(), true);
if (include.Contains("scriptMs")) addm("scriptMs", s.ScriptTimeMs.ToString("0.###", System.Globalization.CultureInfo.InvariantCulture), true);
if (include.Contains("physMs")) addm("physMs", s.PhysicsTimeMs.ToString("0.###", System.Globalization.CultureInfo.InvariantCulture), true);
if (include.Contains("netMs")) addm("netMs", s.NetTimeMs.ToString("0.###", System.Globalization.CultureInfo.InvariantCulture), true);
if (include.Contains("scriptErrors")) addm("scriptErrors", s.ScriptErrors.ToString(), true);
if (include.Contains("uptime")) addm("uptimeSec", ((long)s.Uptime.TotalSeconds).ToString(), true);
}
sb.Append('}');
}
else
{
// flat metrics if requested
var s = e.Sample;
if (s != null)
{
if (include.Contains("agents")) add("agents", s.Agents.ToString(), true);
if (include.Contains("prims")) add("prims", s.Prims.ToString(), true);
if (include.Contains("scriptMs")) add("scriptMs", s.ScriptTimeMs.ToString("0.###", System.Globalization.CultureInfo.InvariantCulture), true);
if (include.Contains("physMs")) add("physMs", s.PhysicsTimeMs.ToString("0.###", System.Globalization.CultureInfo.InvariantCulture), true);
if (include.Contains("netMs")) add("netMs", s.NetTimeMs.ToString("0.###", System.Globalization.CultureInfo.InvariantCulture), true);
if (include.Contains("scriptErrors")) add("scriptErrors", s.ScriptErrors.ToString(), true);
if (include.Contains("uptime")) add("uptimeSec", ((long)s.Uptime.TotalSeconds).ToString(), true);
}
}
sb.Append('}');
if (i < events.Count - 1) sb.Append(',');
}
sb.Append(']');
return sb.ToString();
}
private bool PostJson(string url, string json)
{
try
{
var req = (HttpWebRequest)WebRequest.Create(url);
req.Method = "POST";
req.ContentType = "application/json";
req.Timeout = _connectTimeoutMs;
req.ReadWriteTimeout = _sendTimeoutMs;
var bytes = Encoding.UTF8.GetBytes(json);
req.ContentLength = bytes.Length;
using (var s = req.GetRequestStream())
s.Write(bytes, 0, bytes.Length);
using var resp = (HttpWebResponse)req.GetResponse();
var code = (int)resp.StatusCode;
if (code >= 200 && code < 300)
return true;
Log.Warn($"[ALERTS] Webhook returned HTTP {code}");
return false;
}
catch (WebException wex)
{
try
{
var resp = wex.Response as HttpWebResponse;
if (resp != null)
Log.Warn($"[ALERTS] Webhook failed: HTTP {(int)resp.StatusCode} {resp.StatusDescription}");
else
Log.Warn($"[ALERTS] Webhook failed: {wex.Message}");
}
catch { }
return false;
}
catch (Exception ex)
{
Log.Warn($"[ALERTS] Webhook exception: {ex.Message}");
return false;
}
}
private static IEnumerable<string> SplitCsv(string s)
{
if (string.IsNullOrWhiteSpace(s)) yield break;
foreach (var part in s.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries))
yield return part.Trim();
}
private static string E(string s)
{
if (string.IsNullOrEmpty(s)) return "";
return s.Replace("\\", "\\\\").Replace("\"", "\\\"");
}
private static AlertSeverity? ParseSeverity(string v)
{
if (Enum.TryParse<AlertSeverity>(v, true, out var sev))
return sev;
return null;
}
private static AlertSeverity MapSeverity(HealthSeverity sev)
{
return sev switch
{
HealthSeverity.Error => AlertSeverity.Error,
HealthSeverity.Warn => AlertSeverity.Warn,
HealthSeverity.Info => AlertSeverity.Info,
_ => AlertSeverity.Trace
};
}
}
}
@@ -0,0 +1,3 @@
# OpenSimRegionWarmupHealthGuard
> Warning - this module collection is under heavy development.
@@ -0,0 +1,20 @@
# RegionAutoHeal
## Configuration
~~~ini
[Modules]
RegionAutoHeal = enabled
[RegionAutoHeal]
; Sicherer Start: nur simulieren (Logs/Alerts), keine Änderungen
DryRun = true
; Echte Script-Resets nur bei expliziter Freigabe
EnableScriptReset = false
; Best-Effort-Drosselung „lauter“ Updater
ThrottleHeavyUpdaters = true
ThrottleThresholdUpdatesPerSec = 30
; Ab wann reagiert werden soll (Skriptfehler-Burst)
ScriptErrorBurstThreshold = 25
; Sperrzeit zwischen Eingriffen (Sekunden)
CooldownSec = 60
~~~
@@ -0,0 +1,22 @@
# RegionHealthMonitor
## Configuration
~~~ini
[Modules]
RegionHealthMonitor = enabled
[RegionHealthMonitor]
HealthIntervalSec = 30
WarnScriptTimeMs = 12.0
WarnPhysicsTimeMs = 6.0
WarnNetTimeMs = 6.0
WarnScriptErrors = 10
; Optional CSV export (leave empty to disable)
MetricsExportFile =
~~~
## Console commands
health status
health export
@@ -0,0 +1,24 @@
# RegionMetricsExporter
## Configuration
~~~ini
[Modules]
RegionMetricsExporter = enabled
[RegionMetricsExporter]
; HTTP pull endpoint for Prometheus (low overhead)
HttpPort = 9109
; Bind only to loopback by default (recommended). Set 0.0.0.0 to expose network-wide.
BindAddress = 127.0.0.1
; Metric name prefix
MetricsPrefix = opensim_region_
; Include region name as a label
IncludeRegionLabel = true
~~~
## Notes
Hinweise:
Dieses Modul liest HealthSamples vom RegionHealthMonitor (IRegionHealthBus). Stelle sicher, dass RegionHealthMonitor aktiv ist.
In Multi-Region-Prozessen sollte jede Region einen eigenen Port nutzen (oder du baust eine gemeinsame, prozessweite Export-Instanz).
Endpoint: http://BindAddress:HttpPort/metrics
Pull-only, minimaler Ressourcenverbrauch.
@@ -0,0 +1,32 @@
# RegionPolicyEngine
## Configuration
~~~ini
[Modules]
RegionPolicyEngine = enabled
[RegionPolicyEngine]
Enabled = true
; Prüfintervall (Sekunden)
CheckIntervalSec = 60
; Liste von Profilen (kommasepariert)
Profiles = Nightly,Event
; Einfaches Stundenfenster (Beispiele):
; 20-6 -> zwischen 20:00 und 06:59
; 8-18 -> zwischen 08:00 und 18:59
; 22 -> genau um 22 Uhr
Nightly.Cron = * * 20-6
; Key=Value;Key=Value (Overlays, die die Module verstehen müssen)
Nightly.Overrides = HealthIntervalSec=60; DeepWarmupLimit=400; MinSeverity=Error
Event.Cron =
Event.Overrides = ThrottleHeavyUpdaters=false
~~~
## Notes
Die Overlays sind bewusst generisch (Key=Value). Ein Modul, das Overlays unterstützen soll, implementiert optional IPolicyOverlayConsumer und mappt relevante Keys auf interne Settings.
Die mitgelieferten Module funktionieren auch ohne Overlay-Unterstützung. Du kannst Overlay-Support schrittweise in einzelnen Modulen ergänzen (z. B. HealthMonitor: HealthIntervalSec, WebhookAlerts: MinSeverity, Warmup: DeepWarmupLimit, AutoHeal: ThrottleHeavyUpdaters).
Das Cron-Feld ist hier stark vereinfacht (Stundenfenster). Für echtes Cron-Verhalten könnte später ein Parser ergänzt werden.
@@ -0,0 +1,19 @@
# RegionWarmup
## Configuration
~~~ini
[Modules]
RegionWarmup = enabled
[RegionWarmup]
; Sofortiges Warmup bei Regionstart
WarmupOnRegionLoaded = true
; Einzelne Warmup-Schritte
TouchTerrain = true
PreloadAssets = true
PrimeScriptVM = true
; Optionaler tiefer Warmup-Scan (Anzahl Objekte) nach Verzögerung
DeepWarmupLimit = 200
DeepWarmupDelaySec = 30
~~~
@@ -0,0 +1,35 @@
# RegionWebhookAlerts
## Configuration
~~~ini
[Modules]
RegionWebhookAlerts = enabled
[RegionWebhookAlerts]
; Generic JSON webhook endpoint (e.g., n8n webhook URL)
Url = https://your-n8n-host/webhook/abc123
; Minimal severity to send: Trace|Info|Warn|Error (default Warn)
MinSeverity = Warn
; Collect incidents for N seconds and send in one POST
BatchWindowSec = 10
; Max POSTs per minute (rate limit)
RateLimitPerMin = 20
; Timeouts (ms)
ConnectTimeoutMs = 5000
SendTimeoutMs = 5000
; Which fields to include in payload.
; Available keys:
; ts,region,severity,message,metrics,agents,prims,scriptMs,physMs,netMs,scriptErrors,uptime
PayloadFields = region,agents,prims,scriptMs,physMs,netMs,scriptErrors,uptime,severity,message,ts
~~~
## Notes
Unterstützt generisches JSON. Payload ist ein Array von Events.
Batching + Rate-Limit verhindern Spam. Für sofortige Zustellung BatchWindowSec verkleinern.
Abonniert Health-Events vom RegionHealthMonitor (IRegionHealthBus). Stelle sicher, dass dieser aktiviert ist.
@@ -0,0 +1,68 @@
<?xml version="1.0" encoding="utf-8"?>
<Project frameworkVersion="v8_0" name="OpenSimRegionWarmupHealthGuard" path="addon-modules/OpenSimRegionWarmupHealthGuard/Modules" type="Library" language="C#">
<!--
prebuild.xml for OpenSim addon-modules/OpenSimRegionWarmupHealthGuard
Builds a single assembly containing all region modules:
- RegionHealthMonitor
- RegionMetricsExporter
- RegionWebhookAlerts
- RegionAutoHeal
- RegionWarmup
- RegionPolicyEngine
-->
<Configuration name="Debug" platform="AnyCPU">
<Options>
<OutputPath>../../../bin/</OutputPath>
<Optimize>false</Optimize>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<WarningLevel>4</WarningLevel>
<DebugSymbols>true</DebugSymbols>
</Options>
</Configuration>
<Configuration name="Release" platform="AnyCPU">
<Options>
<OutputPath>../../../bin/</OutputPath>
<Optimize>true</Optimize>
<DefineConstants>TRACE</DefineConstants>
<WarningLevel>4</WarningLevel>
<DebugSymbols>false</DebugSymbols>
</Options>
</Configuration>
<ReferencePath>../../../bin/</ReferencePath>
<Reference name="System" />
<Reference name="System.Core" />
<Reference name="System.Xml" />
<Reference name="System.Xml.Linq" />
<Reference name="System.Net.Http" />
<Reference name="Mono.Addins" />
<Reference name="log4net" />
<Reference name="Nini"/>
<!-- OpenSim core references (relative to repo root) -->
<Reference name="OpenSim.Framework" />
<Reference name="OpenSim.Region.Framework" />
<Reference name="OpenMetaverseTypes" />
<Reference name="OpenMetaverse" />
<Files>
<!-- Source files -->
<Match pattern="addon-modules/OpenSimRegionWarmupHealthGuard/Modules/*.cs" />
<!--
<Match pattern="addon-modules/OpenSimRegionWarmupHealthGuard/Modules/RegionMetricsExporter.cs" />
<Match pattern="addon-modules/OpenSimRegionWarmupHealthGuard/Modules/RegionWebhookAlerts.cs" />
<Match pattern="addon-modules/OpenSimRegionWarmupHealthGuard/Modules/RegionAutoHeal.cs" />
<Match pattern="addon-modules/OpenSimRegionWarmupHealthGuard/Modules/RegionWarmupModule.cs" />
<Match pattern="addon-modules/OpenSimRegionWarmupHealthGuard/Modules/RegionPolicyEngineModule.cs" />
-->
<!-- Optional docs/examples (not compiled) -->
<Exclude pattern="addon-modules/OpenSimRegionWarmupHealthGuard/*.md" />
<Exclude pattern="addon-modules/OpenSimRegionWarmupHealthGuard/*.MD" />
<Exclude pattern="addon-modules/OpenSimRegionWarmupHealthGuard/*.ini.example" />
</Files>
</Project>