mirror of
https://github.com/jamenai/opensim-addon-modules-exp.git
synced 2026-07-27 21:10:25 +00:00
added OpenSimRegionWarmupHealthGuard Modules
This commit is contained in:
@@ -0,0 +1,355 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Reflection;
|
||||
using System.Threading;
|
||||
using log4net;
|
||||
using Mono.Addins;
|
||||
using Nini.Config;
|
||||
using OpenMetaverse;
|
||||
using OpenSim.Framework;
|
||||
using OpenSim.Framework.Console;
|
||||
using OpenSim.Region.Framework.Interfaces;
|
||||
using OpenSim.Region.Framework.Scenes;
|
||||
|
||||
[assembly: Addin("RegionAutoHeal", "1.0.0")]
|
||||
[assembly: AddinDependency("OpenSim.Region.Framework", OpenSim.VersionInfo.VersionNumber)]
|
||||
[assembly: AddinDescription("Auto-heal (dry-run by default): targeted script reset and best-effort throttling of heavy updaters.")]
|
||||
[assembly: AddinAuthor("Christopher Händler")]
|
||||
|
||||
namespace OpenSimRegionWarmupHealthGuard.Modules
|
||||
{
|
||||
[Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "RegionAutoHeal")]
|
||||
public class RegionAutoHealModule : INonSharedRegionModule
|
||||
{
|
||||
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
|
||||
|
||||
private Scene _scene;
|
||||
private bool _enabled;
|
||||
|
||||
// Config
|
||||
private bool _dryRun = true; // safe default
|
||||
private bool _enableScriptReset = false; // needs explicit opt-in
|
||||
private bool _throttleHeavyUpdaters = true;
|
||||
private int _throttleThresholdUpdatesPerSec = 30;
|
||||
private int _scriptErrorBurstThreshold = 25; // incident filter
|
||||
private int _cooldownSec = 60; // avoid repeated actions
|
||||
|
||||
// State
|
||||
private IRegionHealthBus _bus;
|
||||
private DateTime _lastActionUtc = DateTime.MinValue;
|
||||
private volatile int _opGuard;
|
||||
|
||||
public string Name => "RegionAutoHeal";
|
||||
public Type ReplaceableInterface => null;
|
||||
|
||||
public void Initialise(IConfigSource source)
|
||||
{
|
||||
var modules = source.Configs["Modules"];
|
||||
if (modules == null) return;
|
||||
|
||||
var status = modules.GetString("RegionAutoHeal", string.Empty);
|
||||
if (!string.Equals(status, "enabled", StringComparison.OrdinalIgnoreCase)) return;
|
||||
|
||||
var cfg = source.Configs["RegionAutoHeal"] ?? source.AddConfig("RegionAutoHeal");
|
||||
|
||||
_dryRun = cfg.GetBoolean("DryRun", _dryRun);
|
||||
_enableScriptReset = cfg.GetBoolean("EnableScriptReset", _enableScriptReset);
|
||||
_throttleHeavyUpdaters = cfg.GetBoolean("ThrottleHeavyUpdaters", _throttleHeavyUpdaters);
|
||||
_throttleThresholdUpdatesPerSec = Math.Max(1, cfg.GetInt("ThrottleThresholdUpdatesPerSec", _throttleThresholdUpdatesPerSec));
|
||||
_scriptErrorBurstThreshold = Math.Max(1, cfg.GetInt("ScriptErrorBurstThreshold", _scriptErrorBurstThreshold));
|
||||
_cooldownSec = Math.Max(10, cfg.GetInt("CooldownSec", _cooldownSec));
|
||||
|
||||
_enabled = true;
|
||||
Log.Info($"[AUTOHEAL] Enabled (DryRun={_dryRun}, Reset={_enableScriptReset}, Throttle={_throttleHeavyUpdaters})");
|
||||
}
|
||||
|
||||
public void AddRegion(Scene scene)
|
||||
{
|
||||
if (!_enabled) return;
|
||||
_scene = scene;
|
||||
|
||||
_bus = _scene.RequestModuleInterface<IRegionHealthBus>();
|
||||
if (_bus == null)
|
||||
{
|
||||
Log.Warn("[AUTOHEAL] No IRegionHealthBus found. Auto-heal will remain idle until RegionHealthMonitor is enabled.");
|
||||
}
|
||||
else
|
||||
{
|
||||
_bus.OnIncident += OnIncident;
|
||||
}
|
||||
|
||||
_scene.RegisterModuleInterface(this);
|
||||
}
|
||||
|
||||
public void RegionLoaded(Scene scene)
|
||||
{
|
||||
if (!_enabled) return;
|
||||
|
||||
MainConsole.Instance.Commands.AddCommand("Region", false, "autoheal",
|
||||
"autoheal",
|
||||
"Auto-heal commands. Usage: autoheal status|dryrun on|off|reset <objectId>",
|
||||
HandleConsole);
|
||||
|
||||
Log.Info($"[AUTOHEAL] Ready for region '{_scene.RegionInfo.RegionName}'.");
|
||||
}
|
||||
|
||||
public void RemoveRegion(Scene scene)
|
||||
{
|
||||
if (!_enabled) return;
|
||||
|
||||
if (_bus != null)
|
||||
_bus.OnIncident -= OnIncident;
|
||||
|
||||
_scene.UnregisterModuleInterface(this);
|
||||
_scene = null;
|
||||
}
|
||||
|
||||
public void Close()
|
||||
{
|
||||
}
|
||||
|
||||
private void HandleConsole(string module, string[] args)
|
||||
{
|
||||
if (_scene == null) { MainConsole.Instance.Output("[AUTOHEAL] No active scene."); return; }
|
||||
if (args.Length < 2)
|
||||
{
|
||||
PrintHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
switch (args[1].ToLowerInvariant())
|
||||
{
|
||||
case "status":
|
||||
MainConsole.Instance.Output($"[AUTOHEAL] DryRun={_dryRun} EnableScriptReset={_enableScriptReset} Throttle={_throttleHeavyUpdaters} ThresholdUpd/s={_throttleThresholdUpdatesPerSec} CooldownSec={_cooldownSec}");
|
||||
break;
|
||||
|
||||
case "dryrun":
|
||||
if (args.Length >= 3)
|
||||
{
|
||||
bool on = args[2].Equals("on", StringComparison.OrdinalIgnoreCase);
|
||||
bool off = args[2].Equals("off", StringComparison.OrdinalIgnoreCase);
|
||||
if (on || off)
|
||||
{
|
||||
_dryRun = on;
|
||||
MainConsole.Instance.Output($"[AUTOHEAL] DryRun={_dryRun}");
|
||||
break;
|
||||
}
|
||||
}
|
||||
PrintHelp();
|
||||
break;
|
||||
|
||||
case "reset":
|
||||
if (args.Length >= 3)
|
||||
{
|
||||
if (UUID.TryParse(args[2], out var id))
|
||||
{
|
||||
RunOnceSafe(() => TryResetObjectScripts(id, reason: "manual"));
|
||||
break;
|
||||
}
|
||||
MainConsole.Instance.Output("[AUTOHEAL] invalid UUID");
|
||||
break;
|
||||
}
|
||||
PrintHelp();
|
||||
break;
|
||||
|
||||
default:
|
||||
PrintHelp();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private void PrintHelp()
|
||||
{
|
||||
MainConsole.Instance.Output("autoheal status - show status");
|
||||
MainConsole.Instance.Output("autoheal dryrun on|off - toggle dry-run mode");
|
||||
MainConsole.Instance.Output("autoheal reset <objectId> - targeted script reset (requires EnableScriptReset=true)");
|
||||
}
|
||||
|
||||
private void OnIncident(HealthEvent e)
|
||||
{
|
||||
// Only react to WARN/ERROR and when thresholds suggest action
|
||||
if (e == null || e.Sample == null) return;
|
||||
if (e.Severity < HealthSeverity.Warn) return;
|
||||
|
||||
// basic cooldown
|
||||
if ((DateTime.UtcNow - _lastActionUtc).TotalSeconds < _cooldownSec)
|
||||
return;
|
||||
|
||||
// Heuristics: large script error burst
|
||||
if (e.Sample.ScriptErrors >= _scriptErrorBurstThreshold)
|
||||
{
|
||||
RunOnceSafe(() =>
|
||||
{
|
||||
// 1) find suspicious objects (heuristic, best-effort)
|
||||
var suspects = FindSuspectObjects(max: 10);
|
||||
|
||||
// 2) reset scripts (if enabled) or log suggestion
|
||||
foreach (var so in suspects)
|
||||
{
|
||||
TryResetObjectScripts(so.UUID, reason: $"auto: errorBurst({e.Sample.ScriptErrors})");
|
||||
}
|
||||
|
||||
// 3) throttle heavy updaters
|
||||
if (_throttleHeavyUpdaters)
|
||||
BestEffortThrottleHeavyUpdaters(max: 10);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private void RunOnceSafe(Action action)
|
||||
{
|
||||
if (Interlocked.Exchange(ref _opGuard, 1) == 1)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
action();
|
||||
_lastActionUtc = DateTime.UtcNow;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Warn($"[AUTOHEAL] action failed: {ex.Message}");
|
||||
}
|
||||
finally
|
||||
{
|
||||
Interlocked.Exchange(ref _opGuard, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Heuristic: find objects with many scripts or high part count as suspects
|
||||
private IEnumerable<SceneObjectGroup> FindSuspectObjects(int max)
|
||||
{
|
||||
var result = new List<SceneObjectGroup>(max);
|
||||
try
|
||||
{
|
||||
var all = _scene.GetSceneObjectGroups();
|
||||
// simple score: script count + part count
|
||||
foreach (var so in all)
|
||||
{
|
||||
int scripts = 0;
|
||||
try
|
||||
{
|
||||
foreach (var p in so.Parts)
|
||||
scripts += p.Inventory?.GetInventoryItems()?.Count ?? 0;
|
||||
}
|
||||
catch { }
|
||||
|
||||
int score = scripts + so.Parts?.Length ?? 0;
|
||||
// crude threshold
|
||||
if (score >= 20)
|
||||
{
|
||||
result.Add(so);
|
||||
if (result.Count >= max) break;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch { }
|
||||
Log.Info($"[AUTOHEAL] suspects found: {result.Count}");
|
||||
return result;
|
||||
}
|
||||
|
||||
private void TryResetObjectScripts(UUID objectId, string reason)
|
||||
{
|
||||
var sog = _scene.GetSceneObjectGroup(objectId);
|
||||
if (sog == null)
|
||||
{
|
||||
Log.Info($"[AUTOHEAL] object not found for reset: {objectId}");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!_enableScriptReset)
|
||||
{
|
||||
Log.Warn($"[AUTOHEAL] (dry) would reset scripts of '{sog.Name}' ({objectId}) reason={reason}");
|
||||
return;
|
||||
}
|
||||
|
||||
if (_dryRun)
|
||||
{
|
||||
Log.Warn($"[AUTOHEAL] (dry) reset scripts of '{sog.Name}' ({objectId}) reason={reason}");
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var engine = _scene.RequestModuleInterface<IScriptModule>();
|
||||
if (engine == null)
|
||||
{
|
||||
Log.Warn("[AUTOHEAL] no script engine, cannot reset scripts.");
|
||||
return;
|
||||
}
|
||||
|
||||
int count = 0;
|
||||
foreach (var part in sog.Parts)
|
||||
{
|
||||
try
|
||||
{
|
||||
// @todo: ResetScript????
|
||||
engine.ResetScript(part.UUID, UUID.Zero); // best-effort API call
|
||||
count++;
|
||||
}
|
||||
catch { }
|
||||
}
|
||||
Log.Warn($"[AUTOHEAL] reset scripts of '{sog.Name}' parts={count} reason={reason}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Warn($"[AUTOHEAL] reset failed: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private void BestEffortThrottleHeavyUpdaters(int max)
|
||||
{
|
||||
try
|
||||
{
|
||||
int throttled = 0;
|
||||
foreach (var so in _scene.GetSceneObjectGroups())
|
||||
{
|
||||
if (throttled >= max) break;
|
||||
|
||||
// crude detection: objects with many parts and recent updates
|
||||
var parts = so.Parts;
|
||||
if (parts == null || parts.Length < 1) continue;
|
||||
|
||||
int localUpdates = 0;
|
||||
foreach (var p in parts)
|
||||
{
|
||||
// Heuristic placeholder: if a part's last update was very recent, count it.
|
||||
// OpenSim does not provide a standard per-part updates/sec API; so we log suggestions.
|
||||
// If your fork exposes update rates, plug them here and compare to _throttleThresholdUpdatesPerSec.
|
||||
// For now we treat "big & busy" objects as candidates.
|
||||
if (p.UpdateFlag != 0) localUpdates++;
|
||||
}
|
||||
|
||||
if (parts.Length >= 10 || localUpdates >= 5)
|
||||
{
|
||||
if (_dryRun)
|
||||
{
|
||||
Log.Warn($"[AUTOHEAL] (dry) would throttle '{so.Name}' parts={parts.Length} updates~{localUpdates}/tick");
|
||||
}
|
||||
else
|
||||
{
|
||||
// Best-effort: disable constant updates we control (e.g., temporary turning off dynamics/particles where sensible)
|
||||
foreach (var p in parts)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Example: reduce alpha mode/particle sources if available
|
||||
// Without stable generic API, we only log
|
||||
}
|
||||
catch { }
|
||||
}
|
||||
Log.Warn($"[AUTOHEAL] throttled '{so.Name}' (best-effort)");
|
||||
}
|
||||
throttled++;
|
||||
}
|
||||
}
|
||||
if (throttled > 0)
|
||||
Log.Warn($"[AUTOHEAL] throttled objects: {throttled}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Warn($"[AUTOHEAL] throttle failed: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Timers;
|
||||
using log4net;
|
||||
using Mono.Addins;
|
||||
using Nini.Config;
|
||||
using OpenSim.Framework;
|
||||
using OpenSim.Framework.Console;
|
||||
using OpenSim.Region.Framework.Interfaces;
|
||||
using OpenSim.Region.Framework.Scenes;
|
||||
|
||||
[assembly: Addin("RegionHealthMonitor", "1.0.0")]
|
||||
[assembly: AddinDependency("OpenSim.Region.Framework", OpenSim.VersionInfo.VersionNumber)]
|
||||
[assembly: AddinDescription("Region health polling + threshold warnings + CSV export + health event bus.")]
|
||||
[assembly: AddinAuthor("Christopher Händler")]
|
||||
|
||||
namespace OpenSimRegionWarmupHealthGuard.Modules
|
||||
{
|
||||
// Simple health sample DTO
|
||||
public class HealthSample
|
||||
{
|
||||
public DateTime TimestampUtc;
|
||||
public string Region;
|
||||
public int Agents;
|
||||
public int Prims;
|
||||
public float ScriptTimeMs;
|
||||
public float PhysicsTimeMs;
|
||||
public float NetTimeMs;
|
||||
public int ScriptErrors;
|
||||
public TimeSpan Uptime;
|
||||
}
|
||||
|
||||
public enum HealthSeverity { Trace, Info, Warn, Error }
|
||||
|
||||
public class HealthEvent
|
||||
{
|
||||
public DateTime TimestampUtc;
|
||||
public string Region;
|
||||
public HealthSeverity Severity;
|
||||
public string Message;
|
||||
public HealthSample Sample; // optional snapshot of metrics
|
||||
}
|
||||
|
||||
// Event-bus interface for other modules (alerts/export/auto-heal/policy)
|
||||
public interface IRegionHealthBus
|
||||
{
|
||||
event Action<HealthSample> OnSample; // every poll
|
||||
event Action<HealthEvent> OnIncident; // only on rule/threshold violation
|
||||
|
||||
void PublishSample(HealthSample s);
|
||||
void PublishIncident(HealthEvent e);
|
||||
}
|
||||
|
||||
public class RegionHealthBus : IRegionHealthBus
|
||||
{
|
||||
public event Action<HealthSample> OnSample;
|
||||
public event Action<HealthEvent> OnIncident;
|
||||
|
||||
public void PublishSample(HealthSample s) => OnSample?.Invoke(s);
|
||||
public void PublishIncident(HealthEvent e) => OnIncident?.Invoke(e);
|
||||
}
|
||||
|
||||
[Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "RegionHealthMonitor")]
|
||||
public class RegionHealthMonitorModule : INonSharedRegionModule
|
||||
{
|
||||
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
|
||||
|
||||
private Scene _scene;
|
||||
private bool _enabled;
|
||||
|
||||
// Config
|
||||
private int _healthIntervalSec = 30;
|
||||
private float _warnScriptTimeMs = 12.0f;
|
||||
private float _warnPhysicsTimeMs = 6.0f;
|
||||
private float _warnNetTimeMs = 6.0f;
|
||||
private int _warnScriptErrors = 10;
|
||||
private string _metricsExportFile = ""; // optional CSV export
|
||||
|
||||
private Timer _healthTimer;
|
||||
private volatile int _opGuard;
|
||||
|
||||
// Expose bus for other modules
|
||||
private readonly RegionHealthBus _bus = new RegionHealthBus();
|
||||
public IRegionHealthBus Bus => _bus;
|
||||
|
||||
public string Name => "RegionHealthMonitor";
|
||||
public Type ReplaceableInterface => null;
|
||||
|
||||
public void Initialise(IConfigSource source)
|
||||
{
|
||||
var modules = source.Configs["Modules"];
|
||||
if (modules == null) return;
|
||||
|
||||
var status = modules.GetString("RegionHealthMonitor", string.Empty);
|
||||
if (!string.Equals(status, "enabled", StringComparison.OrdinalIgnoreCase)) return;
|
||||
|
||||
var cfg = source.Configs["RegionHealthMonitor"] ?? source.AddConfig("RegionHealthMonitor");
|
||||
|
||||
_healthIntervalSec = Math.Max(5, cfg.GetInt("HealthIntervalSec", _healthIntervalSec));
|
||||
_warnScriptTimeMs = (float)cfg.GetDouble("WarnScriptTimeMs", _warnScriptTimeMs);
|
||||
_warnPhysicsTimeMs = (float)cfg.GetDouble("WarnPhysicsTimeMs", _warnPhysicsTimeMs);
|
||||
_warnNetTimeMs = (float)cfg.GetDouble("WarnNetTimeMs", _warnNetTimeMs);
|
||||
_warnScriptErrors = Math.Max(0, cfg.GetInt("WarnScriptErrors", _warnScriptErrors));
|
||||
_metricsExportFile = cfg.GetString("MetricsExportFile", _metricsExportFile);
|
||||
|
||||
_enabled = true;
|
||||
Log.Info("[HEALTH] RegionHealthMonitor enabled");
|
||||
}
|
||||
|
||||
public void AddRegion(Scene scene)
|
||||
{
|
||||
if (!_enabled) return;
|
||||
_scene = scene;
|
||||
|
||||
// Make bus discoverable to other modules
|
||||
_scene.RegisterModuleInterface<IRegionHealthBus>(_bus);
|
||||
_scene.RegisterModuleInterface(this);
|
||||
}
|
||||
|
||||
public void RegionLoaded(Scene scene)
|
||||
{
|
||||
if (!_enabled) return;
|
||||
|
||||
MainConsole.Instance.Commands.AddCommand("Region", false, "health",
|
||||
"health",
|
||||
"Health monitor commands. Usage: health status|export",
|
||||
HandleConsole);
|
||||
|
||||
_healthTimer = new Timer(_healthIntervalSec * 1000) { AutoReset = true };
|
||||
_healthTimer.Elapsed += (_, __) => SafeRun("HealthPoll", PollHealth);
|
||||
_healthTimer.Start();
|
||||
|
||||
Log.Info($"[HEALTH] Ready for region '{_scene.RegionInfo.RegionName}'. Interval={_healthIntervalSec}s.");
|
||||
}
|
||||
|
||||
public void RemoveRegion(Scene scene)
|
||||
{
|
||||
if (!_enabled) return;
|
||||
_healthTimer?.Stop();
|
||||
_healthTimer?.Dispose();
|
||||
_healthTimer = null;
|
||||
|
||||
_scene.UnregisterModuleInterface<IRegionHealthBus>(_bus);
|
||||
_scene.UnregisterModuleInterface(this);
|
||||
_scene = null;
|
||||
}
|
||||
|
||||
public void Close()
|
||||
{
|
||||
_healthTimer?.Stop();
|
||||
_healthTimer?.Dispose();
|
||||
_healthTimer = null;
|
||||
}
|
||||
|
||||
private void HandleConsole(string module, string[] args)
|
||||
{
|
||||
if (_scene == null) { MainConsole.Instance.Output("[HEALTH] No active scene."); return; }
|
||||
if (args.Length < 2)
|
||||
{
|
||||
MainConsole.Instance.Output("health status - show current health");
|
||||
MainConsole.Instance.Output("health export - export one CSV line (if MetricsExportFile configured)");
|
||||
return;
|
||||
}
|
||||
|
||||
switch (args[1].ToLowerInvariant())
|
||||
{
|
||||
case "status":
|
||||
ReportHealth(toConsole: true);
|
||||
break;
|
||||
case "export":
|
||||
var path = ExportMetricsOnce();
|
||||
MainConsole.Instance.Output(string.IsNullOrEmpty(path)
|
||||
? "[HEALTH] Export disabled or failed."
|
||||
: $"[HEALTH] Exported metrics to: {path}");
|
||||
break;
|
||||
default:
|
||||
MainConsole.Instance.Output("health status|export");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private void SafeRun(string tag, Action action)
|
||||
{
|
||||
if (System.Threading.Interlocked.Exchange(ref _opGuard, 1) == 1)
|
||||
return;
|
||||
|
||||
try { action(); }
|
||||
catch (Exception e) { Log.Warn($"[HEALTH] {tag} failed: {e.Message}"); }
|
||||
finally { System.Threading.Interlocked.Exchange(ref _opGuard, 0); }
|
||||
}
|
||||
|
||||
private void PollHealth()
|
||||
{
|
||||
var s = CollectStats();
|
||||
|
||||
// Publish every sample
|
||||
_bus.PublishSample(s);
|
||||
|
||||
// Threshold checks -> incidents
|
||||
if (s.ScriptTimeMs >= _warnScriptTimeMs)
|
||||
PublishIncident(HealthSeverity.Warn, $"High script time: {s.ScriptTimeMs:0.00} ms", s);
|
||||
|
||||
if (s.PhysicsTimeMs >= _warnPhysicsTimeMs)
|
||||
PublishIncident(HealthSeverity.Warn, $"High physics time: {s.PhysicsTimeMs:0.00} ms", s);
|
||||
|
||||
if (s.NetTimeMs >= _warnNetTimeMs)
|
||||
PublishIncident(HealthSeverity.Warn, $"High network time: {s.NetTimeMs:0.00} ms", s);
|
||||
|
||||
if (s.ScriptErrors >= _warnScriptErrors)
|
||||
PublishIncident(HealthSeverity.Warn, $"High script errors: {s.ScriptErrors}", s);
|
||||
|
||||
ExportMetrics(s);
|
||||
}
|
||||
|
||||
private void PublishIncident(HealthSeverity sev, string msg, HealthSample s)
|
||||
{
|
||||
var e = new HealthEvent
|
||||
{
|
||||
TimestampUtc = DateTime.UtcNow,
|
||||
Region = _scene.RegionInfo.RegionName,
|
||||
Severity = sev,
|
||||
Message = msg,
|
||||
Sample = s
|
||||
};
|
||||
_bus.PublishIncident(e);
|
||||
if (sev >= HealthSeverity.Warn)
|
||||
Log.Warn($"[HEALTH] {msg}");
|
||||
else
|
||||
Log.Info($"[HEALTH] {msg}");
|
||||
}
|
||||
|
||||
private void ReportHealth(bool toConsole)
|
||||
{
|
||||
var s = CollectStats();
|
||||
var line = $"[Health] Agents={s.Agents} Prims={s.Prims} " +
|
||||
$"ScriptMs={s.ScriptTimeMs:0.00} PhysMs={s.PhysicsTimeMs:0.00} NetMs={s.NetTimeMs:0.00} " +
|
||||
$"ScriptErrors={s.ScriptErrors} Uptime={s.Uptime:hh\\:mm\\:ss}";
|
||||
if (toConsole) MainConsole.Instance.Output(line);
|
||||
Log.Info(line);
|
||||
}
|
||||
|
||||
private HealthSample CollectStats()
|
||||
{
|
||||
float scriptMs = 0f, physMs = 0f, netMs = 0f;
|
||||
try
|
||||
{
|
||||
var stats = _scene.StatsReporter; // 0.9.3.x provides this
|
||||
if (stats != null)
|
||||
{
|
||||
scriptMs = stats.ScriptMs;
|
||||
physMs = stats.PhysicsMs;
|
||||
netMs = stats.NetMs;
|
||||
}
|
||||
}
|
||||
catch { }
|
||||
|
||||
int agents = 0, prims = 0, scriptErrors = 0;
|
||||
try
|
||||
{
|
||||
agents = _scene.GetRootAgentCount() + _scene.GetChildAgentCount();
|
||||
prims = _scene.GetTotalObjectsCount();
|
||||
}
|
||||
catch { }
|
||||
|
||||
try
|
||||
{
|
||||
var engine = _scene.RequestModuleInterface<IScriptModule>();
|
||||
if (engine != null)
|
||||
scriptErrors = engine.GetScriptErrorCount();
|
||||
}
|
||||
catch { }
|
||||
|
||||
TimeSpan uptime = TimeSpan.Zero;
|
||||
try { uptime = DateTime.UtcNow - _scene.RegionInfo.StartupTime; } catch { }
|
||||
|
||||
return new HealthSample
|
||||
{
|
||||
TimestampUtc = DateTime.UtcNow,
|
||||
Region = _scene.RegionInfo.RegionName,
|
||||
Agents = agents,
|
||||
Prims = prims,
|
||||
ScriptTimeMs = scriptMs,
|
||||
PhysicsTimeMs = physMs,
|
||||
NetTimeMs = netMs,
|
||||
ScriptErrors = scriptErrors,
|
||||
Uptime = uptime
|
||||
};
|
||||
}
|
||||
|
||||
private void ExportMetrics(HealthSample s)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(_metricsExportFile))
|
||||
return;
|
||||
|
||||
bool header = !File.Exists(_metricsExportFile);
|
||||
using (var sw = new StreamWriter(_metricsExportFile, append: true))
|
||||
{
|
||||
if (header)
|
||||
sw.WriteLine("ts,region,agents,prims,script_ms,phys_ms,net_ms,script_errors,uptime_s");
|
||||
|
||||
sw.WriteLine($"{DateTime.UtcNow:O},{Csv(_scene.RegionInfo.RegionName)},{s.Agents},{s.Prims}," +
|
||||
$"{s.ScriptTimeMs:0.###},{s.PhysicsTimeMs:0.###},{s.NetTimeMs:0.###}," +
|
||||
$"{s.ScriptErrors},{(long)s.Uptime.TotalSeconds}");
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log.Warn($"[HEALTH] CSV export failed: {e.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private string ExportMetricsOnce()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(_metricsExportFile))
|
||||
return null;
|
||||
|
||||
var s = CollectStats();
|
||||
ExportMetrics(s);
|
||||
return _metricsExportFile;
|
||||
}
|
||||
catch { return null; }
|
||||
}
|
||||
|
||||
private string Csv(string s) => "\"" + s.Replace("\"", "\"\"") + "\"";
|
||||
}
|
||||
}
|
||||
+312
@@ -0,0 +1,312 @@
|
||||
using System;
|
||||
using System.Globalization;
|
||||
using System.Net;
|
||||
using System.Reflection;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using log4net;
|
||||
using Mono.Addins;
|
||||
using Nini.Config;
|
||||
using OpenSim.Framework;
|
||||
using OpenSim.Framework.Console;
|
||||
using OpenSim.Region.Framework.Interfaces;
|
||||
using OpenSim.Region.Framework.Scenes;
|
||||
|
||||
[assembly: Addin("RegionMetricsExporter", "1.0.0")]
|
||||
[assembly: AddinDependency("OpenSim.Region.Framework", OpenSim.VersionInfo.VersionNumber)]
|
||||
[assembly: AddinDescription("Prometheus (HTTP pull) metrics exporter for RegionHealthMonitor.")]
|
||||
[assembly: AddinAuthor("Christopher Händler")]
|
||||
|
||||
namespace OpenSimRegionWarmupHealthGuard.Modules
|
||||
{
|
||||
[Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "RegionMetricsExporter")]
|
||||
public class RegionMetricsExporterModule : INonSharedRegionModule
|
||||
{
|
||||
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
|
||||
|
||||
private Scene _scene;
|
||||
private bool _enabled;
|
||||
|
||||
// Config
|
||||
private int _httpPort = 9109;
|
||||
private string _metricsPrefix = "opensim_region_";
|
||||
private string _bindAddress = "127.0.0.1"; // default loopback for safety
|
||||
private bool _includeRegionLabel = true;
|
||||
|
||||
// State
|
||||
private HttpListener _listener;
|
||||
private Thread _httpThread;
|
||||
private volatile bool _httpRunning;
|
||||
private readonly object _lastLock = new object();
|
||||
private HealthSample _lastSample; // last health sample for this region
|
||||
|
||||
// Bus
|
||||
private IRegionHealthBus _bus;
|
||||
|
||||
public string Name => "RegionMetricsExporter";
|
||||
public Type ReplaceableInterface => null;
|
||||
|
||||
public void Initialise(IConfigSource source)
|
||||
{
|
||||
var modules = source.Configs["Modules"];
|
||||
if (modules == null) return;
|
||||
|
||||
var status = modules.GetString("RegionMetricsExporter", string.Empty);
|
||||
if (!string.Equals(status, "enabled", StringComparison.OrdinalIgnoreCase)) return;
|
||||
|
||||
var cfg = source.Configs["RegionMetricsExporter"] ?? source.AddConfig("RegionMetricsExporter");
|
||||
|
||||
_httpPort = Math.Max(1, cfg.GetInt("HttpPort", _httpPort));
|
||||
_metricsPrefix = cfg.GetString("MetricsPrefix", _metricsPrefix);
|
||||
_bindAddress = cfg.GetString("BindAddress", _bindAddress);
|
||||
_includeRegionLabel = cfg.GetBoolean("IncludeRegionLabel", _includeRegionLabel);
|
||||
|
||||
_enabled = true;
|
||||
Log.Info($"[METRICS] RegionMetricsExporter enabled (port={_httpPort}, bind={_bindAddress})");
|
||||
}
|
||||
|
||||
public void AddRegion(Scene scene)
|
||||
{
|
||||
if (!_enabled) return;
|
||||
_scene = scene;
|
||||
|
||||
// Discover health bus (provided by RegionHealthMonitor)
|
||||
_bus = _scene.RequestModuleInterface<IRegionHealthBus>();
|
||||
if (_bus == null)
|
||||
{
|
||||
Log.Warn("[METRICS] No IRegionHealthBus found. Metrics will remain empty until RegionHealthMonitor is enabled.");
|
||||
}
|
||||
else
|
||||
{
|
||||
_bus.OnSample += OnHealthSample;
|
||||
}
|
||||
|
||||
_scene.RegisterModuleInterface(this);
|
||||
}
|
||||
|
||||
public void RegionLoaded(Scene scene)
|
||||
{
|
||||
if (!_enabled) return;
|
||||
|
||||
// Start HTTP listener per region module instance (separate port or one shared port per process recommended)
|
||||
// Here: single-port per region instance. In multi-region processes, use different ports per region.
|
||||
StartHttp();
|
||||
|
||||
MainConsole.Instance.Commands.AddCommand("Region", false, "metrics",
|
||||
"metrics",
|
||||
"Metrics exporter commands. Usage: metrics status",
|
||||
HandleConsole);
|
||||
|
||||
Log.Info($"[METRICS] Ready for region '{_scene.RegionInfo.RegionName}'. Endpoint: http://{_bindAddress}:{_httpPort}/metrics");
|
||||
}
|
||||
|
||||
public void RemoveRegion(Scene scene)
|
||||
{
|
||||
if (!_enabled) return;
|
||||
|
||||
StopHttp();
|
||||
|
||||
if (_bus != null)
|
||||
_bus.OnSample -= OnHealthSample;
|
||||
|
||||
_scene.UnregisterModuleInterface(this);
|
||||
_scene = null;
|
||||
}
|
||||
|
||||
public void Close()
|
||||
{
|
||||
StopHttp();
|
||||
}
|
||||
|
||||
private void HandleConsole(string module, string[] args)
|
||||
{
|
||||
if (_scene == null) { MainConsole.Instance.Output("[METRICS] No active scene."); return; }
|
||||
if (args.Length < 2)
|
||||
{
|
||||
MainConsole.Instance.Output("metrics status - show exporter status");
|
||||
return;
|
||||
}
|
||||
|
||||
switch (args[1].ToLowerInvariant())
|
||||
{
|
||||
case "status":
|
||||
MainConsole.Instance.Output($"[METRICS] Endpoint: http://{_bindAddress}:{_httpPort}/metrics Running={_httpRunning}");
|
||||
break;
|
||||
default:
|
||||
MainConsole.Instance.Output("metrics status");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private void OnHealthSample(HealthSample s)
|
||||
{
|
||||
lock (_lastLock)
|
||||
{
|
||||
_lastSample = s;
|
||||
}
|
||||
}
|
||||
|
||||
private void StartHttp()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (_httpRunning) return;
|
||||
|
||||
_listener = new HttpListener();
|
||||
_listener.Prefixes.Add($"http://{_bindAddress}:{_httpPort}/metrics/");
|
||||
// Allow also without trailing slash
|
||||
_listener.Prefixes.Add($"http://{_bindAddress}:{_httpPort}/metrics");
|
||||
_listener.Start();
|
||||
|
||||
_httpRunning = true;
|
||||
_httpThread = new Thread(HttpLoop) { IsBackground = true, Name = "RegionMetricsExporter-HTTP" };
|
||||
_httpThread.Start();
|
||||
|
||||
Log.Info($"[METRICS] HTTP metrics endpoint started on http://{_bindAddress}:{_httpPort}/metrics");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log.Error($"[METRICS] Failed to start HTTP listener: {e.Message}");
|
||||
_httpRunning = false;
|
||||
try { _listener?.Stop(); } catch { }
|
||||
_listener = null;
|
||||
}
|
||||
}
|
||||
|
||||
private void StopHttp()
|
||||
{
|
||||
try
|
||||
{
|
||||
_httpRunning = false;
|
||||
try { _listener?.Stop(); } catch { }
|
||||
try { _listener?.Close(); } catch { }
|
||||
_listener = null;
|
||||
}
|
||||
catch { }
|
||||
|
||||
if (_httpThread != null)
|
||||
{
|
||||
try { _httpThread.Join(1000); } catch { }
|
||||
_httpThread = null;
|
||||
}
|
||||
}
|
||||
|
||||
private void HttpLoop()
|
||||
{
|
||||
while (_httpRunning && _listener != null)
|
||||
{
|
||||
HttpListenerContext ctx = null;
|
||||
try
|
||||
{
|
||||
ctx = _listener.GetContext();
|
||||
}
|
||||
catch
|
||||
{
|
||||
if (!_httpRunning) break;
|
||||
continue;
|
||||
}
|
||||
|
||||
ThreadPool.QueueUserWorkItem(_ => HandleRequest(ctx));
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleRequest(HttpListenerContext ctx)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (ctx.Request.HttpMethod != "GET")
|
||||
{
|
||||
ctx.Response.StatusCode = 405;
|
||||
ctx.Response.Close();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!ctx.Request.Url.AbsolutePath.StartsWith("/metrics", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
ctx.Response.StatusCode = 404;
|
||||
ctx.Response.Close();
|
||||
return;
|
||||
}
|
||||
|
||||
var text = BuildPrometheusText();
|
||||
var buffer = Encoding.UTF8.GetBytes(text);
|
||||
ctx.Response.StatusCode = 200;
|
||||
ctx.Response.ContentType = "text/plain; version=0.0.4; charset=utf-8";
|
||||
ctx.Response.ContentEncoding = Encoding.UTF8;
|
||||
ctx.Response.ContentLength64 = buffer.Length;
|
||||
ctx.Response.OutputStream.Write(buffer, 0, buffer.Length);
|
||||
ctx.Response.OutputStream.Close();
|
||||
}
|
||||
catch
|
||||
{
|
||||
try { ctx.Response.StatusCode = 500; ctx.Response.Close(); } catch { }
|
||||
}
|
||||
}
|
||||
|
||||
private string BuildPrometheusText()
|
||||
{
|
||||
HealthSample s;
|
||||
lock (_lastLock)
|
||||
{
|
||||
s = _lastSample;
|
||||
}
|
||||
|
||||
var region = _scene?.RegionInfo?.RegionName ?? "unknown";
|
||||
var sb = new StringBuilder(512);
|
||||
var prefix = _metricsPrefix;
|
||||
|
||||
// HELP/TYPE
|
||||
AppendHelpType(sb, prefix + "agents", "Number of agents (root + child) in region", "gauge");
|
||||
AppendHelpType(sb, prefix + "prims", "Total number of prims/objects in region", "gauge");
|
||||
AppendHelpType(sb, prefix + "script_time_ms", "Script time (ms)", "gauge");
|
||||
AppendHelpType(sb, prefix + "physics_time_ms", "Physics time (ms)", "gauge");
|
||||
AppendHelpType(sb, prefix + "net_time_ms", "Network time (ms)", "gauge");
|
||||
AppendHelpType(sb, prefix + "script_errors", "Script error count", "gauge");
|
||||
AppendHelpType(sb, prefix + "uptime_seconds", "Region uptime (seconds)", "gauge");
|
||||
AppendHelpType(sb, prefix + "sample_timestamp_seconds", "Last sample timestamp (unix seconds)", "gauge");
|
||||
|
||||
var labels = _includeRegionLabel ? $"{{region=\"{EscapeLabel(region)}\"}}" : "";
|
||||
|
||||
if (s == null)
|
||||
{
|
||||
// no sample yet: export zeros
|
||||
sb.AppendLine($"{prefix}agents{labels} 0");
|
||||
sb.AppendLine($"{prefix}prims{labels} 0");
|
||||
sb.AppendLine($"{prefix}script_time_ms{labels} 0");
|
||||
sb.AppendLine($"{prefix}physics_time_ms{labels} 0");
|
||||
sb.AppendLine($"{prefix}net_time_ms{labels} 0");
|
||||
sb.AppendLine($"{prefix}script_errors{labels} 0");
|
||||
sb.AppendLine($"{prefix}uptime_seconds{labels} 0");
|
||||
sb.AppendLine($"{prefix}sample_timestamp_seconds{labels} 0");
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
// Invariant culture for dots
|
||||
var ci = CultureInfo.InvariantCulture;
|
||||
|
||||
sb.AppendLine($"{prefix}agents{labels} {s.Agents.ToString(ci)}");
|
||||
sb.AppendLine($"{prefix}prims{labels} {s.Prims.ToString(ci)}");
|
||||
sb.AppendLine($"{prefix}script_time_ms{labels} {s.ScriptTimeMs.ToString(ci)}");
|
||||
sb.AppendLine($"{prefix}physics_time_ms{labels} {s.PhysicsTimeMs.ToString(ci)}");
|
||||
sb.AppendLine($"{prefix}net_time_ms{labels} {s.NetTimeMs.ToString(ci)}");
|
||||
sb.AppendLine($"{prefix}script_errors{labels} {s.ScriptErrors.ToString(ci)}");
|
||||
sb.AppendLine($"{prefix}uptime_seconds{labels} {((long)s.Uptime.TotalSeconds).ToString(ci)}");
|
||||
var unix = new DateTimeOffset(s.TimestampUtc).ToUnixTimeSeconds();
|
||||
sb.AppendLine($"{prefix}sample_timestamp_seconds{labels} {unix.ToString(ci)}");
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
private static void AppendHelpType(StringBuilder sb, string name, string help, string type)
|
||||
{
|
||||
sb.AppendLine($"# HELP {name} {help}");
|
||||
sb.AppendLine($"# TYPE {name} {type}");
|
||||
}
|
||||
|
||||
private static string EscapeLabel(string value)
|
||||
{
|
||||
if (string.IsNullOrEmpty(value)) return "";
|
||||
return value.Replace("\\", "\\\\").Replace("\"", "\\\"");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Timers;
|
||||
using log4net;
|
||||
using Mono.Addins;
|
||||
using Nini.Config;
|
||||
using OpenSim.Framework;
|
||||
using OpenSim.Framework.Console;
|
||||
using OpenSim.Region.Framework.Interfaces;
|
||||
using OpenSim.Region.Framework.Scenes;
|
||||
|
||||
[assembly: Addin("RegionPolicyEngine", "1.0.0")]
|
||||
[assembly: AddinDependency("OpenSim.Region.Framework", OpenSim.VersionInfo.VersionNumber)]
|
||||
[assembly: AddinDescription("Applies time-based/profile overlays to other modules' configs (cooperative).")]
|
||||
[assembly: AddinAuthor("Christopher Händler")]
|
||||
|
||||
namespace OpenSimRegionWarmupHealthGuard.Modules
|
||||
{
|
||||
public interface IPolicyOverlayConsumer
|
||||
{
|
||||
// Modules can implement this to accept overlays at runtime.
|
||||
// Keys/values are simple strings; module maps to local settings.
|
||||
void ApplyPolicyOverlay(IDictionary<string, string> overrides, string sourceProfile);
|
||||
}
|
||||
|
||||
[Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "RegionPolicyEngine")]
|
||||
public class RegionPolicyEngineModule : INonSharedRegionModule
|
||||
{
|
||||
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
|
||||
|
||||
private Scene _scene;
|
||||
private bool _enabled;
|
||||
|
||||
private bool _engineEnabled = true;
|
||||
private string[] _profiles = Array.Empty<string>();
|
||||
private readonly Dictionary<string, string> _profileCron = new();
|
||||
private readonly Dictionary<string, Dictionary<string, string>> _profileOverrides = new();
|
||||
|
||||
private Timer _tick;
|
||||
private int _intervalSec = 60; // check every minute
|
||||
|
||||
public string Name => "RegionPolicyEngine";
|
||||
public Type ReplaceableInterface => null;
|
||||
|
||||
public void Initialise(IConfigSource source)
|
||||
{
|
||||
var modules = source.Configs["Modules"];
|
||||
if (modules == null) return;
|
||||
|
||||
var status = modules.GetString("RegionPolicyEngine", string.Empty);
|
||||
if (!string.Equals(status, "enabled", StringComparison.OrdinalIgnoreCase)) return;
|
||||
|
||||
var cfg = source.Configs["RegionPolicyEngine"] ?? source.AddConfig("RegionPolicyEngine");
|
||||
|
||||
_engineEnabled = cfg.GetBoolean("Enabled", _engineEnabled);
|
||||
_intervalSec = Math.Max(15, cfg.GetInt("CheckIntervalSec", _intervalSec));
|
||||
|
||||
var profilesCsv = cfg.GetString("Profiles", "");
|
||||
_profiles = profilesCsv.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries)
|
||||
.Select(s => s.Trim())
|
||||
.ToArray();
|
||||
|
||||
foreach (var p in _profiles)
|
||||
{
|
||||
var cron = cfg.GetString($"{p}.Cron", "");
|
||||
var overridesLine = cfg.GetString($"{p}.Overrides", "");
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(cron))
|
||||
_profileCron[p] = cron;
|
||||
|
||||
var dict = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var kv in overridesLine.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries))
|
||||
{
|
||||
var i = kv.IndexOf('=');
|
||||
if (i > 0)
|
||||
{
|
||||
var k = kv.Substring(0, i).Trim();
|
||||
var v = kv.Substring(i + 1).Trim();
|
||||
if (k.Length > 0) dict[k] = v;
|
||||
}
|
||||
}
|
||||
_profileOverrides[p] = dict;
|
||||
}
|
||||
|
||||
_enabled = _engineEnabled;
|
||||
Log.Info($"[POLICY] RegionPolicyEngine enabled={_engineEnabled} profiles={string.Join(",", _profiles)}");
|
||||
}
|
||||
|
||||
public void AddRegion(Scene scene)
|
||||
{
|
||||
if (!_enabled) return;
|
||||
_scene = scene;
|
||||
_scene.RegisterModuleInterface(this);
|
||||
}
|
||||
|
||||
public void RegionLoaded(Scene scene)
|
||||
{
|
||||
if (!_enabled) return;
|
||||
|
||||
MainConsole.Instance.Commands.AddCommand("Region", false, "policy",
|
||||
"policy",
|
||||
"Policy engine commands. Usage: policy status|apply <Profile>|dryrun <Profile>",
|
||||
HandleConsole);
|
||||
|
||||
_tick = new Timer(_intervalSec * 1000) { AutoReset = true };
|
||||
_tick.Elapsed += (_, __) => EvaluateAndApply();
|
||||
_tick.Start();
|
||||
|
||||
Log.Info($"[POLICY] Ready for region '{_scene.RegionInfo.RegionName}'. check={_intervalSec}s");
|
||||
}
|
||||
|
||||
public void RemoveRegion(Scene scene)
|
||||
{
|
||||
if (!_enabled) return;
|
||||
|
||||
try { _tick?.Stop(); _tick?.Dispose(); } catch { }
|
||||
_tick = null;
|
||||
|
||||
_scene.UnregisterModuleInterface(this);
|
||||
_scene = null;
|
||||
}
|
||||
|
||||
public void Close()
|
||||
{
|
||||
try { _tick?.Stop(); _tick?.Dispose(); } catch { }
|
||||
_tick = null;
|
||||
}
|
||||
|
||||
private void HandleConsole(string module, string[] args)
|
||||
{
|
||||
if (_scene == null) { MainConsole.Instance.Output("[POLICY] No active scene."); return; }
|
||||
if (args.Length < 2)
|
||||
{
|
||||
PrintHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
var cmd = args[1].ToLowerInvariant();
|
||||
if (cmd == "status")
|
||||
{
|
||||
MainConsole.Instance.Output($"[POLICY] Enabled={_engineEnabled} Profiles={string.Join(",", _profiles)}");
|
||||
foreach (var p in _profiles)
|
||||
{
|
||||
_profileCron.TryGetValue(p, out var cron);
|
||||
_profileOverrides.TryGetValue(p, out var dict);
|
||||
MainConsole.Instance.Output($" - {p}: Cron='{cron}' Overrides='{string.Join(";", dict.Select(kv => kv.Key + \"=\" + kv.Value))}'");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if ((cmd == "apply" || cmd == "dryrun") && args.Length >= 3)
|
||||
{
|
||||
var profile = args[2];
|
||||
if (!_profileOverrides.TryGetValue(profile, out var ov))
|
||||
{
|
||||
MainConsole.Instance.Output($"[POLICY] Profile not found: {profile}");
|
||||
return;
|
||||
}
|
||||
|
||||
if (cmd == "dryrun")
|
||||
{
|
||||
MainConsole.Instance.Output($"[POLICY] DryRun profile={profile} -> {string.Join(";", ov.Select(kv => kv.Key + \"=\" + kv.Value))}");
|
||||
return;
|
||||
}
|
||||
|
||||
ApplyOverlay(ov, profile);
|
||||
MainConsole.Instance.Output($"[POLICY] Applied profile={profile}");
|
||||
return;
|
||||
}
|
||||
|
||||
PrintHelp();
|
||||
}
|
||||
|
||||
private void PrintHelp()
|
||||
{
|
||||
MainConsole.Instance.Output("policy status - show profiles and config");
|
||||
MainConsole.Instance.Output("policy apply <Profile> - apply overrides now");
|
||||
MainConsole.Instance.Output("policy dryrun <Profile> - show what would be applied");
|
||||
}
|
||||
|
||||
private void EvaluateAndApply()
|
||||
{
|
||||
// Minimalistic time matcher:
|
||||
// Cron format (simplified): mm HH hhRange? (e.g., "0 0 20-6" -> every minute between 20 and 6)
|
||||
// For real cron parsing you could add a tiny parser; here we implement a simple hours window.
|
||||
var now = DateTime.Now;
|
||||
foreach (var p in _profiles)
|
||||
{
|
||||
if (!_profileCron.TryGetValue(p, out var cron) || string.IsNullOrWhiteSpace(cron))
|
||||
continue;
|
||||
|
||||
// Expect format: "0 0 20-6" OR "* * 20-6", we only check hours window
|
||||
var parts = cron.Split(new[] { ' ', '\t' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
string hours = parts.Length >= 3 ? parts[2] : parts.LastOrDefault();
|
||||
if (string.IsNullOrWhiteSpace(hours)) continue;
|
||||
|
||||
bool inWindow = IsHourInWindow(now.Hour, hours);
|
||||
if (!inWindow) continue;
|
||||
|
||||
if (_profileOverrides.TryGetValue(p, out var ov))
|
||||
{
|
||||
ApplyOverlay(ov, p);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsHourInWindow(int hour, string expr)
|
||||
{
|
||||
// Accept "20-6" or "8-18" or single "22"
|
||||
if (expr.Contains("-"))
|
||||
{
|
||||
var sp = expr.Split('-');
|
||||
if (sp.Length != 2) return false;
|
||||
if (!int.TryParse(sp[0], out var a)) return false;
|
||||
if (!int.TryParse(sp[1], out var b)) return false;
|
||||
a = (a % 24 + 24) % 24;
|
||||
b = (b % 24 + 24) % 24;
|
||||
|
||||
if (a <= b) return hour >= a && hour <= b; // e.g., 8-18
|
||||
else return hour >= a || hour <= b; // e.g., 20-6 (wrap)
|
||||
}
|
||||
else
|
||||
{
|
||||
return int.TryParse(expr, out var h) && h == hour;
|
||||
}
|
||||
}
|
||||
|
||||
private void ApplyOverlay(IDictionary<string, string> ov, string profile)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Discover consumers and deliver overlay
|
||||
var consumers = new List<IPolicyOverlayConsumer>();
|
||||
|
||||
var health = _scene.RequestModuleInterface<RegionHealthMonitorModule>();
|
||||
if (health is IPolicyOverlayConsumer c1) consumers.Add(c1);
|
||||
|
||||
var warmup = _scene.RequestModuleInterface<RegionWarmupModule>();
|
||||
if (warmup is IPolicyOverlayConsumer c2) consumers.Add(c2);
|
||||
|
||||
var alerts = _scene.RequestModuleInterface<RegionWebhookAlertsModule>();
|
||||
if (alerts is IPolicyOverlayConsumer c3) consumers.Add(c3);
|
||||
|
||||
var auto = _scene.RequestModuleInterface<RegionAutoHealModule>();
|
||||
if (auto is IPolicyOverlayConsumer c4) consumers.Add(c4);
|
||||
|
||||
foreach (var c in consumers.Distinct())
|
||||
{
|
||||
try { c.ApplyPolicyOverlay(ov, profile); }
|
||||
catch (Exception e) { Log.Debug($"[POLICY] overlay error: {e.Message}"); }
|
||||
}
|
||||
|
||||
Log.Info($"[POLICY] overlay applied profile={profile} targets={consumers.Count}");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log.Debug($"[POLICY] apply failed: {e.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Timers;
|
||||
using log4net;
|
||||
using Mono.Addins;
|
||||
using Nini.Config;
|
||||
using OpenSim.Framework;
|
||||
using OpenSim.Framework.Console;
|
||||
using OpenSim.Region.Framework.Interfaces;
|
||||
using OpenSim.Region.Framework.Scenes;
|
||||
|
||||
[assembly: Addin("RegionWarmup", "1.0.0")]
|
||||
[assembly: AddinDependency("OpenSim.Region.Framework", OpenSim.VersionInfo.VersionNumber)]
|
||||
[assembly: AddinDescription("Region warmup: terrain touch, limited asset pre-touch, prime script VM, optional deep warmup tick.")]
|
||||
[assembly: AddinAuthor("Christopher Händler")]
|
||||
|
||||
namespace OpenSimRegionWarmupHealthGuard.Modules
|
||||
{
|
||||
[Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "RegionWarmup")]
|
||||
public class RegionWarmupModule : INonSharedRegionModule
|
||||
{
|
||||
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
|
||||
|
||||
private Scene _scene;
|
||||
private bool _enabled;
|
||||
|
||||
// Config
|
||||
private bool _warmupOnRegionLoaded = true;
|
||||
private bool _preloadAssets = true;
|
||||
private bool _primeScriptVm = true;
|
||||
private bool _touchTerrain = true;
|
||||
private int _deepWarmupLimit = 200;
|
||||
private int _deepWarmupDelaySec = 30;
|
||||
|
||||
private Timer _deepTimer;
|
||||
|
||||
public string Name => "RegionWarmup";
|
||||
public Type ReplaceableInterface => null;
|
||||
|
||||
public void Initialise(IConfigSource source)
|
||||
{
|
||||
var modules = source.Configs["Modules"];
|
||||
if (modules == null) return;
|
||||
|
||||
var status = modules.GetString("RegionWarmup", string.Empty);
|
||||
if (!string.Equals(status, "enabled", StringComparison.OrdinalIgnoreCase)) return;
|
||||
|
||||
var cfg = source.Configs["RegionWarmup"] ?? source.AddConfig("RegionWarmup");
|
||||
|
||||
_warmupOnRegionLoaded = cfg.GetBoolean("WarmupOnRegionLoaded", _warmupOnRegionLoaded);
|
||||
_preloadAssets = cfg.GetBoolean("PreloadAssets", _preloadAssets);
|
||||
_primeScriptVm = cfg.GetBoolean("PrimeScriptVM", _primeScriptVm);
|
||||
_touchTerrain = cfg.GetBoolean("TouchTerrain", _touchTerrain);
|
||||
_deepWarmupLimit = Math.Max(0, cfg.GetInt("DeepWarmupLimit", _deepWarmupLimit));
|
||||
_deepWarmupDelaySec = Math.Max(0, cfg.GetInt("DeepWarmupDelaySec", _deepWarmupDelaySec));
|
||||
|
||||
_enabled = true;
|
||||
Log.Info("[WARMUP] RegionWarmup enabled");
|
||||
}
|
||||
|
||||
public void AddRegion(Scene scene)
|
||||
{
|
||||
if (!_enabled) return;
|
||||
_scene = scene;
|
||||
_scene.RegisterModuleInterface(this);
|
||||
}
|
||||
|
||||
public void RegionLoaded(Scene scene)
|
||||
{
|
||||
if (!_enabled) return;
|
||||
|
||||
MainConsole.Instance.Commands.AddCommand("Region", false, "warmup",
|
||||
"warmup",
|
||||
"Warmup commands. Usage: warmup run|status",
|
||||
HandleConsole);
|
||||
|
||||
if (_warmupOnRegionLoaded)
|
||||
{
|
||||
RunWarmup(tag: "startup");
|
||||
}
|
||||
|
||||
if (_deepWarmupLimit > 0 && _deepWarmupDelaySec > 0)
|
||||
{
|
||||
_deepTimer = new Timer(_deepWarmupDelaySec * 1000) { AutoReset = false };
|
||||
_deepTimer.Elapsed += (_, __) =>
|
||||
{
|
||||
try
|
||||
{
|
||||
DeepWarmup(_deepWarmupLimit);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log.Debug($"[WARMUP] deep warmup failed: {e.Message}");
|
||||
}
|
||||
};
|
||||
_deepTimer.Start();
|
||||
}
|
||||
|
||||
Log.Info($"[WARMUP] Ready for region '{_scene.RegionInfo.RegionName}'.");
|
||||
}
|
||||
|
||||
public void RemoveRegion(Scene scene)
|
||||
{
|
||||
if (!_enabled) return;
|
||||
|
||||
try { _deepTimer?.Stop(); _deepTimer?.Dispose(); } catch { }
|
||||
_deepTimer = null;
|
||||
|
||||
_scene.UnregisterModuleInterface(this);
|
||||
_scene = null;
|
||||
}
|
||||
|
||||
public void Close()
|
||||
{
|
||||
try { _deepTimer?.Stop(); _deepTimer?.Dispose(); } catch { }
|
||||
_deepTimer = null;
|
||||
}
|
||||
|
||||
private void HandleConsole(string module, string[] args)
|
||||
{
|
||||
if (_scene == null) { MainConsole.Instance.Output("[WARMUP] No active scene."); return; }
|
||||
if (args.Length < 2)
|
||||
{
|
||||
MainConsole.Instance.Output("warmup status - show config");
|
||||
MainConsole.Instance.Output("warmup run - run warmup now");
|
||||
return;
|
||||
}
|
||||
|
||||
switch (args[1].ToLowerInvariant())
|
||||
{
|
||||
case "status":
|
||||
MainConsole.Instance.Output($"[WARMUP] OnLoad={_warmupOnRegionLoaded} Terrain={_touchTerrain} PreloadAssets={_preloadAssets} PrimeVM={_primeScriptVm} DeepLimit={_deepWarmupLimit} DeepDelay={_deepWarmupDelaySec}s");
|
||||
break;
|
||||
case "run":
|
||||
RunWarmup(tag: "manual");
|
||||
MainConsole.Instance.Output("[WARMUP] Warmup completed.");
|
||||
break;
|
||||
default:
|
||||
MainConsole.Instance.Output("warmup status|run");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private void RunWarmup(string tag)
|
||||
{
|
||||
if (_touchTerrain)
|
||||
{
|
||||
try
|
||||
{
|
||||
var t = _scene.TerrainChannel;
|
||||
var x = _scene.RegionInfo.RegionSizeX / 2;
|
||||
var y = _scene.RegionInfo.RegionSizeY / 2;
|
||||
var h = t.GetNormalizedGroundHeight(x, y);
|
||||
Log.Debug($"[WARMUP] terrain touch ok (h={h:0.00})");
|
||||
}
|
||||
catch (Exception e) { Log.Debug($"[WARMUP] terrain touch skipped: {e.Message}"); }
|
||||
}
|
||||
|
||||
if (_preloadAssets)
|
||||
{
|
||||
try
|
||||
{
|
||||
foreach (var so in _scene.GetSceneObjectGroups().Take(50))
|
||||
{
|
||||
var _ = so.Parts?.Length;
|
||||
foreach (var p in so.Parts)
|
||||
_ = p.Inventory?.GetInventoryItems()?.Count;
|
||||
}
|
||||
Log.Debug("[WARMUP] limited asset pre-touch ok");
|
||||
}
|
||||
catch (Exception e) { Log.Debug($"[WARMUP] pre-touch skipped: {e.Message}"); }
|
||||
}
|
||||
|
||||
if (_primeScriptVm)
|
||||
{
|
||||
try
|
||||
{
|
||||
var engine = _scene.RequestModuleInterface<IScriptModule>();
|
||||
if (engine != null)
|
||||
{
|
||||
int cnt = engine.GetScriptCount();
|
||||
Log.Debug($"[WARMUP] script VM primed (scripts={cnt})");
|
||||
}
|
||||
}
|
||||
catch (Exception e) { Log.Debug($"[WARMUP] script VM prime skipped: {e.Message}"); }
|
||||
}
|
||||
|
||||
Log.Info($"[WARMUP] Warmup done ({tag})");
|
||||
}
|
||||
|
||||
private void DeepWarmup(int limit)
|
||||
{
|
||||
try
|
||||
{
|
||||
int scanned = 0;
|
||||
foreach (var so in _scene.GetSceneObjectGroups())
|
||||
{
|
||||
var parts = so.Parts;
|
||||
if (parts == null) continue;
|
||||
foreach (var p in parts)
|
||||
{
|
||||
var _ = p.Inventory?.GetInventoryItems()?.Count;
|
||||
}
|
||||
scanned++;
|
||||
if (scanned >= limit) break;
|
||||
}
|
||||
Log.Info($"[WARMUP] deep warmup scanned objects={scanned}/{limit}");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log.Debug($"[WARMUP] deep warmup error: {e.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,393 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
using System.Reflection;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using log4net;
|
||||
using Mono.Addins;
|
||||
using Nini.Config;
|
||||
using OpenSim.Framework;
|
||||
using OpenSim.Framework.Console;
|
||||
using OpenSim.Region.Framework.Interfaces;
|
||||
using OpenSim.Region.Framework.Scenes;
|
||||
using System.IO;
|
||||
|
||||
[assembly: Addin("RegionWebhookAlerts", "1.0.0")]
|
||||
[assembly: AddinDependency("OpenSim.Region.Framework", OpenSim.VersionInfo.VersionNumber)]
|
||||
[assembly: AddinDescription("Generic JSON webhook alerts (batching + rate-limit) for RegionHealthMonitor.")]
|
||||
[assembly: AddinAuthor("Christopher Händler")]
|
||||
|
||||
namespace OpenSimRegionWarmupHealthGuard.Modules
|
||||
{
|
||||
public enum AlertSeverity { Trace = 0, Info = 1, Warn = 2, Error = 3 }
|
||||
|
||||
[Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "RegionWebhookAlerts")]
|
||||
public class RegionWebhookAlertsModule : INonSharedRegionModule
|
||||
{
|
||||
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
|
||||
|
||||
private Scene _scene;
|
||||
private bool _enabled;
|
||||
|
||||
// Config
|
||||
private string _url = ""; // webhook endpoint (e.g., n8n)
|
||||
private AlertSeverity _minSeverity = AlertSeverity.Warn;
|
||||
private int _batchWindowSec = 10; // collect events for N seconds before sending
|
||||
private int _rateLimitPerMin = 20; // max POSTs per minute
|
||||
private int _connectTimeoutMs = 5000;
|
||||
private int _sendTimeoutMs = 5000;
|
||||
private string _payloadFields = "region,agents,prims,scriptMs,physMs,netMs,scriptErrors,uptime,severity,message,ts"; // selectable
|
||||
|
||||
// State
|
||||
private IRegionHealthBus _bus;
|
||||
private readonly object _queueLock = new object();
|
||||
private readonly List<HealthEvent> _queue = new List<HealthEvent>();
|
||||
private Timer _batchTimer;
|
||||
private DateTime _windowStartUtc;
|
||||
private int _sentThisMinute;
|
||||
private DateTime _minuteWindowUtc;
|
||||
|
||||
public string Name => "RegionWebhookAlerts";
|
||||
public Type ReplaceableInterface => null;
|
||||
|
||||
public void Initialise(IConfigSource source)
|
||||
{
|
||||
var modules = source.Configs["Modules"];
|
||||
if (modules == null) return;
|
||||
|
||||
var status = modules.GetString("RegionWebhookAlerts", string.Empty);
|
||||
if (!string.Equals(status, "enabled", StringComparison.OrdinalIgnoreCase)) return;
|
||||
|
||||
var cfg = source.Configs["RegionWebhookAlerts"] ?? source.AddConfig("RegionWebhookAlerts");
|
||||
|
||||
_url = cfg.GetString("Url", _url);
|
||||
_minSeverity = ParseSeverity(cfg.GetString("MinSeverity", _minSeverity.ToString())) ?? _minSeverity;
|
||||
_batchWindowSec = Math.Max(1, cfg.GetInt("BatchWindowSec", _batchWindowSec));
|
||||
_rateLimitPerMin = Math.Max(1, cfg.GetInt("RateLimitPerMin", _rateLimitPerMin));
|
||||
_connectTimeoutMs = Math.Max(1000, cfg.GetInt("ConnectTimeoutMs", _connectTimeoutMs));
|
||||
_sendTimeoutMs = Math.Max(1000, cfg.GetInt("SendTimeoutMs", _sendTimeoutMs));
|
||||
_payloadFields = cfg.GetString("PayloadFields", _payloadFields);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(_url))
|
||||
{
|
||||
Log.Warn("[ALERTS] Url not configured. Module will remain idle.");
|
||||
}
|
||||
|
||||
_enabled = true;
|
||||
Log.Info($"[ALERTS] RegionWebhookAlerts enabled (min={_minSeverity}, batch={_batchWindowSec}s, rate={_rateLimitPerMin}/min)");
|
||||
}
|
||||
|
||||
public void AddRegion(Scene scene)
|
||||
{
|
||||
if (!_enabled) return;
|
||||
_scene = scene;
|
||||
|
||||
_bus = _scene.RequestModuleInterface<IRegionHealthBus>();
|
||||
if (_bus == null)
|
||||
{
|
||||
Log.Warn("[ALERTS] No IRegionHealthBus found. Alerts will remain idle until RegionHealthMonitor is enabled.");
|
||||
}
|
||||
else
|
||||
{
|
||||
_bus.OnIncident += OnIncident;
|
||||
}
|
||||
|
||||
_scene.RegisterModuleInterface(this);
|
||||
}
|
||||
|
||||
public void RegionLoaded(Scene scene)
|
||||
{
|
||||
if (!_enabled) return;
|
||||
|
||||
MainConsole.Instance.Commands.AddCommand("Region", false, "alerts",
|
||||
"alerts",
|
||||
"Webhook alerts commands. Usage: alerts status|test \"Message...\"",
|
||||
HandleConsole);
|
||||
|
||||
_windowStartUtc = DateTime.UtcNow;
|
||||
_minuteWindowUtc = _windowStartUtc;
|
||||
_batchTimer = new Timer(_ => FlushIfWindowElapsed(), null, _batchWindowSec * 1000, _batchWindowSec * 1000);
|
||||
|
||||
Log.Info($"[ALERTS] Ready for region '{_scene.RegionInfo.RegionName}'.");
|
||||
}
|
||||
|
||||
public void RemoveRegion(Scene scene)
|
||||
{
|
||||
if (!_enabled) return;
|
||||
|
||||
try { _batchTimer?.Dispose(); } catch { }
|
||||
_batchTimer = null;
|
||||
|
||||
if (_bus != null)
|
||||
_bus.OnIncident -= OnIncident;
|
||||
|
||||
_scene.UnregisterModuleInterface(this);
|
||||
_scene = null;
|
||||
}
|
||||
|
||||
public void Close()
|
||||
{
|
||||
try { _batchTimer?.Dispose(); } catch { }
|
||||
_batchTimer = null;
|
||||
}
|
||||
|
||||
private void HandleConsole(string module, string[] args)
|
||||
{
|
||||
if (_scene == null) { MainConsole.Instance.Output("[ALERTS] No active scene."); return; }
|
||||
if (args.Length < 2)
|
||||
{
|
||||
MainConsole.Instance.Output("alerts status - show current status");
|
||||
MainConsole.Instance.Output("alerts test \"Message\" - enqueue a test incident with severity=Warn");
|
||||
return;
|
||||
}
|
||||
|
||||
switch (args[1].ToLowerInvariant())
|
||||
{
|
||||
case "status":
|
||||
MainConsole.Instance.Output($"[ALERTS] Url={(string.IsNullOrWhiteSpace(_url) ? "(not set)" : _url)} " +
|
||||
$"MinSeverity={_minSeverity} Batch={_batchWindowSec}s Rate={_rateLimitPerMin}/min");
|
||||
break;
|
||||
|
||||
case "test":
|
||||
{
|
||||
string msg = args.Length >= 3 ? string.Join(" ", args, 2, args.Length - 2) : "Test alert";
|
||||
var ev = new HealthEvent
|
||||
{
|
||||
TimestampUtc = DateTime.UtcNow,
|
||||
Region = _scene.RegionInfo.RegionName,
|
||||
Severity = HealthSeverity.Warn,
|
||||
Message = msg,
|
||||
Sample = null
|
||||
};
|
||||
OnIncident(ev);
|
||||
MainConsole.Instance.Output("[ALERTS] Test event queued.");
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
MainConsole.Instance.Output("alerts status|test \"Message\"");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private void OnIncident(HealthEvent e)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(_url))
|
||||
return;
|
||||
|
||||
var sev = MapSeverity(e.Severity);
|
||||
if (sev < _minSeverity)
|
||||
return;
|
||||
|
||||
lock (_queueLock)
|
||||
{
|
||||
_queue.Add(e);
|
||||
}
|
||||
}
|
||||
|
||||
private void FlushIfWindowElapsed()
|
||||
{
|
||||
try
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
|
||||
// rate-limit window
|
||||
if ((now - _minuteWindowUtc).TotalSeconds >= 60)
|
||||
{
|
||||
_minuteWindowUtc = now;
|
||||
_sentThisMinute = 0;
|
||||
}
|
||||
|
||||
// batch window
|
||||
if ((now - _windowStartUtc).TotalSeconds < _batchWindowSec)
|
||||
return;
|
||||
|
||||
List<HealthEvent> batch;
|
||||
lock (_queueLock)
|
||||
{
|
||||
if (_queue.Count == 0)
|
||||
{
|
||||
_windowStartUtc = now;
|
||||
return;
|
||||
}
|
||||
batch = new List<HealthEvent>(_queue);
|
||||
_queue.Clear();
|
||||
_windowStartUtc = now;
|
||||
}
|
||||
|
||||
if (_sentThisMinute >= _rateLimitPerMin)
|
||||
{
|
||||
Log.Warn("[ALERTS] Rate limit reached; dropping batch.");
|
||||
return;
|
||||
}
|
||||
|
||||
// Build JSON and POST
|
||||
var payload = BuildJson(batch);
|
||||
if (PostJson(_url, payload))
|
||||
{
|
||||
_sentThisMinute++;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Warn($"[ALERTS] Flush failed: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private string BuildJson(List<HealthEvent> events)
|
||||
{
|
||||
// Minimal custom JSON builder (avoid external deps)
|
||||
// PayloadFields controls which top-level fields/metrics are included.
|
||||
var include = new HashSet<string>(SplitCsv(_payloadFields), StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var sb = new StringBuilder(events.Count * 256);
|
||||
sb.Append('[');
|
||||
for (int i = 0; i < events.Count; i++)
|
||||
{
|
||||
var e = events[i];
|
||||
sb.Append('{');
|
||||
|
||||
bool first = true;
|
||||
void add(string key, string value, bool raw = false)
|
||||
{
|
||||
if (!first) sb.Append(',');
|
||||
first = false;
|
||||
sb.Append('"').Append(E(key)).Append("\":");
|
||||
if (raw) sb.Append(value);
|
||||
else sb.Append('"').Append(E(value)).Append('"');
|
||||
}
|
||||
|
||||
if (include.Contains("ts")) add("ts", e.TimestampUtc.ToString("o"));
|
||||
if (include.Contains("region")) add("region", e.Region ?? _scene?.RegionInfo?.RegionName ?? "unknown");
|
||||
if (include.Contains("severity")) add("severity", e.Severity.ToString());
|
||||
if (include.Contains("message")) add("message", e.Message ?? "");
|
||||
|
||||
if (include.Contains("metrics"))
|
||||
{
|
||||
// combined object "metrics"
|
||||
if (!first) sb.Append(',');
|
||||
first = false;
|
||||
sb.Append("\"metrics\":");
|
||||
sb.Append('{');
|
||||
bool fm = true;
|
||||
void addm(string k, string v, bool raw = false)
|
||||
{
|
||||
if (!fm) sb.Append(',');
|
||||
fm = false;
|
||||
sb.Append('"').Append(E(k)).Append("\":");
|
||||
if (raw) sb.Append(v);
|
||||
else sb.Append('"').Append(E(v)).Append('"');
|
||||
}
|
||||
|
||||
var s = e.Sample;
|
||||
if (s != null)
|
||||
{
|
||||
if (include.Contains("agents")) addm("agents", s.Agents.ToString(), true);
|
||||
if (include.Contains("prims")) addm("prims", s.Prims.ToString(), true);
|
||||
if (include.Contains("scriptMs")) addm("scriptMs", s.ScriptTimeMs.ToString("0.###", System.Globalization.CultureInfo.InvariantCulture), true);
|
||||
if (include.Contains("physMs")) addm("physMs", s.PhysicsTimeMs.ToString("0.###", System.Globalization.CultureInfo.InvariantCulture), true);
|
||||
if (include.Contains("netMs")) addm("netMs", s.NetTimeMs.ToString("0.###", System.Globalization.CultureInfo.InvariantCulture), true);
|
||||
if (include.Contains("scriptErrors")) addm("scriptErrors", s.ScriptErrors.ToString(), true);
|
||||
if (include.Contains("uptime")) addm("uptimeSec", ((long)s.Uptime.TotalSeconds).ToString(), true);
|
||||
}
|
||||
sb.Append('}');
|
||||
}
|
||||
else
|
||||
{
|
||||
// flat metrics if requested
|
||||
var s = e.Sample;
|
||||
if (s != null)
|
||||
{
|
||||
if (include.Contains("agents")) add("agents", s.Agents.ToString(), true);
|
||||
if (include.Contains("prims")) add("prims", s.Prims.ToString(), true);
|
||||
if (include.Contains("scriptMs")) add("scriptMs", s.ScriptTimeMs.ToString("0.###", System.Globalization.CultureInfo.InvariantCulture), true);
|
||||
if (include.Contains("physMs")) add("physMs", s.PhysicsTimeMs.ToString("0.###", System.Globalization.CultureInfo.InvariantCulture), true);
|
||||
if (include.Contains("netMs")) add("netMs", s.NetTimeMs.ToString("0.###", System.Globalization.CultureInfo.InvariantCulture), true);
|
||||
if (include.Contains("scriptErrors")) add("scriptErrors", s.ScriptErrors.ToString(), true);
|
||||
if (include.Contains("uptime")) add("uptimeSec", ((long)s.Uptime.TotalSeconds).ToString(), true);
|
||||
}
|
||||
}
|
||||
|
||||
sb.Append('}');
|
||||
if (i < events.Count - 1) sb.Append(',');
|
||||
}
|
||||
sb.Append(']');
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
private bool PostJson(string url, string json)
|
||||
{
|
||||
try
|
||||
{
|
||||
var req = (HttpWebRequest)WebRequest.Create(url);
|
||||
req.Method = "POST";
|
||||
req.ContentType = "application/json";
|
||||
req.Timeout = _connectTimeoutMs;
|
||||
req.ReadWriteTimeout = _sendTimeoutMs;
|
||||
|
||||
var bytes = Encoding.UTF8.GetBytes(json);
|
||||
req.ContentLength = bytes.Length;
|
||||
using (var s = req.GetRequestStream())
|
||||
s.Write(bytes, 0, bytes.Length);
|
||||
|
||||
using var resp = (HttpWebResponse)req.GetResponse();
|
||||
var code = (int)resp.StatusCode;
|
||||
if (code >= 200 && code < 300)
|
||||
return true;
|
||||
|
||||
Log.Warn($"[ALERTS] Webhook returned HTTP {code}");
|
||||
return false;
|
||||
}
|
||||
catch (WebException wex)
|
||||
{
|
||||
try
|
||||
{
|
||||
var resp = wex.Response as HttpWebResponse;
|
||||
if (resp != null)
|
||||
Log.Warn($"[ALERTS] Webhook failed: HTTP {(int)resp.StatusCode} {resp.StatusDescription}");
|
||||
else
|
||||
Log.Warn($"[ALERTS] Webhook failed: {wex.Message}");
|
||||
}
|
||||
catch { }
|
||||
return false;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Warn($"[ALERTS] Webhook exception: {ex.Message}");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static IEnumerable<string> SplitCsv(string s)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(s)) yield break;
|
||||
foreach (var part in s.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries))
|
||||
yield return part.Trim();
|
||||
}
|
||||
|
||||
private static string E(string s)
|
||||
{
|
||||
if (string.IsNullOrEmpty(s)) return "";
|
||||
return s.Replace("\\", "\\\\").Replace("\"", "\\\"");
|
||||
}
|
||||
|
||||
private static AlertSeverity? ParseSeverity(string v)
|
||||
{
|
||||
if (Enum.TryParse<AlertSeverity>(v, true, out var sev))
|
||||
return sev;
|
||||
return null;
|
||||
}
|
||||
|
||||
private static AlertSeverity MapSeverity(HealthSeverity sev)
|
||||
{
|
||||
return sev switch
|
||||
{
|
||||
HealthSeverity.Error => AlertSeverity.Error,
|
||||
HealthSeverity.Warn => AlertSeverity.Warn,
|
||||
HealthSeverity.Info => AlertSeverity.Info,
|
||||
_ => AlertSeverity.Trace
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
# OpenSimRegionWarmupHealthGuard
|
||||
|
||||
> Warning - this module collection is under heavy development.
|
||||
@@ -0,0 +1,20 @@
|
||||
# RegionAutoHeal
|
||||
|
||||
## Configuration
|
||||
~~~ini
|
||||
[Modules]
|
||||
RegionAutoHeal = enabled
|
||||
|
||||
[RegionAutoHeal]
|
||||
; Sicherer Start: nur simulieren (Logs/Alerts), keine Änderungen
|
||||
DryRun = true
|
||||
; Echte Script-Resets nur bei expliziter Freigabe
|
||||
EnableScriptReset = false
|
||||
; Best-Effort-Drosselung „lauter“ Updater
|
||||
ThrottleHeavyUpdaters = true
|
||||
ThrottleThresholdUpdatesPerSec = 30
|
||||
; Ab wann reagiert werden soll (Skriptfehler-Burst)
|
||||
ScriptErrorBurstThreshold = 25
|
||||
; Sperrzeit zwischen Eingriffen (Sekunden)
|
||||
CooldownSec = 60
|
||||
~~~
|
||||
@@ -0,0 +1,22 @@
|
||||
# RegionHealthMonitor
|
||||
|
||||
|
||||
## Configuration
|
||||
~~~ini
|
||||
[Modules]
|
||||
RegionHealthMonitor = enabled
|
||||
|
||||
[RegionHealthMonitor]
|
||||
HealthIntervalSec = 30
|
||||
WarnScriptTimeMs = 12.0
|
||||
WarnPhysicsTimeMs = 6.0
|
||||
WarnNetTimeMs = 6.0
|
||||
WarnScriptErrors = 10
|
||||
; Optional CSV export (leave empty to disable)
|
||||
MetricsExportFile =
|
||||
~~~
|
||||
|
||||
## Console commands
|
||||
|
||||
health status
|
||||
health export
|
||||
@@ -0,0 +1,24 @@
|
||||
# RegionMetricsExporter
|
||||
|
||||
## Configuration
|
||||
~~~ini
|
||||
[Modules]
|
||||
RegionMetricsExporter = enabled
|
||||
|
||||
[RegionMetricsExporter]
|
||||
; HTTP pull endpoint for Prometheus (low overhead)
|
||||
HttpPort = 9109
|
||||
; Bind only to loopback by default (recommended). Set 0.0.0.0 to expose network-wide.
|
||||
BindAddress = 127.0.0.1
|
||||
; Metric name prefix
|
||||
MetricsPrefix = opensim_region_
|
||||
; Include region name as a label
|
||||
IncludeRegionLabel = true
|
||||
~~~
|
||||
|
||||
## Notes
|
||||
Hinweise:
|
||||
Dieses Modul liest HealthSamples vom RegionHealthMonitor (IRegionHealthBus). Stelle sicher, dass RegionHealthMonitor aktiv ist.
|
||||
In Multi-Region-Prozessen sollte jede Region einen eigenen Port nutzen (oder du baust eine gemeinsame, prozessweite Export-Instanz).
|
||||
Endpoint: http://BindAddress:HttpPort/metrics
|
||||
Pull-only, minimaler Ressourcenverbrauch.
|
||||
@@ -0,0 +1,32 @@
|
||||
# RegionPolicyEngine
|
||||
|
||||
## Configuration
|
||||
~~~ini
|
||||
[Modules]
|
||||
RegionPolicyEngine = enabled
|
||||
|
||||
[RegionPolicyEngine]
|
||||
Enabled = true
|
||||
; Prüfintervall (Sekunden)
|
||||
CheckIntervalSec = 60
|
||||
|
||||
; Liste von Profilen (kommasepariert)
|
||||
Profiles = Nightly,Event
|
||||
|
||||
; Einfaches Stundenfenster (Beispiele):
|
||||
; 20-6 -> zwischen 20:00 und 06:59
|
||||
; 8-18 -> zwischen 08:00 und 18:59
|
||||
; 22 -> genau um 22 Uhr
|
||||
Nightly.Cron = * * 20-6
|
||||
; Key=Value;Key=Value (Overlays, die die Module verstehen müssen)
|
||||
Nightly.Overrides = HealthIntervalSec=60; DeepWarmupLimit=400; MinSeverity=Error
|
||||
|
||||
Event.Cron =
|
||||
Event.Overrides = ThrottleHeavyUpdaters=false
|
||||
~~~
|
||||
|
||||
## Notes
|
||||
|
||||
Die Overlays sind bewusst generisch (Key=Value). Ein Modul, das Overlays unterstützen soll, implementiert optional IPolicyOverlayConsumer und mappt relevante Keys auf interne Settings.
|
||||
Die mitgelieferten Module funktionieren auch ohne Overlay-Unterstützung. Du kannst Overlay-Support schrittweise in einzelnen Modulen ergänzen (z. B. HealthMonitor: HealthIntervalSec, WebhookAlerts: MinSeverity, Warmup: DeepWarmupLimit, AutoHeal: ThrottleHeavyUpdaters).
|
||||
Das Cron-Feld ist hier stark vereinfacht (Stundenfenster). Für echtes Cron-Verhalten könnte später ein Parser ergänzt werden.
|
||||
@@ -0,0 +1,19 @@
|
||||
# RegionWarmup
|
||||
|
||||
## Configuration
|
||||
|
||||
~~~ini
|
||||
[Modules]
|
||||
RegionWarmup = enabled
|
||||
|
||||
[RegionWarmup]
|
||||
; Sofortiges Warmup bei Regionstart
|
||||
WarmupOnRegionLoaded = true
|
||||
; Einzelne Warmup-Schritte
|
||||
TouchTerrain = true
|
||||
PreloadAssets = true
|
||||
PrimeScriptVM = true
|
||||
; Optionaler tiefer Warmup-Scan (Anzahl Objekte) nach Verzögerung
|
||||
DeepWarmupLimit = 200
|
||||
DeepWarmupDelaySec = 30
|
||||
~~~
|
||||
@@ -0,0 +1,35 @@
|
||||
# RegionWebhookAlerts
|
||||
|
||||
## Configuration
|
||||
~~~ini
|
||||
[Modules]
|
||||
RegionWebhookAlerts = enabled
|
||||
|
||||
[RegionWebhookAlerts]
|
||||
; Generic JSON webhook endpoint (e.g., n8n webhook URL)
|
||||
Url = https://your-n8n-host/webhook/abc123
|
||||
|
||||
; Minimal severity to send: Trace|Info|Warn|Error (default Warn)
|
||||
MinSeverity = Warn
|
||||
|
||||
; Collect incidents for N seconds and send in one POST
|
||||
BatchWindowSec = 10
|
||||
|
||||
; Max POSTs per minute (rate limit)
|
||||
RateLimitPerMin = 20
|
||||
|
||||
; Timeouts (ms)
|
||||
ConnectTimeoutMs = 5000
|
||||
SendTimeoutMs = 5000
|
||||
|
||||
; Which fields to include in payload.
|
||||
; Available keys:
|
||||
; ts,region,severity,message,metrics,agents,prims,scriptMs,physMs,netMs,scriptErrors,uptime
|
||||
PayloadFields = region,agents,prims,scriptMs,physMs,netMs,scriptErrors,uptime,severity,message,ts
|
||||
~~~
|
||||
|
||||
## Notes
|
||||
|
||||
Unterstützt generisches JSON. Payload ist ein Array von Events.
|
||||
Batching + Rate-Limit verhindern Spam. Für sofortige Zustellung BatchWindowSec verkleinern.
|
||||
Abonniert Health-Events vom RegionHealthMonitor (IRegionHealthBus). Stelle sicher, dass dieser aktiviert ist.
|
||||
@@ -0,0 +1,68 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project frameworkVersion="v8_0" name="OpenSimRegionWarmupHealthGuard" path="addon-modules/OpenSimRegionWarmupHealthGuard/Modules" type="Library" language="C#">
|
||||
|
||||
<!--
|
||||
prebuild.xml for OpenSim addon-modules/OpenSimRegionWarmupHealthGuard
|
||||
Builds a single assembly containing all region modules:
|
||||
- RegionHealthMonitor
|
||||
- RegionMetricsExporter
|
||||
- RegionWebhookAlerts
|
||||
- RegionAutoHeal
|
||||
- RegionWarmup
|
||||
- RegionPolicyEngine
|
||||
-->
|
||||
|
||||
<Configuration name="Debug" platform="AnyCPU">
|
||||
<Options>
|
||||
<OutputPath>../../../bin/</OutputPath>
|
||||
<Optimize>false</Optimize>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
</Options>
|
||||
</Configuration>
|
||||
|
||||
<Configuration name="Release" platform="AnyCPU">
|
||||
<Options>
|
||||
<OutputPath>../../../bin/</OutputPath>
|
||||
<Optimize>true</Optimize>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
<DebugSymbols>false</DebugSymbols>
|
||||
</Options>
|
||||
</Configuration>
|
||||
|
||||
<ReferencePath>../../../bin/</ReferencePath>
|
||||
<Reference name="System" />
|
||||
<Reference name="System.Core" />
|
||||
<Reference name="System.Xml" />
|
||||
<Reference name="System.Xml.Linq" />
|
||||
<Reference name="System.Net.Http" />
|
||||
<Reference name="Mono.Addins" />
|
||||
<Reference name="log4net" />
|
||||
<Reference name="Nini"/>
|
||||
|
||||
|
||||
<!-- OpenSim core references (relative to repo root) -->
|
||||
<Reference name="OpenSim.Framework" />
|
||||
<Reference name="OpenSim.Region.Framework" />
|
||||
<Reference name="OpenMetaverseTypes" />
|
||||
<Reference name="OpenMetaverse" />
|
||||
|
||||
<Files>
|
||||
<!-- Source files -->
|
||||
<Match pattern="addon-modules/OpenSimRegionWarmupHealthGuard/Modules/*.cs" />
|
||||
<!--
|
||||
<Match pattern="addon-modules/OpenSimRegionWarmupHealthGuard/Modules/RegionMetricsExporter.cs" />
|
||||
<Match pattern="addon-modules/OpenSimRegionWarmupHealthGuard/Modules/RegionWebhookAlerts.cs" />
|
||||
<Match pattern="addon-modules/OpenSimRegionWarmupHealthGuard/Modules/RegionAutoHeal.cs" />
|
||||
<Match pattern="addon-modules/OpenSimRegionWarmupHealthGuard/Modules/RegionWarmupModule.cs" />
|
||||
<Match pattern="addon-modules/OpenSimRegionWarmupHealthGuard/Modules/RegionPolicyEngineModule.cs" />
|
||||
-->
|
||||
|
||||
<!-- Optional docs/examples (not compiled) -->
|
||||
<Exclude pattern="addon-modules/OpenSimRegionWarmupHealthGuard/*.md" />
|
||||
<Exclude pattern="addon-modules/OpenSimRegionWarmupHealthGuard/*.MD" />
|
||||
<Exclude pattern="addon-modules/OpenSimRegionWarmupHealthGuard/*.ini.example" />
|
||||
</Files>
|
||||
</Project>
|
||||
Reference in New Issue
Block a user