diff --git a/opensim/Data/ITSAssetAdminData.cs b/opensim/Data/ITSAssetAdminData.cs deleted file mode 100644 index 3133f90..0000000 --- a/opensim/Data/ITSAssetAdminData.cs +++ /dev/null @@ -1,62 +0,0 @@ -using System; - -namespace OpenSim.Data -{ - public struct TSAssetMoveReport - { - public string Source; - public string Destination; - public int CandidateCount; - public int AlreadyInTargetCount; - public int InsertedCount; - public int DeletedFromSourceCount; - public int IndexAffectedCount; - } - - public struct TSAssetMoveOptions - { - public bool ResetCheckpoint; - public int BatchSize; - public int CommandTimeoutSeconds; - } - - public struct TSAssetFindReport - { - public string AssetId; - public bool Found; - public string TableName; - public bool HasIndexEntry; - public int IndexAssetType; - } - - public struct TSAssetVerifyReport - { - public string Scope; - public int TablesChecked; - public int TotalRows; - public int MissingIndexRows; - public int WrongIndexTypeRows; - public int OrphanIndexRows; - public int LegacyRowsWithIndex; - } - - public struct TSAssetReindexReport - { - public string Scope; - public int TablesProcessed; - public int RowsScanned; - public int IndexRowsUpserted; - public int IndexRowsDeleted; - } - - public interface ITSAssetAdminData - { - bool TryMoveAssets(string from, string to, out TSAssetMoveReport report, out string errorMessage); - bool TryMoveAssets(string from, string to, TSAssetMoveOptions options, out TSAssetMoveReport report, out string errorMessage); - bool TryPreviewMoveAssets(string from, string to, out TSAssetMoveReport report, out string errorMessage); - bool TryFindAssetLocation(string assetId, out TSAssetFindReport report, out string errorMessage); - bool TryVerifyAssets(string scope, out TSAssetVerifyReport report, out string errorMessage); - bool TryReindexAssets(string scope, TSAssetMoveOptions options, out TSAssetReindexReport report, out string errorMessage); - bool TryCleanLegacyIndex(TSAssetMoveOptions options, out TSAssetReindexReport report, out string errorMessage); - } -} diff --git a/opensim/Data/MySQL/MySQLtsAssetData.cs b/opensim/Data/MySQL/MySQLtsAssetData.cs deleted file mode 100644 index 5d64c9b..0000000 --- a/opensim/Data/MySQL/MySQLtsAssetData.cs +++ /dev/null @@ -1,1687 +0,0 @@ -/* - * Copyright (c) Contributors, http://opensimulator.org/ - * See CONTRIBUTORS.TXT for a full list of copyright holders. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the OpenSimulator Project nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -using System; -using System.Collections.Generic; -using System.Data; -using System.Globalization; -using System.Reflection; -using System.Text.RegularExpressions; -using log4net; -using MySql.Data.MySqlClient; -using OpenMetaverse; -using OpenSim.Framework; -using OpenSim.Data; - -namespace OpenSim.Data.MySQL -{ - /// - /// MySQL storage provider that stores assets in type-specific tables: - /// assets_{assetType}. A legacy fallback to table "assets" can be enabled explicitly. - /// - public class MySQLtsAssetData : AssetDataBase, ITSAssetAdminData - { - private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - - private const string LegacyTableName = "assets"; - private const string IndexTableName = "tsassets_index"; - private const string MoveCheckpointTableName = "tsassets_move_checkpoint"; - private const string TypedTableNameFormat = "assets_{0}"; - private const bool DefaultFallbackToLegacy = false; - private const int DefaultTsAdminBatchSize = 1000; - private const int DefaultTsAdminCommandTimeoutSeconds = 300; - - private readonly object m_tableSync = new object(); - private readonly HashSet m_initializedTables = new HashSet(StringComparer.OrdinalIgnoreCase); - - private string m_connectionString; - private bool m_fallbackToLegacy; - private int m_tsAdminBatchSize = DefaultTsAdminBatchSize; - private int m_tsAdminCommandTimeoutSeconds = DefaultTsAdminCommandTimeoutSeconds; - - protected virtual Assembly Assembly - { - get { return GetType().Assembly; } - } - - #region IPlugin Members - - public override string Version - { - get { return "1.0.0.0"; } - } - - public override string Name - { - get { return "MySQL TSAsset storage engine"; } - } - - public override void Initialise(string connect) - { - if (string.IsNullOrEmpty(connect)) - throw new ArgumentException("Connection string must not be null or empty", nameof(connect)); - - m_connectionString = connect; - m_fallbackToLegacy = TryReadBooleanSetting(connect, "TSFallbackToLegacyAssets", DefaultFallbackToLegacy); - m_tsAdminBatchSize = Math.Max(1, TryReadIntSetting(connect, "TSAdminBatchSize", DefaultTsAdminBatchSize)); - m_tsAdminCommandTimeoutSeconds = Math.Max(30, TryReadIntSetting(connect, "TSAdminCommandTimeoutSeconds", DefaultTsAdminCommandTimeoutSeconds)); - - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - - Migration migration = new Migration(dbcon, Assembly, "TSAssetStore"); - migration.Update(); - - EnsureIndexTable(dbcon); - EnsureMoveCheckpointTable(dbcon); - } - } - - public override void Initialise() - { - throw new NotImplementedException(); - } - - public override void Dispose() - { - } - - #endregion - - #region IAssetDataPlugin Members - - public override AssetBase GetAsset(UUID assetID) - { - try - { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - - if (TryGetIndexedAssetType(dbcon, assetID, out sbyte indexedType)) - { - string typedTable = GetTypeTableName(indexedType); - - AssetBase typedAsset = GetAssetFromTable(dbcon, typedTable, assetID); - if (typedAsset != null) - return typedAsset; - - RemoveIndexRow(dbcon, assetID); - } - - if (m_fallbackToLegacy) - { - AssetBase legacyAsset = GetAssetFromTable(dbcon, LegacyTableName, assetID); - if (legacyAsset != null) - { - TryStoreTypedWithExistingConnection(dbcon, legacyAsset); - return legacyAsset; - } - } - } - } - catch (Exception e) - { - m_log.ErrorFormat("[TSASSET DB]: MySQL failure fetching asset {0}. Error: {1}", assetID, e.Message); - } - - return null; - } - - public override bool StoreAsset(AssetBase asset) - { - if (asset == null) - return false; - - try - { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - return TryStoreTypedWithExistingConnection(dbcon, asset); - } - } - catch (Exception e) - { - m_log.ErrorFormat("[TSASSET DB]: MySQL failure storing asset {0}. Error: {1}", asset.FullID, e.Message); - return false; - } - } - - public override bool[] AssetsExist(UUID[] uuids) - { - if (uuids == null || uuids.Length == 0) - return new bool[0]; - - bool[] results = new bool[uuids.Length]; - Dictionary> positions = new Dictionary>(); - - for (int i = 0; i < uuids.Length; i++) - { - if (!positions.TryGetValue(uuids[i], out List indexes)) - { - indexes = new List(); - positions[uuids[i]] = indexes; - } - - indexes.Add(i); - } - - try - { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - - HashSet found = QueryExistingIds(dbcon, IndexTableName, "id", uuids); - - if (m_fallbackToLegacy) - { - UUID[] missing = BuildMissingArray(uuids, found); - if (missing.Length > 0) - { - HashSet legacyFound = QueryExistingIds(dbcon, LegacyTableName, "id", missing); - foreach (UUID id in legacyFound) - found.Add(id); - } - } - - foreach (UUID id in found) - { - if (positions.TryGetValue(id, out List idxList)) - { - foreach (int idx in idxList) - results[idx] = true; - } - } - } - } - catch (Exception e) - { - m_log.ErrorFormat("[TSASSET DB]: MySQL failure checking asset existence. Error: {0}", e.Message); - } - - return results; - } - - public override List FetchAssetMetadataSet(int start, int count) - { - List result = new List(Math.Max(0, count)); - List> pending = new List>(Math.Max(0, count)); - - if (count <= 0) - return result; - - try - { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - - using (MySqlCommand cmd = new MySqlCommand( - $"SELECT id, assetType FROM `{IndexTableName}` ORDER BY updated_at DESC LIMIT ?start, ?count", - dbcon)) - { - cmd.Parameters.AddWithValue("?start", start); - cmd.Parameters.AddWithValue("?count", count); - - using (MySqlDataReader dbReader = cmd.ExecuteReader()) - { - while (dbReader.Read()) - { - UUID id = DBGuid.FromDB(dbReader["id"]); - int typeInt = Convert.ToInt32(dbReader["assetType"], CultureInfo.InvariantCulture); - sbyte type = Convert.ToSByte(typeInt, CultureInfo.InvariantCulture); - pending.Add(new KeyValuePair(id, type)); - } - } - - for (int i = 0; i < pending.Count; i++) - { - AssetMetadata metadata = GetMetadataByIdAndType(dbcon, pending[i].Key, pending[i].Value); - if (metadata != null) - result.Add(metadata); - } - } - } - } - catch (Exception e) - { - m_log.ErrorFormat("[TSASSET DB]: MySQL failure fetching metadata set from {0}, count {1}. Error: {2}", start, count, e.Message); - } - - return result; - } - - public override bool Delete(string id) - { - if (!UUID.TryParse(id, out UUID assetId)) - return false; - - try - { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - - using (MySqlTransaction tx = dbcon.BeginTransaction()) - { - try - { - if (TryGetIndexedAssetType(dbcon, tx, assetId, out sbyte indexedType)) - { - string typedTable = GetTypeTableName(indexedType); - - using (MySqlCommand deleteTyped = new MySqlCommand($"DELETE FROM `{typedTable}` WHERE id=?id", dbcon, tx)) - { - deleteTyped.Parameters.AddWithValue("?id", assetId.ToString()); - deleteTyped.ExecuteNonQuery(); - } - } - - using (MySqlCommand deleteIndex = new MySqlCommand($"DELETE FROM `{IndexTableName}` WHERE id=?id", dbcon, tx)) - { - deleteIndex.Parameters.AddWithValue("?id", assetId.ToString()); - deleteIndex.ExecuteNonQuery(); - } - - if (m_fallbackToLegacy) - { - using (MySqlCommand deleteLegacy = new MySqlCommand($"DELETE FROM `{LegacyTableName}` WHERE id=?id", dbcon, tx)) - { - deleteLegacy.Parameters.AddWithValue("?id", assetId.ToString()); - deleteLegacy.ExecuteNonQuery(); - } - } - - tx.Commit(); - return true; - } - catch - { - tx.Rollback(); - throw; - } - } - } - } - catch (Exception e) - { - m_log.ErrorFormat("[TSASSET DB]: MySQL failure deleting asset {0}. Error: {1}", id, e.Message); - return false; - } - } - - public bool TryMoveAssets(string from, string to, out TSAssetMoveReport report, out string errorMessage) - { - TSAssetMoveOptions options = new TSAssetMoveOptions - { - ResetCheckpoint = false, - BatchSize = 0, - CommandTimeoutSeconds = 0 - }; - - return TryMoveAssets(from, to, options, out report, out errorMessage); - } - - public bool TryMoveAssets(string from, string to, TSAssetMoveOptions options, out TSAssetMoveReport report, out string errorMessage) - { - report = new TSAssetMoveReport - { - Source = from ?? string.Empty, - Destination = to ?? string.Empty, - CandidateCount = 0, - AlreadyInTargetCount = 0, - InsertedCount = 0, - DeletedFromSourceCount = 0, - IndexAffectedCount = 0 - }; - - errorMessage = string.Empty; - - if (!TryParseMoveTable(from, out MoveTableSpec sourceSpec, out string parseSourceError)) - { - errorMessage = parseSourceError; - return false; - } - - if (!TryParseMoveTable(to, out MoveTableSpec targetSpec, out string parseTargetError)) - { - errorMessage = parseTargetError; - return false; - } - - if (sourceSpec.TableName.Equals(targetSpec.TableName, StringComparison.OrdinalIgnoreCase)) - { - errorMessage = string.Format(CultureInfo.InvariantCulture, "Source and target resolve to the same table '{0}'", sourceSpec.TableName); - return false; - } - - try - { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - - EnsureIndexTable(dbcon); - - if (!sourceSpec.IsLegacy && !TableExists(dbcon, sourceSpec.TableName)) - return true; - - if (targetSpec.IsLegacy) - { - if (!TableExists(dbcon, targetSpec.TableName)) - { - errorMessage = string.Format(CultureInfo.InvariantCulture, "Target table '{0}' does not exist", targetSpec.TableName); - return false; - } - } - else - { - EnsureTypeTable(targetSpec.TableName); - } - - string whereClause = BuildMoveWhereClause(sourceSpec, targetSpec); - int sourceAssetTypeFilter = targetSpec.IsLegacy ? 0 : targetSpec.AssetType; - int effectiveBatchSize = options.BatchSize > 0 ? options.BatchSize : m_tsAdminBatchSize; - int effectiveCommandTimeoutSeconds = options.CommandTimeoutSeconds > 0 ? options.CommandTimeoutSeconds : m_tsAdminCommandTimeoutSeconds; - string operationKey = BuildMoveOperationKey(sourceSpec, targetSpec); - string cursorAfter = string.Empty; - - if (options.ResetCheckpoint) - DeleteMoveCheckpoint(dbcon, operationKey, effectiveCommandTimeoutSeconds); - - cursorAfter = GetMoveCheckpoint(dbcon, operationKey, effectiveCommandTimeoutSeconds); - - while (true) - { - using (MySqlTransaction tx = dbcon.BeginTransaction()) - { - try - { - List batchIds = GetMoveBatchIds( - dbcon, - tx, - sourceSpec, - targetSpec, - sourceAssetTypeFilter, - whereClause, - cursorAfter, - effectiveBatchSize, - effectiveCommandTimeoutSeconds); - if (batchIds.Count == 0) - { - tx.Commit(); - break; - } - - report.CandidateCount += batchIds.Count; - - int alreadyInTarget = CountRowsByIds(dbcon, tx, targetSpec.TableName, batchIds, effectiveCommandTimeoutSeconds); - report.AlreadyInTargetCount += alreadyInTarget; - - InsertBatchIntoTarget(dbcon, tx, sourceSpec, targetSpec, batchIds, effectiveCommandTimeoutSeconds); - - int nowInTarget = CountRowsByIds(dbcon, tx, targetSpec.TableName, batchIds, effectiveCommandTimeoutSeconds); - report.InsertedCount += Math.Max(0, nowInTarget - alreadyInTarget); - - if (targetSpec.IsLegacy) - report.IndexAffectedCount += DeleteRowsByIds(dbcon, tx, IndexTableName, batchIds, effectiveCommandTimeoutSeconds); - else - report.IndexAffectedCount += UpsertIndexRowsByIds(dbcon, tx, targetSpec.AssetType, batchIds, effectiveCommandTimeoutSeconds); - - report.DeletedFromSourceCount += DeleteRowsByIds(dbcon, tx, sourceSpec.TableName, batchIds, effectiveCommandTimeoutSeconds); - - cursorAfter = batchIds[batchIds.Count - 1]; - UpsertMoveCheckpoint(dbcon, tx, operationKey, cursorAfter, effectiveCommandTimeoutSeconds); - - tx.Commit(); - } - catch - { - TryRollback(tx); - throw; - } - } - } - - DeleteMoveCheckpoint(dbcon, operationKey, effectiveCommandTimeoutSeconds); - - return true; - } - } - catch (Exception e) - { - errorMessage = BuildExceptionMessage(e); - m_log.ErrorFormat("[TSASSET DB]: MySQL failure moving assets from {0} to {1}. Error: {2}", from, to, e.Message); - return false; - } - } - - public bool TryPreviewMoveAssets(string from, string to, out TSAssetMoveReport report, out string errorMessage) - { - report = new TSAssetMoveReport - { - Source = from ?? string.Empty, - Destination = to ?? string.Empty, - CandidateCount = 0, - AlreadyInTargetCount = 0, - InsertedCount = 0, - DeletedFromSourceCount = 0, - IndexAffectedCount = 0 - }; - - errorMessage = string.Empty; - - if (!TryParseMoveTable(from, out MoveTableSpec sourceSpec, out string parseSourceError)) - { - errorMessage = parseSourceError; - return false; - } - - if (!TryParseMoveTable(to, out MoveTableSpec targetSpec, out string parseTargetError)) - { - errorMessage = parseTargetError; - return false; - } - - if (sourceSpec.TableName.Equals(targetSpec.TableName, StringComparison.OrdinalIgnoreCase)) - { - errorMessage = string.Format(CultureInfo.InvariantCulture, "Source and target resolve to the same table '{0}'", sourceSpec.TableName); - return false; - } - - try - { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - - if (!sourceSpec.IsLegacy && !TableExists(dbcon, sourceSpec.TableName)) - return true; - - if (!targetSpec.IsLegacy && !TableExists(dbcon, targetSpec.TableName)) - EnsureTypeTable(targetSpec.TableName); - - if (targetSpec.IsLegacy && !TableExists(dbcon, targetSpec.TableName)) - { - errorMessage = string.Format(CultureInfo.InvariantCulture, "Target table '{0}' does not exist", targetSpec.TableName); - return false; - } - - string whereClause = BuildMoveWhereClause(sourceSpec, targetSpec); - int sourceAssetTypeFilter = targetSpec.IsLegacy ? 0 : targetSpec.AssetType; - - using (MySqlTransaction tx = dbcon.BeginTransaction(IsolationLevel.ReadCommitted)) - { - try - { - report.CandidateCount = ExecuteCount( - dbcon, - tx, - $"SELECT COUNT(*) FROM `{sourceSpec.TableName}` s {whereClause}", - sourceAssetTypeFilter, - sourceSpec, - targetSpec); - - report.AlreadyInTargetCount = ExecuteCount( - dbcon, - tx, - $"SELECT COUNT(*) FROM `{sourceSpec.TableName}` s INNER JOIN `{targetSpec.TableName}` t ON t.id = s.id {whereClause}", - sourceAssetTypeFilter, - sourceSpec, - targetSpec); - - report.InsertedCount = Math.Max(0, report.CandidateCount - report.AlreadyInTargetCount); - report.DeletedFromSourceCount = report.CandidateCount; - - if (targetSpec.IsLegacy) - report.IndexAffectedCount = report.CandidateCount; - else - report.IndexAffectedCount = report.CandidateCount; - - tx.Commit(); - return true; - } - catch - { - TryRollback(tx); - throw; - } - } - } - } - catch (Exception e) - { - errorMessage = BuildExceptionMessage(e); - m_log.ErrorFormat("[TSASSET DB]: MySQL failure previewing move from {0} to {1}. Error: {2}", from, to, e.Message); - return false; - } - } - - public bool TryFindAssetLocation(string assetId, out TSAssetFindReport report, out string errorMessage) - { - report = new TSAssetFindReport - { - AssetId = assetId ?? string.Empty, - Found = false, - TableName = string.Empty, - HasIndexEntry = false, - IndexAssetType = 0 - }; - - errorMessage = string.Empty; - - if (!UUID.TryParse(assetId, out UUID parsedId)) - { - errorMessage = "Invalid asset UUID"; - return false; - } - - try - { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - - if (TryGetIndexedAssetType(dbcon, parsedId, out sbyte indexType)) - { - report.HasIndexEntry = true; - report.IndexAssetType = indexType; - - string typedTable = GetTypeTableName(indexType); - if (TableExists(dbcon, typedTable)) - { - if (ExecuteCountById(dbcon, typedTable, parsedId) > 0) - { - report.Found = true; - report.TableName = typedTable; - return true; - } - } - } - - if (TableExists(dbcon, LegacyTableName) && ExecuteCountById(dbcon, LegacyTableName, parsedId) > 0) - { - report.Found = true; - report.TableName = LegacyTableName; - return true; - } - - List types = GetExistingTypedTableTypes(dbcon); - for (int i = 0; i < types.Count; i++) - { - string tableName = GetTypeTableName(types[i]); - if (report.HasIndexEntry && tableName.Equals(GetTypeTableName((sbyte)report.IndexAssetType), StringComparison.OrdinalIgnoreCase)) - continue; - - if (ExecuteCountById(dbcon, tableName, parsedId) > 0) - { - report.Found = true; - report.TableName = tableName; - return true; - } - } - - return true; - } - } - catch (Exception e) - { - errorMessage = e.Message; - m_log.ErrorFormat("[TSASSET DB]: MySQL failure finding asset {0}. Error: {1}", assetId, e.Message); - return false; - } - } - - public bool TryVerifyAssets(string scope, out TSAssetVerifyReport report, out string errorMessage) - { - report = new TSAssetVerifyReport - { - Scope = string.IsNullOrWhiteSpace(scope) ? "all" : scope.Trim(), - TablesChecked = 0, - TotalRows = 0, - MissingIndexRows = 0, - WrongIndexTypeRows = 0, - OrphanIndexRows = 0, - LegacyRowsWithIndex = 0 - }; - - errorMessage = string.Empty; - - try - { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - EnsureIndexTable(dbcon); - - string normalizedScope = string.IsNullOrWhiteSpace(scope) ? "all" : scope.Trim(); - List verifyTables = new List(); - - if (normalizedScope.Equals("all", StringComparison.OrdinalIgnoreCase)) - { - if (TableExists(dbcon, LegacyTableName)) - { - verifyTables.Add(new MoveTableSpec - { - TableName = LegacyTableName, - IsLegacy = true, - AssetType = 0 - }); - } - - List types = GetExistingTypedTableTypes(dbcon); - for (int i = 0; i < types.Count; i++) - { - verifyTables.Add(new MoveTableSpec - { - TableName = GetTypeTableName(types[i]), - IsLegacy = false, - AssetType = types[i] - }); - } - } - else - { - if (!TryParseMoveTable(normalizedScope, out MoveTableSpec requestedTable, out string parseError)) - { - errorMessage = parseError; - return false; - } - - if (!TableExists(dbcon, requestedTable.TableName)) - { - return true; - } - - verifyTables.Add(requestedTable); - } - - for (int i = 0; i < verifyTables.Count; i++) - { - MoveTableSpec table = verifyTables[i]; - report.TablesChecked++; - report.TotalRows += ExecuteCountRaw(dbcon, string.Format(CultureInfo.InvariantCulture, "SELECT COUNT(*) FROM `{0}`", table.TableName)); - - if (table.IsLegacy) - { - report.LegacyRowsWithIndex += ExecuteCountRaw( - dbcon, - $"SELECT COUNT(*) FROM `{LegacyTableName}` l INNER JOIN `{IndexTableName}` i ON i.id = l.id"); - continue; - } - - report.MissingIndexRows += ExecuteCountRaw( - dbcon, - $"SELECT COUNT(*) FROM `{table.TableName}` t LEFT JOIN `{IndexTableName}` i ON i.id = t.id WHERE i.id IS NULL"); - - report.WrongIndexTypeRows += ExecuteCountRaw( - dbcon, - $"SELECT COUNT(*) FROM `{table.TableName}` t INNER JOIN `{IndexTableName}` i ON i.id = t.id WHERE i.assetType <> {table.AssetType.ToString(CultureInfo.InvariantCulture)}"); - - report.OrphanIndexRows += ExecuteCountRaw( - dbcon, - $"SELECT COUNT(*) FROM `{IndexTableName}` i LEFT JOIN `{table.TableName}` t ON t.id = i.id WHERE i.assetType = {table.AssetType.ToString(CultureInfo.InvariantCulture)} AND t.id IS NULL"); - } - - return true; - } - } - catch (Exception e) - { - errorMessage = e.Message; - m_log.ErrorFormat("[TSASSET DB]: MySQL failure verifying scope {0}. Error: {1}", scope, e.Message); - return false; - } - } - - public bool TryReindexAssets(string scope, TSAssetMoveOptions options, out TSAssetReindexReport report, out string errorMessage) - { - report = new TSAssetReindexReport - { - Scope = string.IsNullOrWhiteSpace(scope) ? "all" : scope.Trim(), - TablesProcessed = 0, - RowsScanned = 0, - IndexRowsUpserted = 0, - IndexRowsDeleted = 0 - }; - - errorMessage = string.Empty; - - try - { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - EnsureIndexTable(dbcon); - EnsureMoveCheckpointTable(dbcon); - - int effectiveBatchSize = options.BatchSize > 0 ? options.BatchSize : m_tsAdminBatchSize; - int effectiveCommandTimeoutSeconds = options.CommandTimeoutSeconds > 0 ? options.CommandTimeoutSeconds : m_tsAdminCommandTimeoutSeconds; - - if (!TryResolveScopeTables(dbcon, scope, out List tables, out errorMessage)) - return false; - - for (int tableIndex = 0; tableIndex < tables.Count; tableIndex++) - { - MoveTableSpec table = tables[tableIndex]; - report.TablesProcessed++; - - string opKey = string.Format(CultureInfo.InvariantCulture, "reindex:{0}", table.TableName); - if (options.ResetCheckpoint) - DeleteMoveCheckpoint(dbcon, opKey, effectiveCommandTimeoutSeconds); - - string cursorAfter = GetMoveCheckpoint(dbcon, opKey, effectiveCommandTimeoutSeconds); - - while (true) - { - using (MySqlTransaction tx = dbcon.BeginTransaction()) - { - try - { - List ids = GetBatchIdsByTable(dbcon, tx, table.TableName, cursorAfter, effectiveBatchSize, effectiveCommandTimeoutSeconds); - if (ids.Count == 0) - { - tx.Commit(); - break; - } - - report.RowsScanned += ids.Count; - - if (table.IsLegacy) - report.IndexRowsDeleted += DeleteRowsByIds(dbcon, tx, IndexTableName, ids, effectiveCommandTimeoutSeconds); - else - report.IndexRowsUpserted += UpsertIndexRowsByIds(dbcon, tx, table.AssetType, ids, effectiveCommandTimeoutSeconds); - - cursorAfter = ids[ids.Count - 1]; - UpsertMoveCheckpoint(dbcon, tx, opKey, cursorAfter, effectiveCommandTimeoutSeconds); - - tx.Commit(); - } - catch - { - TryRollback(tx); - throw; - } - } - } - - DeleteMoveCheckpoint(dbcon, opKey, effectiveCommandTimeoutSeconds); - } - - return true; - } - } - catch (Exception e) - { - errorMessage = BuildExceptionMessage(e); - m_log.ErrorFormat("[TSASSET DB]: MySQL failure reindexing scope {0}. Error: {1}", scope, e.Message); - return false; - } - } - - public bool TryCleanLegacyIndex(TSAssetMoveOptions options, out TSAssetReindexReport report, out string errorMessage) - { - return TryReindexAssets("assets", options, out report, out errorMessage); - } - - #endregion - - private sealed class MoveTableSpec - { - public string TableName; - public bool IsLegacy; - public sbyte AssetType; - } - - private static bool TryParseMoveTable(string token, out MoveTableSpec spec, out string error) - { - spec = null; - error = string.Empty; - - if (string.IsNullOrWhiteSpace(token)) - { - error = "Table token is empty"; - return false; - } - - string normalized = token.Trim(); - - if (normalized.Equals(LegacyTableName, StringComparison.OrdinalIgnoreCase)) - { - spec = new MoveTableSpec - { - TableName = LegacyTableName, - IsLegacy = true, - AssetType = 0 - }; - return true; - } - - if (sbyte.TryParse(normalized, NumberStyles.Integer, CultureInfo.InvariantCulture, out sbyte directType)) - { - spec = new MoveTableSpec - { - TableName = GetTypeTableName(directType), - IsLegacy = false, - AssetType = directType - }; - return true; - } - - Match typedName = Regex.Match(normalized, "^assets_(-?\\d+)$", RegexOptions.IgnoreCase); - if (typedName.Success) - { - if (sbyte.TryParse(typedName.Groups[1].Value, NumberStyles.Integer, CultureInfo.InvariantCulture, out sbyte parsedType)) - { - spec = new MoveTableSpec - { - TableName = GetTypeTableName(parsedType), - IsLegacy = false, - AssetType = parsedType - }; - return true; - } - } - - error = string.Format(CultureInfo.InvariantCulture, "Invalid table/type token '{0}'. Use 'assets', '' or 'assets_'", token); - return false; - } - - private bool TryResolveScopeTables(MySqlConnection dbcon, string scope, out List tables, out string error) - { - tables = new List(); - error = string.Empty; - - string normalizedScope = string.IsNullOrWhiteSpace(scope) ? "all" : scope.Trim(); - - if (normalizedScope.Equals("all", StringComparison.OrdinalIgnoreCase)) - { - if (TableExists(dbcon, LegacyTableName)) - { - tables.Add(new MoveTableSpec - { - TableName = LegacyTableName, - IsLegacy = true, - AssetType = 0 - }); - } - - List types = GetExistingTypedTableTypes(dbcon); - for (int i = 0; i < types.Count; i++) - { - tables.Add(new MoveTableSpec - { - TableName = GetTypeTableName(types[i]), - IsLegacy = false, - AssetType = types[i] - }); - } - - return true; - } - - if (!TryParseMoveTable(normalizedScope, out MoveTableSpec requestedTable, out string parseError)) - { - error = parseError; - return false; - } - - if (!TableExists(dbcon, requestedTable.TableName)) - return true; - - tables.Add(requestedTable); - return true; - } - - private static string BuildMoveWhereClause(MoveTableSpec source, MoveTableSpec target) - { - if (source.IsLegacy && !target.IsLegacy) - return "WHERE s.assetType = ?sourceTypeFilter"; - - return string.Empty; - } - - private static void BindMoveParameters(MySqlCommand cmd, int sourceAssetTypeFilter, MoveTableSpec source, MoveTableSpec target) - { - if (source.IsLegacy && !target.IsLegacy) - cmd.Parameters.AddWithValue("?sourceTypeFilter", sourceAssetTypeFilter); - } - - private static bool TableExists(MySqlConnection dbcon, string tableName) - { - using (MySqlCommand cmd = new MySqlCommand( - "SELECT 1 FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = ?tableName LIMIT 1", - dbcon)) - { - cmd.Parameters.AddWithValue("?tableName", tableName); - object value = cmd.ExecuteScalar(); - return value != null && value != DBNull.Value; - } - } - - private static int ExecuteCountById(MySqlConnection dbcon, string tableName, UUID id) - { - using (MySqlCommand cmd = new MySqlCommand($"SELECT COUNT(*) FROM `{tableName}` WHERE id=?id", dbcon)) - { - cmd.Parameters.AddWithValue("?id", id.ToString()); - object scalar = cmd.ExecuteScalar(); - if (scalar == null || scalar is DBNull) - return 0; - - return Convert.ToInt32(scalar, CultureInfo.InvariantCulture); - } - } - - private static int ExecuteCountRaw(MySqlConnection dbcon, string sql) - { - using (MySqlCommand cmd = new MySqlCommand(sql, dbcon)) - { - object scalar = cmd.ExecuteScalar(); - if (scalar == null || scalar is DBNull) - return 0; - - return Convert.ToInt32(scalar, CultureInfo.InvariantCulture); - } - } - - private static List GetExistingTypedTableTypes(MySqlConnection dbcon) - { - List result = new List(); - - using (MySqlCommand cmd = new MySqlCommand( - "SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name REGEXP '^assets_-?[0-9]+$'", - dbcon)) - using (MySqlDataReader reader = cmd.ExecuteReader()) - { - while (reader.Read()) - { - string tableName = reader["table_name"].ToString(); - Match typedMatch = Regex.Match(tableName, "^assets_(-?\\d+)$", RegexOptions.IgnoreCase); - if (!typedMatch.Success) - continue; - - if (sbyte.TryParse(typedMatch.Groups[1].Value, NumberStyles.Integer, CultureInfo.InvariantCulture, out sbyte type)) - result.Add(type); - } - } - - result.Sort(); - return result; - } - - private static int ExecuteCount(MySqlConnection dbcon, MySqlTransaction tx, string sql, int sourceAssetTypeFilter, MoveTableSpec source, MoveTableSpec target) - { - using (MySqlCommand cmd = new MySqlCommand(sql, dbcon, tx)) - { - cmd.CommandTimeout = DefaultTsAdminCommandTimeoutSeconds; - BindMoveParameters(cmd, sourceAssetTypeFilter, source, target); - object scalar = cmd.ExecuteScalar(); - if (scalar == null || scalar is DBNull) - return 0; - - return Convert.ToInt32(scalar, CultureInfo.InvariantCulture); - } - } - - private static List GetMoveBatchIds( - MySqlConnection dbcon, - MySqlTransaction tx, - MoveTableSpec sourceSpec, - MoveTableSpec targetSpec, - int sourceAssetTypeFilter, - string whereClause, - string cursorAfter, - int batchSize, - int commandTimeoutSeconds) - { - List ids = new List(batchSize); - string scopedWhere = BuildScopedWhereClause(whereClause, !string.IsNullOrEmpty(cursorAfter)); - - using (MySqlCommand cmd = new MySqlCommand( - $"SELECT s.id FROM `{sourceSpec.TableName}` s {scopedWhere} ORDER BY s.id LIMIT ?limit", - dbcon, - tx)) - { - cmd.CommandTimeout = commandTimeoutSeconds; - BindMoveParameters(cmd, sourceAssetTypeFilter, sourceSpec, targetSpec); - - if (!string.IsNullOrEmpty(cursorAfter)) - cmd.Parameters.AddWithValue("?cursorAfter", cursorAfter); - - cmd.Parameters.AddWithValue("?limit", batchSize); - - using (MySqlDataReader reader = cmd.ExecuteReader()) - { - while (reader.Read()) - ids.Add(reader["id"].ToString()); - } - } - - return ids; - } - - private static List GetBatchIdsByTable( - MySqlConnection dbcon, - MySqlTransaction tx, - string tableName, - string cursorAfter, - int batchSize, - int commandTimeoutSeconds) - { - List ids = new List(batchSize); - string whereClause = string.IsNullOrEmpty(cursorAfter) ? string.Empty : "WHERE id > ?cursorAfter"; - - using (MySqlCommand cmd = new MySqlCommand( - $"SELECT id FROM `{tableName}` {whereClause} ORDER BY id LIMIT ?limit", - dbcon, - tx)) - { - cmd.CommandTimeout = commandTimeoutSeconds; - if (!string.IsNullOrEmpty(cursorAfter)) - cmd.Parameters.AddWithValue("?cursorAfter", cursorAfter); - - cmd.Parameters.AddWithValue("?limit", batchSize); - - using (MySqlDataReader reader = cmd.ExecuteReader()) - { - while (reader.Read()) - ids.Add(reader["id"].ToString()); - } - } - - return ids; - } - - private static int CountRowsByIds(MySqlConnection dbcon, MySqlTransaction tx, string tableName, List ids, int commandTimeoutSeconds) - { - if (ids == null || ids.Count == 0) - return 0; - - using (MySqlCommand cmd = new MySqlCommand( - $"SELECT COUNT(*) FROM `{tableName}` WHERE id IN ({BuildIdInClause(ids.Count)})", - dbcon, - tx)) - { - cmd.CommandTimeout = commandTimeoutSeconds; - AddIdParameters(cmd, ids); - object scalar = cmd.ExecuteScalar(); - if (scalar == null || scalar is DBNull) - return 0; - - return Convert.ToInt32(scalar, CultureInfo.InvariantCulture); - } - } - - private static void InsertBatchIntoTarget(MySqlConnection dbcon, MySqlTransaction tx, MoveTableSpec sourceSpec, MoveTableSpec targetSpec, List ids, int commandTimeoutSeconds) - { - if (ids == null || ids.Count == 0) - return; - - using (MySqlCommand cmd = new MySqlCommand( - $"INSERT IGNORE INTO `{targetSpec.TableName}` " + - "(id, name, description, assetType, local, temporary, create_time, access_time, asset_flags, CreatorID, data) " + - "SELECT s.id, s.name, s.description, " + - (targetSpec.IsLegacy ? "s.assetType" : "?targetAssetType") + - ", s.local, s.temporary, s.create_time, s.access_time, s.asset_flags, s.CreatorID, s.data " + - $"FROM `{sourceSpec.TableName}` s WHERE s.id IN ({BuildIdInClause(ids.Count)})", - dbcon, - tx)) - { - cmd.CommandTimeout = commandTimeoutSeconds; - AddIdParameters(cmd, ids); - - if (!targetSpec.IsLegacy) - cmd.Parameters.AddWithValue("?targetAssetType", targetSpec.AssetType); - - cmd.ExecuteNonQuery(); - } - } - - private static int DeleteRowsByIds(MySqlConnection dbcon, MySqlTransaction tx, string tableName, List ids, int commandTimeoutSeconds) - { - if (ids == null || ids.Count == 0) - return 0; - - using (MySqlCommand cmd = new MySqlCommand( - $"DELETE FROM `{tableName}` WHERE id IN ({BuildIdInClause(ids.Count)})", - dbcon, - tx)) - { - cmd.CommandTimeout = commandTimeoutSeconds; - AddIdParameters(cmd, ids); - return cmd.ExecuteNonQuery(); - } - } - - private static int UpsertIndexRowsByIds(MySqlConnection dbcon, MySqlTransaction tx, sbyte targetType, List ids, int commandTimeoutSeconds) - { - if (ids == null || ids.Count == 0) - return 0; - - using (MySqlCommand cmd = new MySqlCommand( - $"INSERT INTO `{IndexTableName}` (id, assetType, updated_at) " + - $"SELECT id, ?assetType, ?updatedAt FROM `{GetTypeTableName(targetType)}` WHERE id IN ({BuildIdInClause(ids.Count)}) " + - "ON DUPLICATE KEY UPDATE assetType = VALUES(assetType), updated_at = VALUES(updated_at)", - dbcon, - tx)) - { - cmd.CommandTimeout = commandTimeoutSeconds; - cmd.Parameters.AddWithValue("?assetType", targetType); - cmd.Parameters.AddWithValue("?updatedAt", (int)Utils.DateTimeToUnixTime(DateTime.UtcNow)); - AddIdParameters(cmd, ids); - return cmd.ExecuteNonQuery(); - } - } - - private static string BuildScopedWhereClause(string whereClause, bool includeCursor) - { - if (!includeCursor) - return whereClause; - - if (string.IsNullOrWhiteSpace(whereClause)) - return "WHERE s.id > ?cursorAfter"; - - return string.Concat(whereClause, " AND s.id > ?cursorAfter"); - } - - private static string BuildMoveOperationKey(MoveTableSpec sourceSpec, MoveTableSpec targetSpec) - { - return string.Format(CultureInfo.InvariantCulture, "{0}->{1}", sourceSpec.TableName, targetSpec.TableName); - } - - private static void EnsureMoveCheckpointTable(MySqlConnection dbcon) - { - string sql = - $"CREATE TABLE IF NOT EXISTS `{MoveCheckpointTableName}` (" + - "`op_key` varchar(190) NOT NULL," + - "`last_id` char(36) NOT NULL DEFAULT ''," + - "`updated_at` int(11) NOT NULL DEFAULT '0'," + - "PRIMARY KEY (`op_key`)" + - ") ENGINE=InnoDB DEFAULT CHARSET=utf8"; - - using (MySqlCommand cmd = new MySqlCommand(sql, dbcon)) - cmd.ExecuteNonQuery(); - } - - private static string GetMoveCheckpoint(MySqlConnection dbcon, string operationKey, int commandTimeoutSeconds) - { - using (MySqlCommand cmd = new MySqlCommand( - $"SELECT last_id FROM `{MoveCheckpointTableName}` WHERE op_key=?opKey", - dbcon)) - { - cmd.CommandTimeout = commandTimeoutSeconds; - cmd.Parameters.AddWithValue("?opKey", operationKey); - object scalar = cmd.ExecuteScalar(); - if (scalar == null || scalar is DBNull) - return string.Empty; - - return scalar.ToString(); - } - } - - private static void UpsertMoveCheckpoint(MySqlConnection dbcon, MySqlTransaction tx, string operationKey, string lastId, int commandTimeoutSeconds) - { - using (MySqlCommand cmd = new MySqlCommand( - $"INSERT INTO `{MoveCheckpointTableName}` (op_key, last_id, updated_at) VALUES (?opKey, ?lastId, ?updatedAt) " + - "ON DUPLICATE KEY UPDATE last_id = VALUES(last_id), updated_at = VALUES(updated_at)", - dbcon, - tx)) - { - cmd.CommandTimeout = commandTimeoutSeconds; - cmd.Parameters.AddWithValue("?opKey", operationKey); - cmd.Parameters.AddWithValue("?lastId", lastId ?? string.Empty); - cmd.Parameters.AddWithValue("?updatedAt", (int)Utils.DateTimeToUnixTime(DateTime.UtcNow)); - cmd.ExecuteNonQuery(); - } - } - - private static void DeleteMoveCheckpoint(MySqlConnection dbcon, string operationKey, int commandTimeoutSeconds) - { - using (MySqlCommand cmd = new MySqlCommand( - $"DELETE FROM `{MoveCheckpointTableName}` WHERE op_key=?opKey", - dbcon)) - { - cmd.CommandTimeout = commandTimeoutSeconds; - cmd.Parameters.AddWithValue("?opKey", operationKey); - cmd.ExecuteNonQuery(); - } - } - - private static string BuildIdInClause(int count) - { - string[] placeholders = new string[count]; - for (int i = 0; i < count; i++) - placeholders[i] = "?id" + i.ToString(CultureInfo.InvariantCulture); - - return string.Join(",", placeholders); - } - - private static void AddIdParameters(MySqlCommand cmd, List ids) - { - for (int i = 0; i < ids.Count; i++) - cmd.Parameters.AddWithValue("?id" + i.ToString(CultureInfo.InvariantCulture), ids[i]); - } - - private static void TryRollback(MySqlTransaction tx) - { - if (tx == null) - return; - - try - { - tx.Rollback(); - } - catch - { - } - } - - private static string BuildExceptionMessage(Exception e) - { - if (e == null) - return string.Empty; - - string message = e.Message; - Exception inner = e.InnerException; - - while (inner != null) - { - if (!string.IsNullOrEmpty(inner.Message)) - message = string.Concat(message, " | ", inner.Message); - - inner = inner.InnerException; - } - - return message; - } - - private static int TryReadIntSetting(string connectionString, string key, int defaultValue) - { - if (string.IsNullOrEmpty(connectionString) || string.IsNullOrEmpty(key)) - return defaultValue; - - string[] parts = connectionString.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries); - foreach (string rawPart in parts) - { - string part = rawPart.Trim(); - int eqIndex = part.IndexOf('='); - if (eqIndex <= 0 || eqIndex == part.Length - 1) - continue; - - string k = part.Substring(0, eqIndex).Trim(); - if (!k.Equals(key, StringComparison.OrdinalIgnoreCase)) - continue; - - string v = part.Substring(eqIndex + 1).Trim(); - if (int.TryParse(v, NumberStyles.Integer, CultureInfo.InvariantCulture, out int parsed)) - return parsed; - } - - return defaultValue; - } - - private static bool TryReadBooleanSetting(string connectionString, string key, bool defaultValue) - { - if (string.IsNullOrEmpty(connectionString) || string.IsNullOrEmpty(key)) - return defaultValue; - - string[] parts = connectionString.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries); - foreach (string rawPart in parts) - { - string part = rawPart.Trim(); - int eqIndex = part.IndexOf('='); - if (eqIndex <= 0 || eqIndex == part.Length - 1) - continue; - - string k = part.Substring(0, eqIndex).Trim(); - if (!k.Equals(key, StringComparison.OrdinalIgnoreCase)) - continue; - - string v = part.Substring(eqIndex + 1).Trim(); - if (v.Equals("1", StringComparison.OrdinalIgnoreCase) || v.Equals("true", StringComparison.OrdinalIgnoreCase) || v.Equals("yes", StringComparison.OrdinalIgnoreCase)) - return true; - - if (v.Equals("0", StringComparison.OrdinalIgnoreCase) || v.Equals("false", StringComparison.OrdinalIgnoreCase) || v.Equals("no", StringComparison.OrdinalIgnoreCase)) - return false; - } - - return defaultValue; - } - - private static string GetTypeTableName(sbyte assetType) - { - return string.Format(CultureInfo.InvariantCulture, TypedTableNameFormat, assetType); - } - - private void EnsureTypeTable(string tableName) - { - lock (m_tableSync) - { - if (m_initializedTables.Contains(tableName)) - return; - - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - CreateTypeTable(dbcon, tableName); - } - - m_initializedTables.Add(tableName); - } - } - - private void CreateTypeTable(MySqlConnection dbcon, string tableName) - { - string sql = - $"CREATE TABLE IF NOT EXISTS `{tableName}` (" + - "`name` varchar(64) NOT NULL," + - "`description` varchar(64) NOT NULL," + - "`assetType` tinyint(4) NOT NULL," + - "`local` tinyint(1) NOT NULL," + - "`temporary` tinyint(1) NOT NULL," + - "`data` longblob NOT NULL," + - "`id` char(36) NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000'," + - "`create_time` int(11) DEFAULT '0'," + - "`access_time` int(11) DEFAULT '0'," + - "`asset_flags` int(11) NOT NULL DEFAULT '0'," + - "`CreatorID` varchar(128) NOT NULL DEFAULT ''," + - "PRIMARY KEY (`id`)" + - ") ENGINE=InnoDB DEFAULT CHARSET=utf8"; - - using (MySqlCommand cmd = new MySqlCommand(sql, dbcon)) - { - cmd.ExecuteNonQuery(); - } - } - - private static void EnsureIndexTable(MySqlConnection dbcon) - { - string sql = - $"CREATE TABLE IF NOT EXISTS `{IndexTableName}` (" + - "`id` char(36) NOT NULL," + - "`assetType` tinyint(4) NOT NULL," + - "`updated_at` int(11) NOT NULL DEFAULT '0'," + - "PRIMARY KEY (`id`)," + - $"INDEX `idx_{IndexTableName}_assetType` (`assetType`)" + - ") ENGINE=InnoDB DEFAULT CHARSET=utf8"; - - using (MySqlCommand cmd = new MySqlCommand(sql, dbcon)) - { - cmd.ExecuteNonQuery(); - } - } - - private bool TryStoreTypedWithExistingConnection(MySqlConnection dbcon, AssetBase asset) - { - string tableName = GetTypeTableName(asset.Type); - EnsureTypeTable(tableName); - - string assetName = asset.Name ?? string.Empty; - if (assetName.Length > AssetBase.MAX_ASSET_NAME) - assetName = assetName.Substring(0, AssetBase.MAX_ASSET_NAME); - - string assetDescription = asset.Description ?? string.Empty; - if (assetDescription.Length > AssetBase.MAX_ASSET_DESC) - assetDescription = assetDescription.Substring(0, AssetBase.MAX_ASSET_DESC); - - int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow); - - using (MySqlTransaction tx = dbcon.BeginTransaction()) - { - try - { - string upsertAssetSql = - $"REPLACE INTO `{tableName}` " + - "(id, name, description, assetType, local, temporary, create_time, access_time, asset_flags, CreatorID, data) " + - "VALUES(?id, ?name, ?description, ?assetType, ?local, ?temporary, ?create_time, ?access_time, ?asset_flags, ?CreatorID, ?data)"; - - using (MySqlCommand assetCmd = new MySqlCommand(upsertAssetSql, dbcon, tx)) - { - assetCmd.Parameters.AddWithValue("?id", asset.ID); - assetCmd.Parameters.AddWithValue("?name", assetName); - assetCmd.Parameters.AddWithValue("?description", assetDescription); - assetCmd.Parameters.AddWithValue("?assetType", asset.Type); - assetCmd.Parameters.AddWithValue("?local", asset.Local); - assetCmd.Parameters.AddWithValue("?temporary", asset.Temporary); - assetCmd.Parameters.AddWithValue("?create_time", now); - assetCmd.Parameters.AddWithValue("?access_time", now); - assetCmd.Parameters.AddWithValue("?asset_flags", (int)asset.Flags); - assetCmd.Parameters.AddWithValue("?CreatorID", asset.Metadata.CreatorID ?? string.Empty); - assetCmd.Parameters.AddWithValue("?data", asset.Data ?? Array.Empty()); - assetCmd.ExecuteNonQuery(); - } - - using (MySqlCommand indexCmd = new MySqlCommand( - $"REPLACE INTO `{IndexTableName}` (id, assetType, updated_at) VALUES (?id, ?assetType, ?updated_at)", - dbcon, - tx)) - { - indexCmd.Parameters.AddWithValue("?id", asset.ID); - indexCmd.Parameters.AddWithValue("?assetType", asset.Type); - indexCmd.Parameters.AddWithValue("?updated_at", now); - indexCmd.ExecuteNonQuery(); - } - - tx.Commit(); - return true; - } - catch - { - tx.Rollback(); - throw; - } - } - } - - private AssetBase GetAssetFromTable(MySqlConnection dbcon, string tableName, UUID assetID) - { - string sql = - $"SELECT name, description, assetType, local, temporary, asset_flags, CreatorID, data " + - $"FROM `{tableName}` WHERE id=?id"; - - try - { - using (MySqlCommand cmd = new MySqlCommand(sql, dbcon)) - { - cmd.Parameters.AddWithValue("?id", assetID.ToString()); - - using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) - { - if (!dbReader.Read()) - return null; - - AssetBase asset = new AssetBase(assetID, (string)dbReader["name"], (sbyte)dbReader["assetType"], dbReader["CreatorID"].ToString()); - asset.Description = (string)dbReader["description"]; - asset.Local = Convert.ToBoolean(dbReader["local"]); - asset.Temporary = Convert.ToBoolean(dbReader["temporary"]); - asset.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"], CultureInfo.InvariantCulture); - asset.Data = (byte[])dbReader["data"]; - return asset; - } - } - } - catch (MySqlException) - { - return null; - } - } - - private AssetMetadata GetMetadataByIdAndType(MySqlConnection dbcon, UUID id, sbyte type) - { - string tableName = GetTypeTableName(type); - - string sql = - $"SELECT id, name, description, assetType, temporary, asset_flags, CreatorID " + - $"FROM `{tableName}` WHERE id=?id"; - - try - { - using (MySqlCommand cmd = new MySqlCommand(sql, dbcon)) - { - cmd.Parameters.AddWithValue("?id", id.ToString()); - using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) - { - if (dbReader.Read()) - return ReadMetadata(dbReader); - } - } - } - catch (MySqlException) - { - // Type table may not yet exist if no asset of this type was stored. - } - - if (!m_fallbackToLegacy) - return null; - - using (MySqlCommand legacyCmd = new MySqlCommand( - $"SELECT id, name, description, assetType, temporary, asset_flags, CreatorID FROM `{LegacyTableName}` WHERE id=?id", - dbcon)) - { - legacyCmd.Parameters.AddWithValue("?id", id.ToString()); - using (MySqlDataReader dbReader = legacyCmd.ExecuteReader(CommandBehavior.SingleRow)) - { - if (dbReader.Read()) - return ReadMetadata(dbReader); - } - } - - return null; - } - - private static AssetMetadata ReadMetadata(MySqlDataReader dbReader) - { - AssetMetadata metadata = new AssetMetadata(); - metadata.FullID = DBGuid.FromDB(dbReader["id"]); - metadata.Name = dbReader["name"].ToString(); - metadata.Description = dbReader["description"].ToString(); - metadata.Type = Convert.ToSByte(dbReader["assetType"], CultureInfo.InvariantCulture); - metadata.Temporary = Convert.ToBoolean(dbReader["temporary"]); - metadata.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"], CultureInfo.InvariantCulture); - metadata.CreatorID = dbReader["CreatorID"].ToString(); - metadata.SHA1 = Array.Empty(); - return metadata; - } - - private static UUID[] BuildMissingArray(UUID[] requested, HashSet found) - { - List missing = new List(requested.Length); - for (int i = 0; i < requested.Length; i++) - { - if (!found.Contains(requested[i])) - missing.Add(requested[i]); - } - - return missing.ToArray(); - } - - private static HashSet QueryExistingIds(MySqlConnection dbcon, string tableName, string idColumn, UUID[] uuids) - { - HashSet found = new HashSet(); - - if (uuids.Length == 0) - return found; - - const int batchSize = 200; - for (int offset = 0; offset < uuids.Length; offset += batchSize) - { - int take = Math.Min(batchSize, uuids.Length - offset); - string[] placeholders = new string[take]; - - using (MySqlCommand cmd = dbcon.CreateCommand()) - { - for (int i = 0; i < take; i++) - { - string paramName = "?id" + i.ToString(CultureInfo.InvariantCulture); - placeholders[i] = paramName; - cmd.Parameters.AddWithValue(paramName, uuids[offset + i].ToString()); - } - - cmd.CommandText = string.Format( - CultureInfo.InvariantCulture, - "SELECT {0} FROM `{1}` WHERE {0} IN ({2})", - idColumn, - tableName, - string.Join(",", placeholders)); - - using (MySqlDataReader dbReader = cmd.ExecuteReader()) - { - while (dbReader.Read()) - { - UUID id = DBGuid.FromDB(dbReader[idColumn]); - found.Add(id); - } - } - } - } - - return found; - } - - private bool TryGetIndexedAssetType(MySqlConnection dbcon, UUID assetID, out sbyte assetType) - { - return TryGetIndexedAssetType(dbcon, null, assetID, out assetType); - } - - private bool TryGetIndexedAssetType(MySqlConnection dbcon, MySqlTransaction tx, UUID assetID, out sbyte assetType) - { - using (MySqlCommand cmd = new MySqlCommand($"SELECT assetType FROM `{IndexTableName}` WHERE id=?id", dbcon, tx)) - { - cmd.Parameters.AddWithValue("?id", assetID.ToString()); - - object value = cmd.ExecuteScalar(); - if (value == null || value is DBNull) - { - assetType = 0; - return false; - } - - int typeInt = Convert.ToInt32(value, CultureInfo.InvariantCulture); - if (typeInt < sbyte.MinValue || typeInt > sbyte.MaxValue) - { - assetType = 0; - return false; - } - - assetType = (sbyte)typeInt; - return true; - } - } - - private static void RemoveIndexRow(MySqlConnection dbcon, UUID assetID) - { - using (MySqlCommand cmd = new MySqlCommand($"DELETE FROM `{IndexTableName}` WHERE id=?id", dbcon)) - { - cmd.Parameters.AddWithValue("?id", assetID.ToString()); - cmd.ExecuteNonQuery(); - } - } - } -} diff --git a/opensim/Data/MySQL/Resources/TSAssetStore.migrations b/opensim/Data/MySQL/Resources/TSAssetStore.migrations deleted file mode 100644 index 372b3cf..0000000 --- a/opensim/Data/MySQL/Resources/TSAssetStore.migrations +++ /dev/null @@ -1,18 +0,0 @@ -# ----------------- -:VERSION 1 - -BEGIN; - -CREATE TABLE IF NOT EXISTS `tsassets_index` ( - `id` char(36) NOT NULL, - `assetType` tinyint(4) NOT NULL, - `updated_at` int(11) NOT NULL DEFAULT '0', - PRIMARY KEY (`id`), - INDEX `idx_tsassets_index_assetType` (`assetType`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- Type tables are created dynamically at runtime --- using the same schema and name pattern: assets_ --- e.g. assets_1, assets_22, assets_-1, assets_-2. - -COMMIT; diff --git a/opensim/Data/Tests/AssetTests.cs b/opensim/Data/Tests/AssetTests.cs deleted file mode 100644 index 65c04be..0000000 --- a/opensim/Data/Tests/AssetTests.cs +++ /dev/null @@ -1,323 +0,0 @@ -/* - * Copyright (c) Contributors, http://opensimulator.org/ - * See CONTRIBUTORS.TXT for a full list of copyright holders. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the OpenSimulator Project nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -using System; -using System.Collections.Generic; -using log4net.Config; -using NUnit.Framework; -using NUnit.Framework.Constraints; -using OpenMetaverse; -using OpenSim.Framework; -using OpenSim.Tests.Common; -using System.Data.Common; -using log4net; - -// DBMS-specific: -using MySql.Data.MySqlClient; -using OpenSim.Data.MySQL; - -using Mono.Data.Sqlite; -using OpenSim.Data.SQLite; - -namespace OpenSim.Data.Tests -{ - [TestFixture(Description = "Asset store tests (SQLite)")] - public class SQLiteAssetTests : AssetTests - { - } - - [TestFixture(Description = "Asset store tests (MySQL)")] - public class MySqlAssetTests : AssetTests - { - } - - [TestFixture(Description = "Asset store tests (MySQL TSAsset)")] - public class MySqlTsAssetTests : AssetTests - { - protected override void ClearDB() - { - DropTables( - "`tsassets_index`", - "`assets_0`", - "`assets_-2`", - "`assets_49`"); - - ResetMigrations("TSAssetStore"); - } - - [Test] - public void T030_StoreReadKnownNegativeAndMeshTypes() - { - TestHelpers.InMethod(); - - UUID legacyId = UUID.Random(); - UUID meshId = UUID.Random(); - - AssetBase legacyMaterial = new AssetBase(legacyId, "legacy material", -2, UUID.Random().ToString()); - legacyMaterial.Data = new byte[] { 1, 2, 3 }; - - AssetBase mesh = new AssetBase(meshId, "mesh asset", 49, UUID.Random().ToString()); - mesh.Data = new byte[] { 4, 5, 6 }; - - Assert.IsTrue(m_db.StoreAsset(legacyMaterial), "StoreAsset should succeed for type -2"); - Assert.IsTrue(m_db.StoreAsset(mesh), "StoreAsset should succeed for type 49"); - - AssetBase legacyMaterialRead = m_db.GetAsset(legacyId); - AssetBase meshRead = m_db.GetAsset(meshId); - - Assert.That(legacyMaterialRead, Is.Not.Null); - Assert.That(meshRead, Is.Not.Null); - Assert.That(legacyMaterialRead.Type, Is.EqualTo((sbyte)-2)); - Assert.That(meshRead.Type, Is.EqualTo((sbyte)49)); - - bool[] exists = m_db.AssetsExist(new[] { legacyId, meshId }); - Assert.That(exists[0], Is.True); - Assert.That(exists[1], Is.True); - - int tableCount = 0; - ExecQuery( - "SELECT COUNT(*) AS cnt FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name IN ('assets_-2','assets_49')", - true, - reader => - { - tableCount = Convert.ToInt32(reader["cnt"]); - return false; - }); - - Assert.That(tableCount, Is.EqualTo(2), "Expected dynamic type tables assets_-2 and assets_49 to exist"); - } - - [Test] - public void T031_StoreReadBoundarySignedTinyIntAssetTypes() - { - TestHelpers.InMethod(); - - UUID minId = UUID.Random(); - UUID maxId = UUID.Random(); - - AssetBase minTypeAsset = new AssetBase(minId, "min type asset", -128, UUID.Random().ToString()); - minTypeAsset.Data = new byte[] { 10, 20, 30 }; - - AssetBase maxTypeAsset = new AssetBase(maxId, "max type asset", 127, UUID.Random().ToString()); - maxTypeAsset.Data = new byte[] { 40, 50, 60 }; - - Assert.IsTrue(m_db.StoreAsset(minTypeAsset), "StoreAsset should succeed for type -128"); - Assert.IsTrue(m_db.StoreAsset(maxTypeAsset), "StoreAsset should succeed for type 127"); - - AssetBase minTypeRead = m_db.GetAsset(minId); - AssetBase maxTypeRead = m_db.GetAsset(maxId); - - Assert.That(minTypeRead, Is.Not.Null); - Assert.That(maxTypeRead, Is.Not.Null); - Assert.That(minTypeRead.Type, Is.EqualTo((sbyte)-128)); - Assert.That(maxTypeRead.Type, Is.EqualTo((sbyte)127)); - - bool[] exists = m_db.AssetsExist(new[] { minId, maxId }); - Assert.That(exists[0], Is.True); - Assert.That(exists[1], Is.True); - - int tableCount = 0; - ExecQuery( - "SELECT COUNT(*) AS cnt FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name IN ('assets_-128','assets_127')", - true, - reader => - { - tableCount = Convert.ToInt32(reader["cnt"]); - return false; - }); - - Assert.That(tableCount, Is.EqualTo(2), "Expected dynamic type tables assets_-128 and assets_127 to exist"); - } - } - - public class AssetTests : BasicDataServiceTest - where TConn : DbConnection, new() - where TAssetData : AssetDataBase, new() - { - protected TAssetData m_db; - - public UUID uuid1 = UUID.Random(); - public UUID uuid2 = UUID.Random(); - public UUID uuid3 = UUID.Random(); - - public string critter1 = UUID.Random().ToString(); - public string critter2 = UUID.Random().ToString(); - public string critter3 = UUID.Random().ToString(); - - public byte[] data1 = new byte[100]; - - PropertyScrambler scrambler = new PropertyScrambler() - .DontScramble(x => x.ID) - .DontScramble(x => x.Type) - .DontScramble(x => x.FullID) - .DontScramble(x => x.Metadata.ID) - .DontScramble(x => x.Metadata.CreatorID) - .DontScramble(x => x.Metadata.ContentType) - .DontScramble(x => x.Metadata.FullID) - .DontScramble(x => x.Data); - - protected override void InitService(object service) - { - ClearDB(); - m_db = (TAssetData)service; - m_db.Initialise(m_connStr); - } - - protected virtual void ClearDB() - { - DropTables("assets"); - ResetMigrations("AssetStore"); - } - - - [Test] - public void T001_LoadEmpty() - { - TestHelpers.InMethod(); - - bool[] exist = m_db.AssetsExist(new[] { uuid1, uuid2, uuid3 }); - Assert.IsFalse(exist[0]); - Assert.IsFalse(exist[1]); - Assert.IsFalse(exist[2]); - } - - [Test] - public void T010_StoreReadVerifyAssets() - { - TestHelpers.InMethod(); - - AssetBase a1 = new AssetBase(uuid1, "asset one", (sbyte)AssetType.Texture, critter1.ToString()); - AssetBase a2 = new AssetBase(uuid2, "asset two", (sbyte)AssetType.Texture, critter2.ToString()); - AssetBase a3 = new AssetBase(uuid3, "asset three", (sbyte)AssetType.Texture, critter3.ToString()); - a1.Data = data1; - a2.Data = data1; - a3.Data = data1; - - scrambler.Scramble(a1); - scrambler.Scramble(a2); - scrambler.Scramble(a3); - - m_db.StoreAsset(a1); - m_db.StoreAsset(a2); - m_db.StoreAsset(a3); - a1.UploadAttempts = 0; - a2.UploadAttempts = 0; - a3.UploadAttempts = 0; - - AssetBase a1a = m_db.GetAsset(uuid1); - a1a.UploadAttempts = 0; - Assert.That(a1a, Constraints.PropertyCompareConstraint(a1)); - - AssetBase a2a = m_db.GetAsset(uuid2); - a2a.UploadAttempts = 0; - Assert.That(a2a, Constraints.PropertyCompareConstraint(a2)); - - AssetBase a3a = m_db.GetAsset(uuid3); - a3a.UploadAttempts = 0; - Assert.That(a3a, Constraints.PropertyCompareConstraint(a3)); - - scrambler.Scramble(a1a); - scrambler.Scramble(a2a); - scrambler.Scramble(a3a); - - m_db.StoreAsset(a1a); - m_db.StoreAsset(a2a); - m_db.StoreAsset(a3a); - a1a.UploadAttempts = 0; - a2a.UploadAttempts = 0; - a3a.UploadAttempts = 0; - - AssetBase a1b = m_db.GetAsset(uuid1); - a1b.UploadAttempts = 0; - Assert.That(a1b, Constraints.PropertyCompareConstraint(a1a)); - - AssetBase a2b = m_db.GetAsset(uuid2); - a2b.UploadAttempts = 0; - Assert.That(a2b, Constraints.PropertyCompareConstraint(a2a)); - - AssetBase a3b = m_db.GetAsset(uuid3); - a3b.UploadAttempts = 0; - Assert.That(a3b, Constraints.PropertyCompareConstraint(a3a)); - - bool[] exist = m_db.AssetsExist(new[] { uuid1, uuid2, uuid3 }); - Assert.IsTrue(exist[0]); - Assert.IsTrue(exist[1]); - Assert.IsTrue(exist[2]); - - List metadatas = m_db.FetchAssetMetadataSet(0, 1000); - - Assert.That(metadatas.Count >= 3, "FetchAssetMetadataSet() should have returned at least 3 assets!"); - - // It is possible that the Asset table is filled with data, in which case we don't try to find "our" - // assets there: - if (metadatas.Count < 1000) - { - AssetMetadata metadata = metadatas.Find(x => x.FullID == uuid1); - Assert.That(metadata.Name, Is.EqualTo(a1b.Name)); - Assert.That(metadata.Description, Is.EqualTo(a1b.Description)); - Assert.That(metadata.Type, Is.EqualTo(a1b.Type)); - Assert.That(metadata.Temporary, Is.EqualTo(a1b.Temporary)); - Assert.That(metadata.FullID, Is.EqualTo(a1b.FullID)); - } - } - - [Test] - public void T020_CheckForWeirdCreatorID() - { - TestHelpers.InMethod(); - - // It is expected that eventually the CreatorID might be an arbitrary string (an URI) - // rather than a valid UUID (?). This test is to make sure that the database layer does not - // attempt to convert CreatorID to GUID, but just passes it both ways as a string. - AssetBase a1 = new AssetBase(uuid1, "asset one", (sbyte)AssetType.Texture, critter1); - AssetBase a2 = new AssetBase(uuid2, "asset two", (sbyte)AssetType.Texture, "This is not a GUID!"); - AssetBase a3 = new AssetBase(uuid3, "asset three", (sbyte)AssetType.Texture, ""); - a1.Data = data1; - a2.Data = data1; - a3.Data = data1; - - m_db.StoreAsset(a1); - a1.UploadAttempts = 0; - m_db.StoreAsset(a2); - a2.UploadAttempts = 0; - m_db.StoreAsset(a3); - a3.UploadAttempts = 0; - - AssetBase a1a = m_db.GetAsset(uuid1); - a1a.UploadAttempts = 0; - Assert.That(a1a, Constraints.PropertyCompareConstraint(a1)); - - AssetBase a2a = m_db.GetAsset(uuid2); - a2a.UploadAttempts = 0; - Assert.That(a2a, Constraints.PropertyCompareConstraint(a2)); - - AssetBase a3a = m_db.GetAsset(uuid3); - a3a.UploadAttempts = 0; - Assert.That(a3a, Constraints.PropertyCompareConstraint(a3)); - } - } -} \ No newline at end of file diff --git a/opensim/OpenSim/Data/ITSAssetAdminData.cs b/opensim/OpenSim/Data/ITSAssetAdminData.cs deleted file mode 100644 index 3133f90..0000000 --- a/opensim/OpenSim/Data/ITSAssetAdminData.cs +++ /dev/null @@ -1,62 +0,0 @@ -using System; - -namespace OpenSim.Data -{ - public struct TSAssetMoveReport - { - public string Source; - public string Destination; - public int CandidateCount; - public int AlreadyInTargetCount; - public int InsertedCount; - public int DeletedFromSourceCount; - public int IndexAffectedCount; - } - - public struct TSAssetMoveOptions - { - public bool ResetCheckpoint; - public int BatchSize; - public int CommandTimeoutSeconds; - } - - public struct TSAssetFindReport - { - public string AssetId; - public bool Found; - public string TableName; - public bool HasIndexEntry; - public int IndexAssetType; - } - - public struct TSAssetVerifyReport - { - public string Scope; - public int TablesChecked; - public int TotalRows; - public int MissingIndexRows; - public int WrongIndexTypeRows; - public int OrphanIndexRows; - public int LegacyRowsWithIndex; - } - - public struct TSAssetReindexReport - { - public string Scope; - public int TablesProcessed; - public int RowsScanned; - public int IndexRowsUpserted; - public int IndexRowsDeleted; - } - - public interface ITSAssetAdminData - { - bool TryMoveAssets(string from, string to, out TSAssetMoveReport report, out string errorMessage); - bool TryMoveAssets(string from, string to, TSAssetMoveOptions options, out TSAssetMoveReport report, out string errorMessage); - bool TryPreviewMoveAssets(string from, string to, out TSAssetMoveReport report, out string errorMessage); - bool TryFindAssetLocation(string assetId, out TSAssetFindReport report, out string errorMessage); - bool TryVerifyAssets(string scope, out TSAssetVerifyReport report, out string errorMessage); - bool TryReindexAssets(string scope, TSAssetMoveOptions options, out TSAssetReindexReport report, out string errorMessage); - bool TryCleanLegacyIndex(TSAssetMoveOptions options, out TSAssetReindexReport report, out string errorMessage); - } -} diff --git a/opensim/OpenSim/Data/MySQL/MySQLtsAssetData.cs b/opensim/OpenSim/Data/MySQL/MySQLtsAssetData.cs deleted file mode 100644 index 5d64c9b..0000000 --- a/opensim/OpenSim/Data/MySQL/MySQLtsAssetData.cs +++ /dev/null @@ -1,1687 +0,0 @@ -/* - * Copyright (c) Contributors, http://opensimulator.org/ - * See CONTRIBUTORS.TXT for a full list of copyright holders. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the OpenSimulator Project nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -using System; -using System.Collections.Generic; -using System.Data; -using System.Globalization; -using System.Reflection; -using System.Text.RegularExpressions; -using log4net; -using MySql.Data.MySqlClient; -using OpenMetaverse; -using OpenSim.Framework; -using OpenSim.Data; - -namespace OpenSim.Data.MySQL -{ - /// - /// MySQL storage provider that stores assets in type-specific tables: - /// assets_{assetType}. A legacy fallback to table "assets" can be enabled explicitly. - /// - public class MySQLtsAssetData : AssetDataBase, ITSAssetAdminData - { - private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - - private const string LegacyTableName = "assets"; - private const string IndexTableName = "tsassets_index"; - private const string MoveCheckpointTableName = "tsassets_move_checkpoint"; - private const string TypedTableNameFormat = "assets_{0}"; - private const bool DefaultFallbackToLegacy = false; - private const int DefaultTsAdminBatchSize = 1000; - private const int DefaultTsAdminCommandTimeoutSeconds = 300; - - private readonly object m_tableSync = new object(); - private readonly HashSet m_initializedTables = new HashSet(StringComparer.OrdinalIgnoreCase); - - private string m_connectionString; - private bool m_fallbackToLegacy; - private int m_tsAdminBatchSize = DefaultTsAdminBatchSize; - private int m_tsAdminCommandTimeoutSeconds = DefaultTsAdminCommandTimeoutSeconds; - - protected virtual Assembly Assembly - { - get { return GetType().Assembly; } - } - - #region IPlugin Members - - public override string Version - { - get { return "1.0.0.0"; } - } - - public override string Name - { - get { return "MySQL TSAsset storage engine"; } - } - - public override void Initialise(string connect) - { - if (string.IsNullOrEmpty(connect)) - throw new ArgumentException("Connection string must not be null or empty", nameof(connect)); - - m_connectionString = connect; - m_fallbackToLegacy = TryReadBooleanSetting(connect, "TSFallbackToLegacyAssets", DefaultFallbackToLegacy); - m_tsAdminBatchSize = Math.Max(1, TryReadIntSetting(connect, "TSAdminBatchSize", DefaultTsAdminBatchSize)); - m_tsAdminCommandTimeoutSeconds = Math.Max(30, TryReadIntSetting(connect, "TSAdminCommandTimeoutSeconds", DefaultTsAdminCommandTimeoutSeconds)); - - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - - Migration migration = new Migration(dbcon, Assembly, "TSAssetStore"); - migration.Update(); - - EnsureIndexTable(dbcon); - EnsureMoveCheckpointTable(dbcon); - } - } - - public override void Initialise() - { - throw new NotImplementedException(); - } - - public override void Dispose() - { - } - - #endregion - - #region IAssetDataPlugin Members - - public override AssetBase GetAsset(UUID assetID) - { - try - { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - - if (TryGetIndexedAssetType(dbcon, assetID, out sbyte indexedType)) - { - string typedTable = GetTypeTableName(indexedType); - - AssetBase typedAsset = GetAssetFromTable(dbcon, typedTable, assetID); - if (typedAsset != null) - return typedAsset; - - RemoveIndexRow(dbcon, assetID); - } - - if (m_fallbackToLegacy) - { - AssetBase legacyAsset = GetAssetFromTable(dbcon, LegacyTableName, assetID); - if (legacyAsset != null) - { - TryStoreTypedWithExistingConnection(dbcon, legacyAsset); - return legacyAsset; - } - } - } - } - catch (Exception e) - { - m_log.ErrorFormat("[TSASSET DB]: MySQL failure fetching asset {0}. Error: {1}", assetID, e.Message); - } - - return null; - } - - public override bool StoreAsset(AssetBase asset) - { - if (asset == null) - return false; - - try - { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - return TryStoreTypedWithExistingConnection(dbcon, asset); - } - } - catch (Exception e) - { - m_log.ErrorFormat("[TSASSET DB]: MySQL failure storing asset {0}. Error: {1}", asset.FullID, e.Message); - return false; - } - } - - public override bool[] AssetsExist(UUID[] uuids) - { - if (uuids == null || uuids.Length == 0) - return new bool[0]; - - bool[] results = new bool[uuids.Length]; - Dictionary> positions = new Dictionary>(); - - for (int i = 0; i < uuids.Length; i++) - { - if (!positions.TryGetValue(uuids[i], out List indexes)) - { - indexes = new List(); - positions[uuids[i]] = indexes; - } - - indexes.Add(i); - } - - try - { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - - HashSet found = QueryExistingIds(dbcon, IndexTableName, "id", uuids); - - if (m_fallbackToLegacy) - { - UUID[] missing = BuildMissingArray(uuids, found); - if (missing.Length > 0) - { - HashSet legacyFound = QueryExistingIds(dbcon, LegacyTableName, "id", missing); - foreach (UUID id in legacyFound) - found.Add(id); - } - } - - foreach (UUID id in found) - { - if (positions.TryGetValue(id, out List idxList)) - { - foreach (int idx in idxList) - results[idx] = true; - } - } - } - } - catch (Exception e) - { - m_log.ErrorFormat("[TSASSET DB]: MySQL failure checking asset existence. Error: {0}", e.Message); - } - - return results; - } - - public override List FetchAssetMetadataSet(int start, int count) - { - List result = new List(Math.Max(0, count)); - List> pending = new List>(Math.Max(0, count)); - - if (count <= 0) - return result; - - try - { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - - using (MySqlCommand cmd = new MySqlCommand( - $"SELECT id, assetType FROM `{IndexTableName}` ORDER BY updated_at DESC LIMIT ?start, ?count", - dbcon)) - { - cmd.Parameters.AddWithValue("?start", start); - cmd.Parameters.AddWithValue("?count", count); - - using (MySqlDataReader dbReader = cmd.ExecuteReader()) - { - while (dbReader.Read()) - { - UUID id = DBGuid.FromDB(dbReader["id"]); - int typeInt = Convert.ToInt32(dbReader["assetType"], CultureInfo.InvariantCulture); - sbyte type = Convert.ToSByte(typeInt, CultureInfo.InvariantCulture); - pending.Add(new KeyValuePair(id, type)); - } - } - - for (int i = 0; i < pending.Count; i++) - { - AssetMetadata metadata = GetMetadataByIdAndType(dbcon, pending[i].Key, pending[i].Value); - if (metadata != null) - result.Add(metadata); - } - } - } - } - catch (Exception e) - { - m_log.ErrorFormat("[TSASSET DB]: MySQL failure fetching metadata set from {0}, count {1}. Error: {2}", start, count, e.Message); - } - - return result; - } - - public override bool Delete(string id) - { - if (!UUID.TryParse(id, out UUID assetId)) - return false; - - try - { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - - using (MySqlTransaction tx = dbcon.BeginTransaction()) - { - try - { - if (TryGetIndexedAssetType(dbcon, tx, assetId, out sbyte indexedType)) - { - string typedTable = GetTypeTableName(indexedType); - - using (MySqlCommand deleteTyped = new MySqlCommand($"DELETE FROM `{typedTable}` WHERE id=?id", dbcon, tx)) - { - deleteTyped.Parameters.AddWithValue("?id", assetId.ToString()); - deleteTyped.ExecuteNonQuery(); - } - } - - using (MySqlCommand deleteIndex = new MySqlCommand($"DELETE FROM `{IndexTableName}` WHERE id=?id", dbcon, tx)) - { - deleteIndex.Parameters.AddWithValue("?id", assetId.ToString()); - deleteIndex.ExecuteNonQuery(); - } - - if (m_fallbackToLegacy) - { - using (MySqlCommand deleteLegacy = new MySqlCommand($"DELETE FROM `{LegacyTableName}` WHERE id=?id", dbcon, tx)) - { - deleteLegacy.Parameters.AddWithValue("?id", assetId.ToString()); - deleteLegacy.ExecuteNonQuery(); - } - } - - tx.Commit(); - return true; - } - catch - { - tx.Rollback(); - throw; - } - } - } - } - catch (Exception e) - { - m_log.ErrorFormat("[TSASSET DB]: MySQL failure deleting asset {0}. Error: {1}", id, e.Message); - return false; - } - } - - public bool TryMoveAssets(string from, string to, out TSAssetMoveReport report, out string errorMessage) - { - TSAssetMoveOptions options = new TSAssetMoveOptions - { - ResetCheckpoint = false, - BatchSize = 0, - CommandTimeoutSeconds = 0 - }; - - return TryMoveAssets(from, to, options, out report, out errorMessage); - } - - public bool TryMoveAssets(string from, string to, TSAssetMoveOptions options, out TSAssetMoveReport report, out string errorMessage) - { - report = new TSAssetMoveReport - { - Source = from ?? string.Empty, - Destination = to ?? string.Empty, - CandidateCount = 0, - AlreadyInTargetCount = 0, - InsertedCount = 0, - DeletedFromSourceCount = 0, - IndexAffectedCount = 0 - }; - - errorMessage = string.Empty; - - if (!TryParseMoveTable(from, out MoveTableSpec sourceSpec, out string parseSourceError)) - { - errorMessage = parseSourceError; - return false; - } - - if (!TryParseMoveTable(to, out MoveTableSpec targetSpec, out string parseTargetError)) - { - errorMessage = parseTargetError; - return false; - } - - if (sourceSpec.TableName.Equals(targetSpec.TableName, StringComparison.OrdinalIgnoreCase)) - { - errorMessage = string.Format(CultureInfo.InvariantCulture, "Source and target resolve to the same table '{0}'", sourceSpec.TableName); - return false; - } - - try - { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - - EnsureIndexTable(dbcon); - - if (!sourceSpec.IsLegacy && !TableExists(dbcon, sourceSpec.TableName)) - return true; - - if (targetSpec.IsLegacy) - { - if (!TableExists(dbcon, targetSpec.TableName)) - { - errorMessage = string.Format(CultureInfo.InvariantCulture, "Target table '{0}' does not exist", targetSpec.TableName); - return false; - } - } - else - { - EnsureTypeTable(targetSpec.TableName); - } - - string whereClause = BuildMoveWhereClause(sourceSpec, targetSpec); - int sourceAssetTypeFilter = targetSpec.IsLegacy ? 0 : targetSpec.AssetType; - int effectiveBatchSize = options.BatchSize > 0 ? options.BatchSize : m_tsAdminBatchSize; - int effectiveCommandTimeoutSeconds = options.CommandTimeoutSeconds > 0 ? options.CommandTimeoutSeconds : m_tsAdminCommandTimeoutSeconds; - string operationKey = BuildMoveOperationKey(sourceSpec, targetSpec); - string cursorAfter = string.Empty; - - if (options.ResetCheckpoint) - DeleteMoveCheckpoint(dbcon, operationKey, effectiveCommandTimeoutSeconds); - - cursorAfter = GetMoveCheckpoint(dbcon, operationKey, effectiveCommandTimeoutSeconds); - - while (true) - { - using (MySqlTransaction tx = dbcon.BeginTransaction()) - { - try - { - List batchIds = GetMoveBatchIds( - dbcon, - tx, - sourceSpec, - targetSpec, - sourceAssetTypeFilter, - whereClause, - cursorAfter, - effectiveBatchSize, - effectiveCommandTimeoutSeconds); - if (batchIds.Count == 0) - { - tx.Commit(); - break; - } - - report.CandidateCount += batchIds.Count; - - int alreadyInTarget = CountRowsByIds(dbcon, tx, targetSpec.TableName, batchIds, effectiveCommandTimeoutSeconds); - report.AlreadyInTargetCount += alreadyInTarget; - - InsertBatchIntoTarget(dbcon, tx, sourceSpec, targetSpec, batchIds, effectiveCommandTimeoutSeconds); - - int nowInTarget = CountRowsByIds(dbcon, tx, targetSpec.TableName, batchIds, effectiveCommandTimeoutSeconds); - report.InsertedCount += Math.Max(0, nowInTarget - alreadyInTarget); - - if (targetSpec.IsLegacy) - report.IndexAffectedCount += DeleteRowsByIds(dbcon, tx, IndexTableName, batchIds, effectiveCommandTimeoutSeconds); - else - report.IndexAffectedCount += UpsertIndexRowsByIds(dbcon, tx, targetSpec.AssetType, batchIds, effectiveCommandTimeoutSeconds); - - report.DeletedFromSourceCount += DeleteRowsByIds(dbcon, tx, sourceSpec.TableName, batchIds, effectiveCommandTimeoutSeconds); - - cursorAfter = batchIds[batchIds.Count - 1]; - UpsertMoveCheckpoint(dbcon, tx, operationKey, cursorAfter, effectiveCommandTimeoutSeconds); - - tx.Commit(); - } - catch - { - TryRollback(tx); - throw; - } - } - } - - DeleteMoveCheckpoint(dbcon, operationKey, effectiveCommandTimeoutSeconds); - - return true; - } - } - catch (Exception e) - { - errorMessage = BuildExceptionMessage(e); - m_log.ErrorFormat("[TSASSET DB]: MySQL failure moving assets from {0} to {1}. Error: {2}", from, to, e.Message); - return false; - } - } - - public bool TryPreviewMoveAssets(string from, string to, out TSAssetMoveReport report, out string errorMessage) - { - report = new TSAssetMoveReport - { - Source = from ?? string.Empty, - Destination = to ?? string.Empty, - CandidateCount = 0, - AlreadyInTargetCount = 0, - InsertedCount = 0, - DeletedFromSourceCount = 0, - IndexAffectedCount = 0 - }; - - errorMessage = string.Empty; - - if (!TryParseMoveTable(from, out MoveTableSpec sourceSpec, out string parseSourceError)) - { - errorMessage = parseSourceError; - return false; - } - - if (!TryParseMoveTable(to, out MoveTableSpec targetSpec, out string parseTargetError)) - { - errorMessage = parseTargetError; - return false; - } - - if (sourceSpec.TableName.Equals(targetSpec.TableName, StringComparison.OrdinalIgnoreCase)) - { - errorMessage = string.Format(CultureInfo.InvariantCulture, "Source and target resolve to the same table '{0}'", sourceSpec.TableName); - return false; - } - - try - { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - - if (!sourceSpec.IsLegacy && !TableExists(dbcon, sourceSpec.TableName)) - return true; - - if (!targetSpec.IsLegacy && !TableExists(dbcon, targetSpec.TableName)) - EnsureTypeTable(targetSpec.TableName); - - if (targetSpec.IsLegacy && !TableExists(dbcon, targetSpec.TableName)) - { - errorMessage = string.Format(CultureInfo.InvariantCulture, "Target table '{0}' does not exist", targetSpec.TableName); - return false; - } - - string whereClause = BuildMoveWhereClause(sourceSpec, targetSpec); - int sourceAssetTypeFilter = targetSpec.IsLegacy ? 0 : targetSpec.AssetType; - - using (MySqlTransaction tx = dbcon.BeginTransaction(IsolationLevel.ReadCommitted)) - { - try - { - report.CandidateCount = ExecuteCount( - dbcon, - tx, - $"SELECT COUNT(*) FROM `{sourceSpec.TableName}` s {whereClause}", - sourceAssetTypeFilter, - sourceSpec, - targetSpec); - - report.AlreadyInTargetCount = ExecuteCount( - dbcon, - tx, - $"SELECT COUNT(*) FROM `{sourceSpec.TableName}` s INNER JOIN `{targetSpec.TableName}` t ON t.id = s.id {whereClause}", - sourceAssetTypeFilter, - sourceSpec, - targetSpec); - - report.InsertedCount = Math.Max(0, report.CandidateCount - report.AlreadyInTargetCount); - report.DeletedFromSourceCount = report.CandidateCount; - - if (targetSpec.IsLegacy) - report.IndexAffectedCount = report.CandidateCount; - else - report.IndexAffectedCount = report.CandidateCount; - - tx.Commit(); - return true; - } - catch - { - TryRollback(tx); - throw; - } - } - } - } - catch (Exception e) - { - errorMessage = BuildExceptionMessage(e); - m_log.ErrorFormat("[TSASSET DB]: MySQL failure previewing move from {0} to {1}. Error: {2}", from, to, e.Message); - return false; - } - } - - public bool TryFindAssetLocation(string assetId, out TSAssetFindReport report, out string errorMessage) - { - report = new TSAssetFindReport - { - AssetId = assetId ?? string.Empty, - Found = false, - TableName = string.Empty, - HasIndexEntry = false, - IndexAssetType = 0 - }; - - errorMessage = string.Empty; - - if (!UUID.TryParse(assetId, out UUID parsedId)) - { - errorMessage = "Invalid asset UUID"; - return false; - } - - try - { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - - if (TryGetIndexedAssetType(dbcon, parsedId, out sbyte indexType)) - { - report.HasIndexEntry = true; - report.IndexAssetType = indexType; - - string typedTable = GetTypeTableName(indexType); - if (TableExists(dbcon, typedTable)) - { - if (ExecuteCountById(dbcon, typedTable, parsedId) > 0) - { - report.Found = true; - report.TableName = typedTable; - return true; - } - } - } - - if (TableExists(dbcon, LegacyTableName) && ExecuteCountById(dbcon, LegacyTableName, parsedId) > 0) - { - report.Found = true; - report.TableName = LegacyTableName; - return true; - } - - List types = GetExistingTypedTableTypes(dbcon); - for (int i = 0; i < types.Count; i++) - { - string tableName = GetTypeTableName(types[i]); - if (report.HasIndexEntry && tableName.Equals(GetTypeTableName((sbyte)report.IndexAssetType), StringComparison.OrdinalIgnoreCase)) - continue; - - if (ExecuteCountById(dbcon, tableName, parsedId) > 0) - { - report.Found = true; - report.TableName = tableName; - return true; - } - } - - return true; - } - } - catch (Exception e) - { - errorMessage = e.Message; - m_log.ErrorFormat("[TSASSET DB]: MySQL failure finding asset {0}. Error: {1}", assetId, e.Message); - return false; - } - } - - public bool TryVerifyAssets(string scope, out TSAssetVerifyReport report, out string errorMessage) - { - report = new TSAssetVerifyReport - { - Scope = string.IsNullOrWhiteSpace(scope) ? "all" : scope.Trim(), - TablesChecked = 0, - TotalRows = 0, - MissingIndexRows = 0, - WrongIndexTypeRows = 0, - OrphanIndexRows = 0, - LegacyRowsWithIndex = 0 - }; - - errorMessage = string.Empty; - - try - { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - EnsureIndexTable(dbcon); - - string normalizedScope = string.IsNullOrWhiteSpace(scope) ? "all" : scope.Trim(); - List verifyTables = new List(); - - if (normalizedScope.Equals("all", StringComparison.OrdinalIgnoreCase)) - { - if (TableExists(dbcon, LegacyTableName)) - { - verifyTables.Add(new MoveTableSpec - { - TableName = LegacyTableName, - IsLegacy = true, - AssetType = 0 - }); - } - - List types = GetExistingTypedTableTypes(dbcon); - for (int i = 0; i < types.Count; i++) - { - verifyTables.Add(new MoveTableSpec - { - TableName = GetTypeTableName(types[i]), - IsLegacy = false, - AssetType = types[i] - }); - } - } - else - { - if (!TryParseMoveTable(normalizedScope, out MoveTableSpec requestedTable, out string parseError)) - { - errorMessage = parseError; - return false; - } - - if (!TableExists(dbcon, requestedTable.TableName)) - { - return true; - } - - verifyTables.Add(requestedTable); - } - - for (int i = 0; i < verifyTables.Count; i++) - { - MoveTableSpec table = verifyTables[i]; - report.TablesChecked++; - report.TotalRows += ExecuteCountRaw(dbcon, string.Format(CultureInfo.InvariantCulture, "SELECT COUNT(*) FROM `{0}`", table.TableName)); - - if (table.IsLegacy) - { - report.LegacyRowsWithIndex += ExecuteCountRaw( - dbcon, - $"SELECT COUNT(*) FROM `{LegacyTableName}` l INNER JOIN `{IndexTableName}` i ON i.id = l.id"); - continue; - } - - report.MissingIndexRows += ExecuteCountRaw( - dbcon, - $"SELECT COUNT(*) FROM `{table.TableName}` t LEFT JOIN `{IndexTableName}` i ON i.id = t.id WHERE i.id IS NULL"); - - report.WrongIndexTypeRows += ExecuteCountRaw( - dbcon, - $"SELECT COUNT(*) FROM `{table.TableName}` t INNER JOIN `{IndexTableName}` i ON i.id = t.id WHERE i.assetType <> {table.AssetType.ToString(CultureInfo.InvariantCulture)}"); - - report.OrphanIndexRows += ExecuteCountRaw( - dbcon, - $"SELECT COUNT(*) FROM `{IndexTableName}` i LEFT JOIN `{table.TableName}` t ON t.id = i.id WHERE i.assetType = {table.AssetType.ToString(CultureInfo.InvariantCulture)} AND t.id IS NULL"); - } - - return true; - } - } - catch (Exception e) - { - errorMessage = e.Message; - m_log.ErrorFormat("[TSASSET DB]: MySQL failure verifying scope {0}. Error: {1}", scope, e.Message); - return false; - } - } - - public bool TryReindexAssets(string scope, TSAssetMoveOptions options, out TSAssetReindexReport report, out string errorMessage) - { - report = new TSAssetReindexReport - { - Scope = string.IsNullOrWhiteSpace(scope) ? "all" : scope.Trim(), - TablesProcessed = 0, - RowsScanned = 0, - IndexRowsUpserted = 0, - IndexRowsDeleted = 0 - }; - - errorMessage = string.Empty; - - try - { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - EnsureIndexTable(dbcon); - EnsureMoveCheckpointTable(dbcon); - - int effectiveBatchSize = options.BatchSize > 0 ? options.BatchSize : m_tsAdminBatchSize; - int effectiveCommandTimeoutSeconds = options.CommandTimeoutSeconds > 0 ? options.CommandTimeoutSeconds : m_tsAdminCommandTimeoutSeconds; - - if (!TryResolveScopeTables(dbcon, scope, out List tables, out errorMessage)) - return false; - - for (int tableIndex = 0; tableIndex < tables.Count; tableIndex++) - { - MoveTableSpec table = tables[tableIndex]; - report.TablesProcessed++; - - string opKey = string.Format(CultureInfo.InvariantCulture, "reindex:{0}", table.TableName); - if (options.ResetCheckpoint) - DeleteMoveCheckpoint(dbcon, opKey, effectiveCommandTimeoutSeconds); - - string cursorAfter = GetMoveCheckpoint(dbcon, opKey, effectiveCommandTimeoutSeconds); - - while (true) - { - using (MySqlTransaction tx = dbcon.BeginTransaction()) - { - try - { - List ids = GetBatchIdsByTable(dbcon, tx, table.TableName, cursorAfter, effectiveBatchSize, effectiveCommandTimeoutSeconds); - if (ids.Count == 0) - { - tx.Commit(); - break; - } - - report.RowsScanned += ids.Count; - - if (table.IsLegacy) - report.IndexRowsDeleted += DeleteRowsByIds(dbcon, tx, IndexTableName, ids, effectiveCommandTimeoutSeconds); - else - report.IndexRowsUpserted += UpsertIndexRowsByIds(dbcon, tx, table.AssetType, ids, effectiveCommandTimeoutSeconds); - - cursorAfter = ids[ids.Count - 1]; - UpsertMoveCheckpoint(dbcon, tx, opKey, cursorAfter, effectiveCommandTimeoutSeconds); - - tx.Commit(); - } - catch - { - TryRollback(tx); - throw; - } - } - } - - DeleteMoveCheckpoint(dbcon, opKey, effectiveCommandTimeoutSeconds); - } - - return true; - } - } - catch (Exception e) - { - errorMessage = BuildExceptionMessage(e); - m_log.ErrorFormat("[TSASSET DB]: MySQL failure reindexing scope {0}. Error: {1}", scope, e.Message); - return false; - } - } - - public bool TryCleanLegacyIndex(TSAssetMoveOptions options, out TSAssetReindexReport report, out string errorMessage) - { - return TryReindexAssets("assets", options, out report, out errorMessage); - } - - #endregion - - private sealed class MoveTableSpec - { - public string TableName; - public bool IsLegacy; - public sbyte AssetType; - } - - private static bool TryParseMoveTable(string token, out MoveTableSpec spec, out string error) - { - spec = null; - error = string.Empty; - - if (string.IsNullOrWhiteSpace(token)) - { - error = "Table token is empty"; - return false; - } - - string normalized = token.Trim(); - - if (normalized.Equals(LegacyTableName, StringComparison.OrdinalIgnoreCase)) - { - spec = new MoveTableSpec - { - TableName = LegacyTableName, - IsLegacy = true, - AssetType = 0 - }; - return true; - } - - if (sbyte.TryParse(normalized, NumberStyles.Integer, CultureInfo.InvariantCulture, out sbyte directType)) - { - spec = new MoveTableSpec - { - TableName = GetTypeTableName(directType), - IsLegacy = false, - AssetType = directType - }; - return true; - } - - Match typedName = Regex.Match(normalized, "^assets_(-?\\d+)$", RegexOptions.IgnoreCase); - if (typedName.Success) - { - if (sbyte.TryParse(typedName.Groups[1].Value, NumberStyles.Integer, CultureInfo.InvariantCulture, out sbyte parsedType)) - { - spec = new MoveTableSpec - { - TableName = GetTypeTableName(parsedType), - IsLegacy = false, - AssetType = parsedType - }; - return true; - } - } - - error = string.Format(CultureInfo.InvariantCulture, "Invalid table/type token '{0}'. Use 'assets', '' or 'assets_'", token); - return false; - } - - private bool TryResolveScopeTables(MySqlConnection dbcon, string scope, out List tables, out string error) - { - tables = new List(); - error = string.Empty; - - string normalizedScope = string.IsNullOrWhiteSpace(scope) ? "all" : scope.Trim(); - - if (normalizedScope.Equals("all", StringComparison.OrdinalIgnoreCase)) - { - if (TableExists(dbcon, LegacyTableName)) - { - tables.Add(new MoveTableSpec - { - TableName = LegacyTableName, - IsLegacy = true, - AssetType = 0 - }); - } - - List types = GetExistingTypedTableTypes(dbcon); - for (int i = 0; i < types.Count; i++) - { - tables.Add(new MoveTableSpec - { - TableName = GetTypeTableName(types[i]), - IsLegacy = false, - AssetType = types[i] - }); - } - - return true; - } - - if (!TryParseMoveTable(normalizedScope, out MoveTableSpec requestedTable, out string parseError)) - { - error = parseError; - return false; - } - - if (!TableExists(dbcon, requestedTable.TableName)) - return true; - - tables.Add(requestedTable); - return true; - } - - private static string BuildMoveWhereClause(MoveTableSpec source, MoveTableSpec target) - { - if (source.IsLegacy && !target.IsLegacy) - return "WHERE s.assetType = ?sourceTypeFilter"; - - return string.Empty; - } - - private static void BindMoveParameters(MySqlCommand cmd, int sourceAssetTypeFilter, MoveTableSpec source, MoveTableSpec target) - { - if (source.IsLegacy && !target.IsLegacy) - cmd.Parameters.AddWithValue("?sourceTypeFilter", sourceAssetTypeFilter); - } - - private static bool TableExists(MySqlConnection dbcon, string tableName) - { - using (MySqlCommand cmd = new MySqlCommand( - "SELECT 1 FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = ?tableName LIMIT 1", - dbcon)) - { - cmd.Parameters.AddWithValue("?tableName", tableName); - object value = cmd.ExecuteScalar(); - return value != null && value != DBNull.Value; - } - } - - private static int ExecuteCountById(MySqlConnection dbcon, string tableName, UUID id) - { - using (MySqlCommand cmd = new MySqlCommand($"SELECT COUNT(*) FROM `{tableName}` WHERE id=?id", dbcon)) - { - cmd.Parameters.AddWithValue("?id", id.ToString()); - object scalar = cmd.ExecuteScalar(); - if (scalar == null || scalar is DBNull) - return 0; - - return Convert.ToInt32(scalar, CultureInfo.InvariantCulture); - } - } - - private static int ExecuteCountRaw(MySqlConnection dbcon, string sql) - { - using (MySqlCommand cmd = new MySqlCommand(sql, dbcon)) - { - object scalar = cmd.ExecuteScalar(); - if (scalar == null || scalar is DBNull) - return 0; - - return Convert.ToInt32(scalar, CultureInfo.InvariantCulture); - } - } - - private static List GetExistingTypedTableTypes(MySqlConnection dbcon) - { - List result = new List(); - - using (MySqlCommand cmd = new MySqlCommand( - "SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name REGEXP '^assets_-?[0-9]+$'", - dbcon)) - using (MySqlDataReader reader = cmd.ExecuteReader()) - { - while (reader.Read()) - { - string tableName = reader["table_name"].ToString(); - Match typedMatch = Regex.Match(tableName, "^assets_(-?\\d+)$", RegexOptions.IgnoreCase); - if (!typedMatch.Success) - continue; - - if (sbyte.TryParse(typedMatch.Groups[1].Value, NumberStyles.Integer, CultureInfo.InvariantCulture, out sbyte type)) - result.Add(type); - } - } - - result.Sort(); - return result; - } - - private static int ExecuteCount(MySqlConnection dbcon, MySqlTransaction tx, string sql, int sourceAssetTypeFilter, MoveTableSpec source, MoveTableSpec target) - { - using (MySqlCommand cmd = new MySqlCommand(sql, dbcon, tx)) - { - cmd.CommandTimeout = DefaultTsAdminCommandTimeoutSeconds; - BindMoveParameters(cmd, sourceAssetTypeFilter, source, target); - object scalar = cmd.ExecuteScalar(); - if (scalar == null || scalar is DBNull) - return 0; - - return Convert.ToInt32(scalar, CultureInfo.InvariantCulture); - } - } - - private static List GetMoveBatchIds( - MySqlConnection dbcon, - MySqlTransaction tx, - MoveTableSpec sourceSpec, - MoveTableSpec targetSpec, - int sourceAssetTypeFilter, - string whereClause, - string cursorAfter, - int batchSize, - int commandTimeoutSeconds) - { - List ids = new List(batchSize); - string scopedWhere = BuildScopedWhereClause(whereClause, !string.IsNullOrEmpty(cursorAfter)); - - using (MySqlCommand cmd = new MySqlCommand( - $"SELECT s.id FROM `{sourceSpec.TableName}` s {scopedWhere} ORDER BY s.id LIMIT ?limit", - dbcon, - tx)) - { - cmd.CommandTimeout = commandTimeoutSeconds; - BindMoveParameters(cmd, sourceAssetTypeFilter, sourceSpec, targetSpec); - - if (!string.IsNullOrEmpty(cursorAfter)) - cmd.Parameters.AddWithValue("?cursorAfter", cursorAfter); - - cmd.Parameters.AddWithValue("?limit", batchSize); - - using (MySqlDataReader reader = cmd.ExecuteReader()) - { - while (reader.Read()) - ids.Add(reader["id"].ToString()); - } - } - - return ids; - } - - private static List GetBatchIdsByTable( - MySqlConnection dbcon, - MySqlTransaction tx, - string tableName, - string cursorAfter, - int batchSize, - int commandTimeoutSeconds) - { - List ids = new List(batchSize); - string whereClause = string.IsNullOrEmpty(cursorAfter) ? string.Empty : "WHERE id > ?cursorAfter"; - - using (MySqlCommand cmd = new MySqlCommand( - $"SELECT id FROM `{tableName}` {whereClause} ORDER BY id LIMIT ?limit", - dbcon, - tx)) - { - cmd.CommandTimeout = commandTimeoutSeconds; - if (!string.IsNullOrEmpty(cursorAfter)) - cmd.Parameters.AddWithValue("?cursorAfter", cursorAfter); - - cmd.Parameters.AddWithValue("?limit", batchSize); - - using (MySqlDataReader reader = cmd.ExecuteReader()) - { - while (reader.Read()) - ids.Add(reader["id"].ToString()); - } - } - - return ids; - } - - private static int CountRowsByIds(MySqlConnection dbcon, MySqlTransaction tx, string tableName, List ids, int commandTimeoutSeconds) - { - if (ids == null || ids.Count == 0) - return 0; - - using (MySqlCommand cmd = new MySqlCommand( - $"SELECT COUNT(*) FROM `{tableName}` WHERE id IN ({BuildIdInClause(ids.Count)})", - dbcon, - tx)) - { - cmd.CommandTimeout = commandTimeoutSeconds; - AddIdParameters(cmd, ids); - object scalar = cmd.ExecuteScalar(); - if (scalar == null || scalar is DBNull) - return 0; - - return Convert.ToInt32(scalar, CultureInfo.InvariantCulture); - } - } - - private static void InsertBatchIntoTarget(MySqlConnection dbcon, MySqlTransaction tx, MoveTableSpec sourceSpec, MoveTableSpec targetSpec, List ids, int commandTimeoutSeconds) - { - if (ids == null || ids.Count == 0) - return; - - using (MySqlCommand cmd = new MySqlCommand( - $"INSERT IGNORE INTO `{targetSpec.TableName}` " + - "(id, name, description, assetType, local, temporary, create_time, access_time, asset_flags, CreatorID, data) " + - "SELECT s.id, s.name, s.description, " + - (targetSpec.IsLegacy ? "s.assetType" : "?targetAssetType") + - ", s.local, s.temporary, s.create_time, s.access_time, s.asset_flags, s.CreatorID, s.data " + - $"FROM `{sourceSpec.TableName}` s WHERE s.id IN ({BuildIdInClause(ids.Count)})", - dbcon, - tx)) - { - cmd.CommandTimeout = commandTimeoutSeconds; - AddIdParameters(cmd, ids); - - if (!targetSpec.IsLegacy) - cmd.Parameters.AddWithValue("?targetAssetType", targetSpec.AssetType); - - cmd.ExecuteNonQuery(); - } - } - - private static int DeleteRowsByIds(MySqlConnection dbcon, MySqlTransaction tx, string tableName, List ids, int commandTimeoutSeconds) - { - if (ids == null || ids.Count == 0) - return 0; - - using (MySqlCommand cmd = new MySqlCommand( - $"DELETE FROM `{tableName}` WHERE id IN ({BuildIdInClause(ids.Count)})", - dbcon, - tx)) - { - cmd.CommandTimeout = commandTimeoutSeconds; - AddIdParameters(cmd, ids); - return cmd.ExecuteNonQuery(); - } - } - - private static int UpsertIndexRowsByIds(MySqlConnection dbcon, MySqlTransaction tx, sbyte targetType, List ids, int commandTimeoutSeconds) - { - if (ids == null || ids.Count == 0) - return 0; - - using (MySqlCommand cmd = new MySqlCommand( - $"INSERT INTO `{IndexTableName}` (id, assetType, updated_at) " + - $"SELECT id, ?assetType, ?updatedAt FROM `{GetTypeTableName(targetType)}` WHERE id IN ({BuildIdInClause(ids.Count)}) " + - "ON DUPLICATE KEY UPDATE assetType = VALUES(assetType), updated_at = VALUES(updated_at)", - dbcon, - tx)) - { - cmd.CommandTimeout = commandTimeoutSeconds; - cmd.Parameters.AddWithValue("?assetType", targetType); - cmd.Parameters.AddWithValue("?updatedAt", (int)Utils.DateTimeToUnixTime(DateTime.UtcNow)); - AddIdParameters(cmd, ids); - return cmd.ExecuteNonQuery(); - } - } - - private static string BuildScopedWhereClause(string whereClause, bool includeCursor) - { - if (!includeCursor) - return whereClause; - - if (string.IsNullOrWhiteSpace(whereClause)) - return "WHERE s.id > ?cursorAfter"; - - return string.Concat(whereClause, " AND s.id > ?cursorAfter"); - } - - private static string BuildMoveOperationKey(MoveTableSpec sourceSpec, MoveTableSpec targetSpec) - { - return string.Format(CultureInfo.InvariantCulture, "{0}->{1}", sourceSpec.TableName, targetSpec.TableName); - } - - private static void EnsureMoveCheckpointTable(MySqlConnection dbcon) - { - string sql = - $"CREATE TABLE IF NOT EXISTS `{MoveCheckpointTableName}` (" + - "`op_key` varchar(190) NOT NULL," + - "`last_id` char(36) NOT NULL DEFAULT ''," + - "`updated_at` int(11) NOT NULL DEFAULT '0'," + - "PRIMARY KEY (`op_key`)" + - ") ENGINE=InnoDB DEFAULT CHARSET=utf8"; - - using (MySqlCommand cmd = new MySqlCommand(sql, dbcon)) - cmd.ExecuteNonQuery(); - } - - private static string GetMoveCheckpoint(MySqlConnection dbcon, string operationKey, int commandTimeoutSeconds) - { - using (MySqlCommand cmd = new MySqlCommand( - $"SELECT last_id FROM `{MoveCheckpointTableName}` WHERE op_key=?opKey", - dbcon)) - { - cmd.CommandTimeout = commandTimeoutSeconds; - cmd.Parameters.AddWithValue("?opKey", operationKey); - object scalar = cmd.ExecuteScalar(); - if (scalar == null || scalar is DBNull) - return string.Empty; - - return scalar.ToString(); - } - } - - private static void UpsertMoveCheckpoint(MySqlConnection dbcon, MySqlTransaction tx, string operationKey, string lastId, int commandTimeoutSeconds) - { - using (MySqlCommand cmd = new MySqlCommand( - $"INSERT INTO `{MoveCheckpointTableName}` (op_key, last_id, updated_at) VALUES (?opKey, ?lastId, ?updatedAt) " + - "ON DUPLICATE KEY UPDATE last_id = VALUES(last_id), updated_at = VALUES(updated_at)", - dbcon, - tx)) - { - cmd.CommandTimeout = commandTimeoutSeconds; - cmd.Parameters.AddWithValue("?opKey", operationKey); - cmd.Parameters.AddWithValue("?lastId", lastId ?? string.Empty); - cmd.Parameters.AddWithValue("?updatedAt", (int)Utils.DateTimeToUnixTime(DateTime.UtcNow)); - cmd.ExecuteNonQuery(); - } - } - - private static void DeleteMoveCheckpoint(MySqlConnection dbcon, string operationKey, int commandTimeoutSeconds) - { - using (MySqlCommand cmd = new MySqlCommand( - $"DELETE FROM `{MoveCheckpointTableName}` WHERE op_key=?opKey", - dbcon)) - { - cmd.CommandTimeout = commandTimeoutSeconds; - cmd.Parameters.AddWithValue("?opKey", operationKey); - cmd.ExecuteNonQuery(); - } - } - - private static string BuildIdInClause(int count) - { - string[] placeholders = new string[count]; - for (int i = 0; i < count; i++) - placeholders[i] = "?id" + i.ToString(CultureInfo.InvariantCulture); - - return string.Join(",", placeholders); - } - - private static void AddIdParameters(MySqlCommand cmd, List ids) - { - for (int i = 0; i < ids.Count; i++) - cmd.Parameters.AddWithValue("?id" + i.ToString(CultureInfo.InvariantCulture), ids[i]); - } - - private static void TryRollback(MySqlTransaction tx) - { - if (tx == null) - return; - - try - { - tx.Rollback(); - } - catch - { - } - } - - private static string BuildExceptionMessage(Exception e) - { - if (e == null) - return string.Empty; - - string message = e.Message; - Exception inner = e.InnerException; - - while (inner != null) - { - if (!string.IsNullOrEmpty(inner.Message)) - message = string.Concat(message, " | ", inner.Message); - - inner = inner.InnerException; - } - - return message; - } - - private static int TryReadIntSetting(string connectionString, string key, int defaultValue) - { - if (string.IsNullOrEmpty(connectionString) || string.IsNullOrEmpty(key)) - return defaultValue; - - string[] parts = connectionString.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries); - foreach (string rawPart in parts) - { - string part = rawPart.Trim(); - int eqIndex = part.IndexOf('='); - if (eqIndex <= 0 || eqIndex == part.Length - 1) - continue; - - string k = part.Substring(0, eqIndex).Trim(); - if (!k.Equals(key, StringComparison.OrdinalIgnoreCase)) - continue; - - string v = part.Substring(eqIndex + 1).Trim(); - if (int.TryParse(v, NumberStyles.Integer, CultureInfo.InvariantCulture, out int parsed)) - return parsed; - } - - return defaultValue; - } - - private static bool TryReadBooleanSetting(string connectionString, string key, bool defaultValue) - { - if (string.IsNullOrEmpty(connectionString) || string.IsNullOrEmpty(key)) - return defaultValue; - - string[] parts = connectionString.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries); - foreach (string rawPart in parts) - { - string part = rawPart.Trim(); - int eqIndex = part.IndexOf('='); - if (eqIndex <= 0 || eqIndex == part.Length - 1) - continue; - - string k = part.Substring(0, eqIndex).Trim(); - if (!k.Equals(key, StringComparison.OrdinalIgnoreCase)) - continue; - - string v = part.Substring(eqIndex + 1).Trim(); - if (v.Equals("1", StringComparison.OrdinalIgnoreCase) || v.Equals("true", StringComparison.OrdinalIgnoreCase) || v.Equals("yes", StringComparison.OrdinalIgnoreCase)) - return true; - - if (v.Equals("0", StringComparison.OrdinalIgnoreCase) || v.Equals("false", StringComparison.OrdinalIgnoreCase) || v.Equals("no", StringComparison.OrdinalIgnoreCase)) - return false; - } - - return defaultValue; - } - - private static string GetTypeTableName(sbyte assetType) - { - return string.Format(CultureInfo.InvariantCulture, TypedTableNameFormat, assetType); - } - - private void EnsureTypeTable(string tableName) - { - lock (m_tableSync) - { - if (m_initializedTables.Contains(tableName)) - return; - - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - CreateTypeTable(dbcon, tableName); - } - - m_initializedTables.Add(tableName); - } - } - - private void CreateTypeTable(MySqlConnection dbcon, string tableName) - { - string sql = - $"CREATE TABLE IF NOT EXISTS `{tableName}` (" + - "`name` varchar(64) NOT NULL," + - "`description` varchar(64) NOT NULL," + - "`assetType` tinyint(4) NOT NULL," + - "`local` tinyint(1) NOT NULL," + - "`temporary` tinyint(1) NOT NULL," + - "`data` longblob NOT NULL," + - "`id` char(36) NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000'," + - "`create_time` int(11) DEFAULT '0'," + - "`access_time` int(11) DEFAULT '0'," + - "`asset_flags` int(11) NOT NULL DEFAULT '0'," + - "`CreatorID` varchar(128) NOT NULL DEFAULT ''," + - "PRIMARY KEY (`id`)" + - ") ENGINE=InnoDB DEFAULT CHARSET=utf8"; - - using (MySqlCommand cmd = new MySqlCommand(sql, dbcon)) - { - cmd.ExecuteNonQuery(); - } - } - - private static void EnsureIndexTable(MySqlConnection dbcon) - { - string sql = - $"CREATE TABLE IF NOT EXISTS `{IndexTableName}` (" + - "`id` char(36) NOT NULL," + - "`assetType` tinyint(4) NOT NULL," + - "`updated_at` int(11) NOT NULL DEFAULT '0'," + - "PRIMARY KEY (`id`)," + - $"INDEX `idx_{IndexTableName}_assetType` (`assetType`)" + - ") ENGINE=InnoDB DEFAULT CHARSET=utf8"; - - using (MySqlCommand cmd = new MySqlCommand(sql, dbcon)) - { - cmd.ExecuteNonQuery(); - } - } - - private bool TryStoreTypedWithExistingConnection(MySqlConnection dbcon, AssetBase asset) - { - string tableName = GetTypeTableName(asset.Type); - EnsureTypeTable(tableName); - - string assetName = asset.Name ?? string.Empty; - if (assetName.Length > AssetBase.MAX_ASSET_NAME) - assetName = assetName.Substring(0, AssetBase.MAX_ASSET_NAME); - - string assetDescription = asset.Description ?? string.Empty; - if (assetDescription.Length > AssetBase.MAX_ASSET_DESC) - assetDescription = assetDescription.Substring(0, AssetBase.MAX_ASSET_DESC); - - int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow); - - using (MySqlTransaction tx = dbcon.BeginTransaction()) - { - try - { - string upsertAssetSql = - $"REPLACE INTO `{tableName}` " + - "(id, name, description, assetType, local, temporary, create_time, access_time, asset_flags, CreatorID, data) " + - "VALUES(?id, ?name, ?description, ?assetType, ?local, ?temporary, ?create_time, ?access_time, ?asset_flags, ?CreatorID, ?data)"; - - using (MySqlCommand assetCmd = new MySqlCommand(upsertAssetSql, dbcon, tx)) - { - assetCmd.Parameters.AddWithValue("?id", asset.ID); - assetCmd.Parameters.AddWithValue("?name", assetName); - assetCmd.Parameters.AddWithValue("?description", assetDescription); - assetCmd.Parameters.AddWithValue("?assetType", asset.Type); - assetCmd.Parameters.AddWithValue("?local", asset.Local); - assetCmd.Parameters.AddWithValue("?temporary", asset.Temporary); - assetCmd.Parameters.AddWithValue("?create_time", now); - assetCmd.Parameters.AddWithValue("?access_time", now); - assetCmd.Parameters.AddWithValue("?asset_flags", (int)asset.Flags); - assetCmd.Parameters.AddWithValue("?CreatorID", asset.Metadata.CreatorID ?? string.Empty); - assetCmd.Parameters.AddWithValue("?data", asset.Data ?? Array.Empty()); - assetCmd.ExecuteNonQuery(); - } - - using (MySqlCommand indexCmd = new MySqlCommand( - $"REPLACE INTO `{IndexTableName}` (id, assetType, updated_at) VALUES (?id, ?assetType, ?updated_at)", - dbcon, - tx)) - { - indexCmd.Parameters.AddWithValue("?id", asset.ID); - indexCmd.Parameters.AddWithValue("?assetType", asset.Type); - indexCmd.Parameters.AddWithValue("?updated_at", now); - indexCmd.ExecuteNonQuery(); - } - - tx.Commit(); - return true; - } - catch - { - tx.Rollback(); - throw; - } - } - } - - private AssetBase GetAssetFromTable(MySqlConnection dbcon, string tableName, UUID assetID) - { - string sql = - $"SELECT name, description, assetType, local, temporary, asset_flags, CreatorID, data " + - $"FROM `{tableName}` WHERE id=?id"; - - try - { - using (MySqlCommand cmd = new MySqlCommand(sql, dbcon)) - { - cmd.Parameters.AddWithValue("?id", assetID.ToString()); - - using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) - { - if (!dbReader.Read()) - return null; - - AssetBase asset = new AssetBase(assetID, (string)dbReader["name"], (sbyte)dbReader["assetType"], dbReader["CreatorID"].ToString()); - asset.Description = (string)dbReader["description"]; - asset.Local = Convert.ToBoolean(dbReader["local"]); - asset.Temporary = Convert.ToBoolean(dbReader["temporary"]); - asset.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"], CultureInfo.InvariantCulture); - asset.Data = (byte[])dbReader["data"]; - return asset; - } - } - } - catch (MySqlException) - { - return null; - } - } - - private AssetMetadata GetMetadataByIdAndType(MySqlConnection dbcon, UUID id, sbyte type) - { - string tableName = GetTypeTableName(type); - - string sql = - $"SELECT id, name, description, assetType, temporary, asset_flags, CreatorID " + - $"FROM `{tableName}` WHERE id=?id"; - - try - { - using (MySqlCommand cmd = new MySqlCommand(sql, dbcon)) - { - cmd.Parameters.AddWithValue("?id", id.ToString()); - using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) - { - if (dbReader.Read()) - return ReadMetadata(dbReader); - } - } - } - catch (MySqlException) - { - // Type table may not yet exist if no asset of this type was stored. - } - - if (!m_fallbackToLegacy) - return null; - - using (MySqlCommand legacyCmd = new MySqlCommand( - $"SELECT id, name, description, assetType, temporary, asset_flags, CreatorID FROM `{LegacyTableName}` WHERE id=?id", - dbcon)) - { - legacyCmd.Parameters.AddWithValue("?id", id.ToString()); - using (MySqlDataReader dbReader = legacyCmd.ExecuteReader(CommandBehavior.SingleRow)) - { - if (dbReader.Read()) - return ReadMetadata(dbReader); - } - } - - return null; - } - - private static AssetMetadata ReadMetadata(MySqlDataReader dbReader) - { - AssetMetadata metadata = new AssetMetadata(); - metadata.FullID = DBGuid.FromDB(dbReader["id"]); - metadata.Name = dbReader["name"].ToString(); - metadata.Description = dbReader["description"].ToString(); - metadata.Type = Convert.ToSByte(dbReader["assetType"], CultureInfo.InvariantCulture); - metadata.Temporary = Convert.ToBoolean(dbReader["temporary"]); - metadata.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"], CultureInfo.InvariantCulture); - metadata.CreatorID = dbReader["CreatorID"].ToString(); - metadata.SHA1 = Array.Empty(); - return metadata; - } - - private static UUID[] BuildMissingArray(UUID[] requested, HashSet found) - { - List missing = new List(requested.Length); - for (int i = 0; i < requested.Length; i++) - { - if (!found.Contains(requested[i])) - missing.Add(requested[i]); - } - - return missing.ToArray(); - } - - private static HashSet QueryExistingIds(MySqlConnection dbcon, string tableName, string idColumn, UUID[] uuids) - { - HashSet found = new HashSet(); - - if (uuids.Length == 0) - return found; - - const int batchSize = 200; - for (int offset = 0; offset < uuids.Length; offset += batchSize) - { - int take = Math.Min(batchSize, uuids.Length - offset); - string[] placeholders = new string[take]; - - using (MySqlCommand cmd = dbcon.CreateCommand()) - { - for (int i = 0; i < take; i++) - { - string paramName = "?id" + i.ToString(CultureInfo.InvariantCulture); - placeholders[i] = paramName; - cmd.Parameters.AddWithValue(paramName, uuids[offset + i].ToString()); - } - - cmd.CommandText = string.Format( - CultureInfo.InvariantCulture, - "SELECT {0} FROM `{1}` WHERE {0} IN ({2})", - idColumn, - tableName, - string.Join(",", placeholders)); - - using (MySqlDataReader dbReader = cmd.ExecuteReader()) - { - while (dbReader.Read()) - { - UUID id = DBGuid.FromDB(dbReader[idColumn]); - found.Add(id); - } - } - } - } - - return found; - } - - private bool TryGetIndexedAssetType(MySqlConnection dbcon, UUID assetID, out sbyte assetType) - { - return TryGetIndexedAssetType(dbcon, null, assetID, out assetType); - } - - private bool TryGetIndexedAssetType(MySqlConnection dbcon, MySqlTransaction tx, UUID assetID, out sbyte assetType) - { - using (MySqlCommand cmd = new MySqlCommand($"SELECT assetType FROM `{IndexTableName}` WHERE id=?id", dbcon, tx)) - { - cmd.Parameters.AddWithValue("?id", assetID.ToString()); - - object value = cmd.ExecuteScalar(); - if (value == null || value is DBNull) - { - assetType = 0; - return false; - } - - int typeInt = Convert.ToInt32(value, CultureInfo.InvariantCulture); - if (typeInt < sbyte.MinValue || typeInt > sbyte.MaxValue) - { - assetType = 0; - return false; - } - - assetType = (sbyte)typeInt; - return true; - } - } - - private static void RemoveIndexRow(MySqlConnection dbcon, UUID assetID) - { - using (MySqlCommand cmd = new MySqlCommand($"DELETE FROM `{IndexTableName}` WHERE id=?id", dbcon)) - { - cmd.Parameters.AddWithValue("?id", assetID.ToString()); - cmd.ExecuteNonQuery(); - } - } - } -} diff --git a/opensim/OpenSim/Data/MySQL/Resources/TSAssetStore.migrations b/opensim/OpenSim/Data/MySQL/Resources/TSAssetStore.migrations deleted file mode 100644 index 372b3cf..0000000 --- a/opensim/OpenSim/Data/MySQL/Resources/TSAssetStore.migrations +++ /dev/null @@ -1,18 +0,0 @@ -# ----------------- -:VERSION 1 - -BEGIN; - -CREATE TABLE IF NOT EXISTS `tsassets_index` ( - `id` char(36) NOT NULL, - `assetType` tinyint(4) NOT NULL, - `updated_at` int(11) NOT NULL DEFAULT '0', - PRIMARY KEY (`id`), - INDEX `idx_tsassets_index_assetType` (`assetType`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- Type tables are created dynamically at runtime --- using the same schema and name pattern: assets_ --- e.g. assets_1, assets_22, assets_-1, assets_-2. - -COMMIT; diff --git a/opensim/OpenSim/Data/Tests/AssetTests.cs b/opensim/OpenSim/Data/Tests/AssetTests.cs deleted file mode 100644 index 65c04be..0000000 --- a/opensim/OpenSim/Data/Tests/AssetTests.cs +++ /dev/null @@ -1,323 +0,0 @@ -/* - * Copyright (c) Contributors, http://opensimulator.org/ - * See CONTRIBUTORS.TXT for a full list of copyright holders. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the OpenSimulator Project nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -using System; -using System.Collections.Generic; -using log4net.Config; -using NUnit.Framework; -using NUnit.Framework.Constraints; -using OpenMetaverse; -using OpenSim.Framework; -using OpenSim.Tests.Common; -using System.Data.Common; -using log4net; - -// DBMS-specific: -using MySql.Data.MySqlClient; -using OpenSim.Data.MySQL; - -using Mono.Data.Sqlite; -using OpenSim.Data.SQLite; - -namespace OpenSim.Data.Tests -{ - [TestFixture(Description = "Asset store tests (SQLite)")] - public class SQLiteAssetTests : AssetTests - { - } - - [TestFixture(Description = "Asset store tests (MySQL)")] - public class MySqlAssetTests : AssetTests - { - } - - [TestFixture(Description = "Asset store tests (MySQL TSAsset)")] - public class MySqlTsAssetTests : AssetTests - { - protected override void ClearDB() - { - DropTables( - "`tsassets_index`", - "`assets_0`", - "`assets_-2`", - "`assets_49`"); - - ResetMigrations("TSAssetStore"); - } - - [Test] - public void T030_StoreReadKnownNegativeAndMeshTypes() - { - TestHelpers.InMethod(); - - UUID legacyId = UUID.Random(); - UUID meshId = UUID.Random(); - - AssetBase legacyMaterial = new AssetBase(legacyId, "legacy material", -2, UUID.Random().ToString()); - legacyMaterial.Data = new byte[] { 1, 2, 3 }; - - AssetBase mesh = new AssetBase(meshId, "mesh asset", 49, UUID.Random().ToString()); - mesh.Data = new byte[] { 4, 5, 6 }; - - Assert.IsTrue(m_db.StoreAsset(legacyMaterial), "StoreAsset should succeed for type -2"); - Assert.IsTrue(m_db.StoreAsset(mesh), "StoreAsset should succeed for type 49"); - - AssetBase legacyMaterialRead = m_db.GetAsset(legacyId); - AssetBase meshRead = m_db.GetAsset(meshId); - - Assert.That(legacyMaterialRead, Is.Not.Null); - Assert.That(meshRead, Is.Not.Null); - Assert.That(legacyMaterialRead.Type, Is.EqualTo((sbyte)-2)); - Assert.That(meshRead.Type, Is.EqualTo((sbyte)49)); - - bool[] exists = m_db.AssetsExist(new[] { legacyId, meshId }); - Assert.That(exists[0], Is.True); - Assert.That(exists[1], Is.True); - - int tableCount = 0; - ExecQuery( - "SELECT COUNT(*) AS cnt FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name IN ('assets_-2','assets_49')", - true, - reader => - { - tableCount = Convert.ToInt32(reader["cnt"]); - return false; - }); - - Assert.That(tableCount, Is.EqualTo(2), "Expected dynamic type tables assets_-2 and assets_49 to exist"); - } - - [Test] - public void T031_StoreReadBoundarySignedTinyIntAssetTypes() - { - TestHelpers.InMethod(); - - UUID minId = UUID.Random(); - UUID maxId = UUID.Random(); - - AssetBase minTypeAsset = new AssetBase(minId, "min type asset", -128, UUID.Random().ToString()); - minTypeAsset.Data = new byte[] { 10, 20, 30 }; - - AssetBase maxTypeAsset = new AssetBase(maxId, "max type asset", 127, UUID.Random().ToString()); - maxTypeAsset.Data = new byte[] { 40, 50, 60 }; - - Assert.IsTrue(m_db.StoreAsset(minTypeAsset), "StoreAsset should succeed for type -128"); - Assert.IsTrue(m_db.StoreAsset(maxTypeAsset), "StoreAsset should succeed for type 127"); - - AssetBase minTypeRead = m_db.GetAsset(minId); - AssetBase maxTypeRead = m_db.GetAsset(maxId); - - Assert.That(minTypeRead, Is.Not.Null); - Assert.That(maxTypeRead, Is.Not.Null); - Assert.That(minTypeRead.Type, Is.EqualTo((sbyte)-128)); - Assert.That(maxTypeRead.Type, Is.EqualTo((sbyte)127)); - - bool[] exists = m_db.AssetsExist(new[] { minId, maxId }); - Assert.That(exists[0], Is.True); - Assert.That(exists[1], Is.True); - - int tableCount = 0; - ExecQuery( - "SELECT COUNT(*) AS cnt FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name IN ('assets_-128','assets_127')", - true, - reader => - { - tableCount = Convert.ToInt32(reader["cnt"]); - return false; - }); - - Assert.That(tableCount, Is.EqualTo(2), "Expected dynamic type tables assets_-128 and assets_127 to exist"); - } - } - - public class AssetTests : BasicDataServiceTest - where TConn : DbConnection, new() - where TAssetData : AssetDataBase, new() - { - protected TAssetData m_db; - - public UUID uuid1 = UUID.Random(); - public UUID uuid2 = UUID.Random(); - public UUID uuid3 = UUID.Random(); - - public string critter1 = UUID.Random().ToString(); - public string critter2 = UUID.Random().ToString(); - public string critter3 = UUID.Random().ToString(); - - public byte[] data1 = new byte[100]; - - PropertyScrambler scrambler = new PropertyScrambler() - .DontScramble(x => x.ID) - .DontScramble(x => x.Type) - .DontScramble(x => x.FullID) - .DontScramble(x => x.Metadata.ID) - .DontScramble(x => x.Metadata.CreatorID) - .DontScramble(x => x.Metadata.ContentType) - .DontScramble(x => x.Metadata.FullID) - .DontScramble(x => x.Data); - - protected override void InitService(object service) - { - ClearDB(); - m_db = (TAssetData)service; - m_db.Initialise(m_connStr); - } - - protected virtual void ClearDB() - { - DropTables("assets"); - ResetMigrations("AssetStore"); - } - - - [Test] - public void T001_LoadEmpty() - { - TestHelpers.InMethod(); - - bool[] exist = m_db.AssetsExist(new[] { uuid1, uuid2, uuid3 }); - Assert.IsFalse(exist[0]); - Assert.IsFalse(exist[1]); - Assert.IsFalse(exist[2]); - } - - [Test] - public void T010_StoreReadVerifyAssets() - { - TestHelpers.InMethod(); - - AssetBase a1 = new AssetBase(uuid1, "asset one", (sbyte)AssetType.Texture, critter1.ToString()); - AssetBase a2 = new AssetBase(uuid2, "asset two", (sbyte)AssetType.Texture, critter2.ToString()); - AssetBase a3 = new AssetBase(uuid3, "asset three", (sbyte)AssetType.Texture, critter3.ToString()); - a1.Data = data1; - a2.Data = data1; - a3.Data = data1; - - scrambler.Scramble(a1); - scrambler.Scramble(a2); - scrambler.Scramble(a3); - - m_db.StoreAsset(a1); - m_db.StoreAsset(a2); - m_db.StoreAsset(a3); - a1.UploadAttempts = 0; - a2.UploadAttempts = 0; - a3.UploadAttempts = 0; - - AssetBase a1a = m_db.GetAsset(uuid1); - a1a.UploadAttempts = 0; - Assert.That(a1a, Constraints.PropertyCompareConstraint(a1)); - - AssetBase a2a = m_db.GetAsset(uuid2); - a2a.UploadAttempts = 0; - Assert.That(a2a, Constraints.PropertyCompareConstraint(a2)); - - AssetBase a3a = m_db.GetAsset(uuid3); - a3a.UploadAttempts = 0; - Assert.That(a3a, Constraints.PropertyCompareConstraint(a3)); - - scrambler.Scramble(a1a); - scrambler.Scramble(a2a); - scrambler.Scramble(a3a); - - m_db.StoreAsset(a1a); - m_db.StoreAsset(a2a); - m_db.StoreAsset(a3a); - a1a.UploadAttempts = 0; - a2a.UploadAttempts = 0; - a3a.UploadAttempts = 0; - - AssetBase a1b = m_db.GetAsset(uuid1); - a1b.UploadAttempts = 0; - Assert.That(a1b, Constraints.PropertyCompareConstraint(a1a)); - - AssetBase a2b = m_db.GetAsset(uuid2); - a2b.UploadAttempts = 0; - Assert.That(a2b, Constraints.PropertyCompareConstraint(a2a)); - - AssetBase a3b = m_db.GetAsset(uuid3); - a3b.UploadAttempts = 0; - Assert.That(a3b, Constraints.PropertyCompareConstraint(a3a)); - - bool[] exist = m_db.AssetsExist(new[] { uuid1, uuid2, uuid3 }); - Assert.IsTrue(exist[0]); - Assert.IsTrue(exist[1]); - Assert.IsTrue(exist[2]); - - List metadatas = m_db.FetchAssetMetadataSet(0, 1000); - - Assert.That(metadatas.Count >= 3, "FetchAssetMetadataSet() should have returned at least 3 assets!"); - - // It is possible that the Asset table is filled with data, in which case we don't try to find "our" - // assets there: - if (metadatas.Count < 1000) - { - AssetMetadata metadata = metadatas.Find(x => x.FullID == uuid1); - Assert.That(metadata.Name, Is.EqualTo(a1b.Name)); - Assert.That(metadata.Description, Is.EqualTo(a1b.Description)); - Assert.That(metadata.Type, Is.EqualTo(a1b.Type)); - Assert.That(metadata.Temporary, Is.EqualTo(a1b.Temporary)); - Assert.That(metadata.FullID, Is.EqualTo(a1b.FullID)); - } - } - - [Test] - public void T020_CheckForWeirdCreatorID() - { - TestHelpers.InMethod(); - - // It is expected that eventually the CreatorID might be an arbitrary string (an URI) - // rather than a valid UUID (?). This test is to make sure that the database layer does not - // attempt to convert CreatorID to GUID, but just passes it both ways as a string. - AssetBase a1 = new AssetBase(uuid1, "asset one", (sbyte)AssetType.Texture, critter1); - AssetBase a2 = new AssetBase(uuid2, "asset two", (sbyte)AssetType.Texture, "This is not a GUID!"); - AssetBase a3 = new AssetBase(uuid3, "asset three", (sbyte)AssetType.Texture, ""); - a1.Data = data1; - a2.Data = data1; - a3.Data = data1; - - m_db.StoreAsset(a1); - a1.UploadAttempts = 0; - m_db.StoreAsset(a2); - a2.UploadAttempts = 0; - m_db.StoreAsset(a3); - a3.UploadAttempts = 0; - - AssetBase a1a = m_db.GetAsset(uuid1); - a1a.UploadAttempts = 0; - Assert.That(a1a, Constraints.PropertyCompareConstraint(a1)); - - AssetBase a2a = m_db.GetAsset(uuid2); - a2a.UploadAttempts = 0; - Assert.That(a2a, Constraints.PropertyCompareConstraint(a2)); - - AssetBase a3a = m_db.GetAsset(uuid3); - a3a.UploadAttempts = 0; - Assert.That(a3a, Constraints.PropertyCompareConstraint(a3)); - } - } -} \ No newline at end of file diff --git a/opensim/OpenSim/Services/AssetService/TSAssetConnector.cs b/opensim/OpenSim/Services/AssetService/TSAssetConnector.cs deleted file mode 100644 index 81e0fe9..0000000 --- a/opensim/OpenSim/Services/AssetService/TSAssetConnector.cs +++ /dev/null @@ -1,1182 +0,0 @@ -/* - * Copyright (c) Contributors, http://opensimulator.org/ - * See CONTRIBUTORS.TXT for a full list of copyright holders. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the OpenSimulator Project nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -using System; -using System.Collections.Generic; -using System.Collections.Concurrent; -using System.Globalization; -using System.Reflection; -using System.Threading; -using log4net; -using Nini.Config; -using OpenMetaverse; -using OpenSim.Data; -using OpenSim.Framework; -using OpenSim.Services.Base; -using OpenSim.Services.Interfaces; - -namespace OpenSim.Services.AssetService -{ - public class TSAssetConnector : ServiceBase, IAssetService - { - private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - private static readonly object m_commandLock = new object(); - private static bool m_commandsRegistered; - - private readonly Dictionary m_typeDatabases = new Dictionary(); - private readonly Dictionary m_connectionDatabases = new Dictionary(StringComparer.OrdinalIgnoreCase); - private readonly Dictionary m_assetLocationCache = new Dictionary(); - private readonly ConcurrentQueue m_fallbackMigrationQueue = new ConcurrentQueue(); - private readonly ConcurrentDictionary m_fallbackMigrationSet = new ConcurrentDictionary(); - private readonly object m_cacheLock = new object(); - private readonly HashSet m_allowedTypes = new HashSet(); - - private readonly List m_probeOrder = new List(); - - private IAssetDataPlugin m_defaultDatabase; - private IAssetLoader m_assetLoader; - private IAssetService m_fallbackService; - private Thread m_fallbackMigrationThread; - - private bool m_enableFallbackAutoMigration; - private bool m_enableFallbackAutoDelete; - private int m_migrationCheckIntervalMs = 60000; - private int m_migrationBatchSize = 25; - private int m_migrationLowTrafficMaxRequests = 3; - private int m_migrationQueueMax = 50000; - private int m_requestsInWindow; - - public TSAssetConnector(IConfigSource config) - : base(config) - { - IConfig assetConfig = config.Configs["AssetService"]; - if (assetConfig == null) - throw new Exception("No AssetService configuration"); - - IConfig tsConfig = config.Configs["TSAssetService"]; - IConfig dbConfig = config.Configs["DatabaseService"]; - - if (tsConfig == null || !tsConfig.GetBoolean("Enabled", false)) - throw new Exception("TSAssetConnector disabled. Set [TSAssetService] Enabled = true to enable."); - - string storageProvider = string.Empty; - string defaultConnectionString = string.Empty; - - if (tsConfig != null) - { - storageProvider = tsConfig.GetString("StorageProvider", string.Empty); - defaultConnectionString = tsConfig.GetString("ConnectionString", string.Empty); - } - - if (string.IsNullOrEmpty(storageProvider)) - storageProvider = assetConfig.GetString("StorageProvider", string.Empty); - - if (string.IsNullOrEmpty(defaultConnectionString)) - defaultConnectionString = assetConfig.GetString("ConnectionString", string.Empty); - - if (dbConfig != null) - { - if (string.IsNullOrEmpty(storageProvider)) - storageProvider = dbConfig.GetString("StorageProvider", string.Empty); - - if (string.IsNullOrEmpty(defaultConnectionString)) - defaultConnectionString = dbConfig.GetString("ConnectionString", string.Empty); - } - - if (string.IsNullOrEmpty(storageProvider)) - throw new Exception("No StorageProvider configured"); - - if (string.IsNullOrEmpty(defaultConnectionString)) - throw new Exception("Missing database connection string"); - - RegisterConsoleCommands(); - - ParseAllowedTypes(tsConfig); - - if (tsConfig != null) - { - m_enableFallbackAutoMigration = tsConfig.GetBoolean("EnableFallbackAutoMigration", false); - m_enableFallbackAutoDelete = tsConfig.GetBoolean("EnableFallbackAutoDelete", false); - m_migrationCheckIntervalMs = Math.Max(5000, tsConfig.GetInt("MigrationCheckIntervalSeconds", 60) * 1000); - m_migrationBatchSize = Math.Max(1, tsConfig.GetInt("MigrationBatchSize", 25)); - m_migrationLowTrafficMaxRequests = Math.Max(0, tsConfig.GetInt("MigrationLowTrafficMaxRequests", 3)); - m_migrationQueueMax = Math.Max(100, tsConfig.GetInt("MigrationQueueMax", 50000)); - } - - m_defaultDatabase = CreateAndInitDatabase(storageProvider, defaultConnectionString); - AddProbeDatabase(m_defaultDatabase); - - if (tsConfig != null) - { - ParseTypedDatabaseMappings(tsConfig.GetString("AssetDatabases", string.Empty), storageProvider); - ParseTypedDatabaseMappingsFromKeys(tsConfig, storageProvider); - } - - string loaderName = assetConfig.GetString("DefaultAssetLoader", string.Empty); - if (!string.IsNullOrEmpty(loaderName)) - { - m_assetLoader = LoadPlugin(loaderName); - if (m_assetLoader == null) - throw new Exception(string.Format("Asset loader could not be loaded from {0}", loaderName)); - - bool assetLoaderEnabled = assetConfig.GetBoolean("AssetLoaderEnabled", true); - if (assetLoaderEnabled) - { - string loaderArgs = assetConfig.GetString("AssetLoaderArgs", string.Empty); - m_log.InfoFormat("[TSASSET SERVICE]: Loading default asset set from {0}", loaderArgs); - - m_assetLoader.ForEachDefaultXmlAsset( - loaderArgs, - delegate(AssetBase a) - { - if (a == null) - return; - - if (Get(a.ID) == null) - Store(a); - }); - } - } - - string fallbackServiceName = assetConfig.GetString("FallbackService", string.Empty); - if (!string.IsNullOrEmpty(fallbackServiceName)) - { - object[] args = new object[] { config }; - m_fallbackService = LoadPlugin(fallbackServiceName, args); - if (m_fallbackService != null) - { - m_log.Info("[TSASSET SERVICE]: Fallback service loaded"); - - if (m_enableFallbackAutoMigration) - { - m_fallbackMigrationThread = new Thread(FallbackMigrationWorker); - m_fallbackMigrationThread.IsBackground = true; - m_fallbackMigrationThread.Name = "TSAssetFallbackMigration"; - m_fallbackMigrationThread.Start(); - m_log.Info("[TSASSET SERVICE]: Fallback auto-migration worker enabled"); - } - } - else - m_log.Error("[TSASSET SERVICE]: Failed to load fallback service"); - } - - m_log.InfoFormat("[TSASSET SERVICE]: Enabled with {0} type routes", m_typeDatabases.Count); - } - - public TSAssetConnector(IConfigSource config, string configName) - : this(config) - { - } - - public AssetBase Get(string id) - { - Interlocked.Increment(ref m_requestsInWindow); - - if (!UUID.TryParse(id, out UUID assetID)) - { - m_log.WarnFormat("[TSASSET SERVICE]: Could not parse requested asset id {0}", id); - return null; - } - - IAssetDataPlugin cachedDatabase = null; - lock (m_cacheLock) - { - m_assetLocationCache.TryGetValue(assetID, out cachedDatabase); - } - - if (cachedDatabase != null) - { - try - { - AssetBase cachedAsset = cachedDatabase.GetAsset(assetID); - if (cachedAsset != null) - return cachedAsset; - } - catch (Exception e) - { - m_log.WarnFormat("[TSASSET SERVICE]: Cached database lookup failed for asset {0}: {1}", assetID, e.Message); - } - } - - for (int i = 0; i < m_probeOrder.Count; i++) - { - IAssetDataPlugin db = m_probeOrder[i]; - - try - { - AssetBase asset = db.GetAsset(assetID); - if (asset != null) - { - lock (m_cacheLock) - { - m_assetLocationCache[assetID] = db; - } - return asset; - } - } - catch (Exception e) - { - m_log.WarnFormat("[TSASSET SERVICE]: Asset lookup failed for asset {0}: {1}", assetID, e.Message); - } - } - - if (m_fallbackService != null) - { - try - { - AssetBase fallbackAsset = m_fallbackService.Get(id); - if (fallbackAsset != null) - { - string storedId = Store(fallbackAsset); - bool storeOk = !storedId.Equals(UUID.Zero.ToString(), StringComparison.Ordinal); - - if (storeOk && m_enableFallbackAutoDelete) - TryDeleteFromFallback(id); - - if (m_enableFallbackAutoMigration && !storeOk) - EnqueueFallbackMigration(fallbackAsset); - - return fallbackAsset; - } - } - catch (Exception e) - { - m_log.WarnFormat("[TSASSET SERVICE]: Fallback lookup failed for asset {0}: {1}", id, e.Message); - } - } - - return null; - } - - public AssetBase Get(string id, string ForeignAssetService, bool StoreOnLocalGrid) - { - return Get(id); - } - - public AssetMetadata GetMetadata(string id) - { - AssetBase asset = Get(id); - return asset != null ? asset.Metadata : null; - } - - public byte[] GetData(string id) - { - AssetBase asset = Get(id); - return asset != null ? asset.Data : null; - } - - public AssetBase GetCached(string id) - { - return Get(id); - } - - public bool Get(string id, object sender, AssetRetrieved handler) - { - handler(id, sender, Get(id)); - return true; - } - - public void Get(string id, string ForeignAssetService, bool StoreOnLocalGrid, SimpleAssetRetrieved callBack) - { - callBack(Get(id)); - } - - public bool[] AssetsExist(string[] ids) - { - if (ids == null || ids.Length == 0) - return new bool[0]; - - bool[] results = new bool[ids.Length]; - UUID[] uuids = new UUID[ids.Length]; - bool[] valid = new bool[ids.Length]; - - for (int i = 0; i < ids.Length; i++) - { - if (UUID.TryParse(ids[i], out UUID parsed)) - { - uuids[i] = parsed; - valid[i] = true; - } - } - - for (int dbIndex = 0; dbIndex < m_probeOrder.Count; dbIndex++) - { - IAssetDataPlugin db = m_probeOrder[dbIndex]; - UUID[] query = BuildPendingUuidList(uuids, valid, results); - - if (query.Length == 0) - break; - - try - { - bool[] exists = db.AssetsExist(query); - MergeExistenceResults(query, exists, uuids, valid, results, db); - } - catch (Exception e) - { - m_log.WarnFormat("[TSASSET SERVICE]: AssetsExist failed on a database: {0}", e.Message); - } - } - - return results; - } - - public string Store(AssetBase asset) - { - if (asset == null) - return UUID.Zero.ToString(); - - IAssetDataPlugin db = ResolveDatabaseForType(asset.Type); - if (db == null) - { - m_log.WarnFormat("[TSASSET SERVICE]: No database resolved for asset {0}, type {1}", asset.FullID, asset.Type); - return UUID.Zero.ToString(); - } - - try - { - bool stored = db.StoreAsset(asset); - if (!stored) - return UUID.Zero.ToString(); - - lock (m_cacheLock) - { - m_assetLocationCache[asset.FullID] = db; - } - - return asset.ID; - } - catch (Exception e) - { - m_log.ErrorFormat("[TSASSET SERVICE]: Error storing asset {0}: {1}", asset.FullID, e.Message); - return UUID.Zero.ToString(); - } - } - - public bool UpdateContent(string id, byte[] data) - { - AssetBase asset = Get(id); - if (asset == null) - return false; - - asset.Data = data; - return Store(asset) != UUID.Zero.ToString(); - } - - public bool Delete(string id) - { - if (!UUID.TryParse(id, out UUID assetID)) - return false; - - IAssetDataPlugin cachedDatabase = null; - lock (m_cacheLock) - { - m_assetLocationCache.TryGetValue(assetID, out cachedDatabase); - } - - if (cachedDatabase != null) - { - try - { - bool deleted = cachedDatabase.Delete(id); - if (deleted) - { - lock (m_cacheLock) - { - m_assetLocationCache.Remove(assetID); - } - return true; - } - } - catch (Exception e) - { - m_log.WarnFormat("[TSASSET SERVICE]: Cached delete failed for asset {0}: {1}", assetID, e.Message); - } - } - - for (int i = 0; i < m_probeOrder.Count; i++) - { - try - { - if (m_probeOrder[i].Delete(id)) - { - lock (m_cacheLock) - { - m_assetLocationCache.Remove(assetID); - } - return true; - } - } - catch (Exception e) - { - m_log.WarnFormat("[TSASSET SERVICE]: Delete failed for asset {0}: {1}", assetID, e.Message); - } - } - - return false; - } - - private static UUID[] BuildPendingUuidList(UUID[] uuids, bool[] valid, bool[] results) - { - List pending = new List(uuids.Length); - for (int i = 0; i < uuids.Length; i++) - { - if (valid[i] && !results[i]) - pending.Add(uuids[i]); - } - - return pending.ToArray(); - } - - private void MergeExistenceResults(UUID[] query, bool[] exists, UUID[] sourceIds, bool[] valid, bool[] results, IAssetDataPlugin db) - { - if (exists == null) - return; - - int len = Math.Min(query.Length, exists.Length); - for (int i = 0; i < len; i++) - { - if (!exists[i]) - continue; - - UUID foundId = query[i]; - - for (int sourceIndex = 0; sourceIndex < sourceIds.Length; sourceIndex++) - { - if (valid[sourceIndex] && sourceIds[sourceIndex] == foundId) - { - results[sourceIndex] = true; - } - } - - lock (m_cacheLock) - { - m_assetLocationCache[foundId] = db; - } - } - } - - private void ParseAllowedTypes(IConfig tsConfig) - { - if (tsConfig == null) - return; - - string raw = tsConfig.GetString("TSAssetType", string.Empty); - if (string.IsNullOrWhiteSpace(raw)) - return; - - string[] tokens = raw.Split(new char[] { ',', ';', ' ', '\t', '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); - for (int i = 0; i < tokens.Length; i++) - { - if (sbyte.TryParse(tokens[i], NumberStyles.Integer, CultureInfo.InvariantCulture, out sbyte type)) - m_allowedTypes.Add(type); - else - m_log.WarnFormat("[TSASSET SERVICE]: Ignoring invalid TSAssetType entry '{0}'", tokens[i]); - } - } - - private void ParseTypedDatabaseMappings(string rawMappings, string storageProvider) - { - if (string.IsNullOrWhiteSpace(rawMappings)) - return; - - string[] lines = rawMappings.Split(new char[] { '\r', '\n', '|' }, StringSplitOptions.RemoveEmptyEntries); - for (int i = 0; i < lines.Length; i++) - { - string line = lines[i].Trim(); - if (string.IsNullOrEmpty(line)) - continue; - - line = line.Trim('"'); - if (line.EndsWith(";;", StringComparison.Ordinal)) - line = line.Substring(0, line.Length - 2); - else - line = line.TrimEnd(';'); - - int sep = line.IndexOf(':'); - if (sep <= 0 || sep == line.Length - 1) - continue; - - string typeToken = line.Substring(0, sep).Trim(); - string connectionString = line.Substring(sep + 1).Trim(); - - if (!sbyte.TryParse(typeToken, NumberStyles.Integer, CultureInfo.InvariantCulture, out sbyte assetType)) - { - m_log.WarnFormat("[TSASSET SERVICE]: Ignoring invalid asset type mapping '{0}'", typeToken); - continue; - } - - if (m_allowedTypes.Count > 0 && !m_allowedTypes.Contains(assetType)) - continue; - - if (string.IsNullOrEmpty(connectionString)) - { - m_log.WarnFormat("[TSASSET SERVICE]: Ignoring empty connection string for asset type {0}", assetType); - continue; - } - - IAssetDataPlugin db = GetOrCreateDatabase(storageProvider, connectionString); - m_typeDatabases[assetType] = db; - AddProbeDatabase(db); - } - } - - private void ParseTypedDatabaseMappingsFromKeys(IConfig tsConfig, string storageProvider) - { - if (tsConfig == null) - return; - - string[] keys = tsConfig.GetKeys(); - if (keys == null || keys.Length == 0) - return; - - for (int i = 0; i < keys.Length; i++) - { - string key = keys[i]; - if (string.IsNullOrWhiteSpace(key)) - continue; - - string prefix = null; - if (key.StartsWith("AssetDatabase_", StringComparison.OrdinalIgnoreCase)) - prefix = "AssetDatabase_"; - else if (key.StartsWith("AssetDatabase.", StringComparison.OrdinalIgnoreCase)) - prefix = "AssetDatabase."; - - if (prefix == null) - continue; - - string typeToken = key.Substring(prefix.Length).Trim(); - if (!sbyte.TryParse(typeToken, NumberStyles.Integer, CultureInfo.InvariantCulture, out sbyte assetType)) - { - m_log.WarnFormat("[TSASSET SERVICE]: Ignoring invalid asset type key '{0}'", key); - continue; - } - - if (m_allowedTypes.Count > 0 && !m_allowedTypes.Contains(assetType)) - continue; - - string connectionString = tsConfig.GetString(key, string.Empty).Trim(); - if (connectionString.Length >= 2 && - connectionString[0] == '"' && - connectionString[connectionString.Length - 1] == '"') - { - connectionString = connectionString.Substring(1, connectionString.Length - 2).Trim(); - } - - if (string.IsNullOrEmpty(connectionString)) - { - m_log.WarnFormat("[TSASSET SERVICE]: Ignoring empty connection string for asset type key '{0}'", key); - continue; - } - - IAssetDataPlugin db = GetOrCreateDatabase(storageProvider, connectionString); - m_typeDatabases[assetType] = db; - AddProbeDatabase(db); - } - } - - private IAssetDataPlugin ResolveDatabaseForType(sbyte type) - { - if (m_typeDatabases.TryGetValue(type, out IAssetDataPlugin db)) - return db; - - return m_defaultDatabase; - } - - private IAssetDataPlugin GetOrCreateDatabase(string storageProvider, string connectionString) - { - if (m_connectionDatabases.TryGetValue(connectionString, out IAssetDataPlugin existing)) - return existing; - - IAssetDataPlugin created = CreateAndInitDatabase(storageProvider, connectionString); - m_connectionDatabases[connectionString] = created; - return created; - } - - private IAssetDataPlugin CreateAndInitDatabase(string storageProvider, string connectionString) - { - IAssetDataPlugin database = LoadPlugin(storageProvider); - if (database == null) - throw new Exception(string.Format("Could not find a storage interface in the module {0}", storageProvider)); - - database.Initialise(connectionString); - return database; - } - - private void AddProbeDatabase(IAssetDataPlugin db) - { - for (int i = 0; i < m_probeOrder.Count; i++) - { - if (object.ReferenceEquals(m_probeOrder[i], db)) - return; - } - - m_probeOrder.Add(db); - } - - private void EnqueueFallbackMigration(AssetBase asset) - { - if (asset == null) - return; - - if (m_fallbackMigrationSet.Count >= m_migrationQueueMax) - return; - - if (m_fallbackMigrationSet.TryAdd(asset.FullID, 0)) - m_fallbackMigrationQueue.Enqueue(asset); - } - - private void FallbackMigrationWorker() - { - while (true) - { - Thread.Sleep(m_migrationCheckIntervalMs); - - int requests = Interlocked.Exchange(ref m_requestsInWindow, 0); - if (requests > m_migrationLowTrafficMaxRequests) - continue; - - int migrated = 0; - while (migrated < m_migrationBatchSize && m_fallbackMigrationQueue.TryDequeue(out AssetBase asset)) - { - try - { - string id = Store(asset); - if (!id.Equals(UUID.Zero.ToString(), StringComparison.Ordinal)) - { - if (m_enableFallbackAutoDelete) - TryDeleteFromFallback(asset.ID); - - migrated++; - } - } - catch (Exception e) - { - m_log.WarnFormat("[TSASSET SERVICE]: Auto-migration failed for asset {0}: {1}", asset != null ? asset.FullID.ToString() : "", e.Message); - } - finally - { - if (asset != null) - m_fallbackMigrationSet.TryRemove(asset.FullID, out _); - } - } - - if (migrated > 0) - m_log.InfoFormat("[TSASSET SERVICE]: Auto-migrated {0} fallback assets during low traffic", migrated); - } - } - - private void TryDeleteFromFallback(string id) - { - if (m_fallbackService == null || string.IsNullOrEmpty(id)) - return; - - try - { - if (!m_fallbackService.Delete(id)) - m_log.WarnFormat("[TSASSET SERVICE]: Fallback auto-delete failed for asset {0}", id); - } - catch (Exception e) - { - m_log.WarnFormat("[TSASSET SERVICE]: Fallback auto-delete exception for asset {0}: {1}", id, e.Message); - } - } - - private void RegisterConsoleCommands() - { - if (MainConsole.Instance == null) - return; - - lock (m_commandLock) - { - if (m_commandsRegistered) - return; - - MainConsole.Instance.Commands.AddCommand( - "tsasset", - false, - "tsmove", - "tsmove --force [--reset] [--batch=] [--timeout=]", - "Move tsasset rows between tables (requires --force). Supports resume/reset and batch/timeout overrides", - HandleTsMove); - - MainConsole.Instance.Commands.AddCommand( - "tsasset", - false, - "tsshowmove", - "tsshowmove ", - "Preview tsasset move counts without writing changes", - HandleTsShowMove); - - MainConsole.Instance.Commands.AddCommand( - "tsasset", - false, - "tsfind", - "tsfind ", - "Find tsasset table and index status for one asset id", - HandleTsFind); - - MainConsole.Instance.Commands.AddCommand( - "tsasset", - false, - "tsverify", - "tsverify [all|assets||assets_]", - "Verify tsasset table/index consistency", - HandleTsVerify); - - MainConsole.Instance.Commands.AddCommand( - "tsasset", - false, - "tsreindex", - "tsreindex [all|assets||assets_] --force [--batch=] [--timeout=]", - "Rebuild tsasset index entries for typed tables or clean legacy index entries", - HandleTsReindex); - - MainConsole.Instance.Commands.AddCommand( - "tsasset", - false, - "tscleanlegacy", - "tscleanlegacy --force [--batch=] [--timeout=]", - "Remove tsassets_index rows that still point to legacy assets table", - HandleTsCleanLegacy); - - m_commandsRegistered = true; - } - } - - private void HandleTsMove(string module, string[] args) - { - HandleTsMoveInternal(args, false); - } - - private void HandleTsShowMove(string module, string[] args) - { - HandleTsMoveInternal(args, true); - } - - private void HandleTsFind(string module, string[] args) - { - if (args == null || args.Length < 2) - { - MainConsole.Instance.Output("Syntax: tsfind "); - return; - } - - string assetId = args[1]; - List uniqueDatabases = BuildUniqueDatabases(); - - int supportedDatabases = 0; - bool foundAny = false; - - for (int i = 0; i < uniqueDatabases.Count; i++) - { - if (!(uniqueDatabases[i] is ITSAssetAdminData adminData)) - continue; - - supportedDatabases++; - - if (!adminData.TryFindAssetLocation(assetId, out TSAssetFindReport report, out string errorMessage)) - { - MainConsole.Instance.Output(string.Format("tsfind failed on database #{0}: {1}", i + 1, errorMessage)); - return; - } - - if (!report.Found) - continue; - - foundAny = true; - MainConsole.Instance.Output( - string.Format( - CultureInfo.InvariantCulture, - "tsfind {0}: table={1}, index={2}, index-type={3}", - report.AssetId, - string.IsNullOrEmpty(report.TableName) ? "" : report.TableName, - report.HasIndexEntry ? "yes" : "no", - report.HasIndexEntry ? report.IndexAssetType.ToString(CultureInfo.InvariantCulture) : "n/a")); - } - - if (supportedDatabases == 0) - { - MainConsole.Instance.Output("tsfind is not supported by the configured asset data plugin(s)"); - return; - } - - if (!foundAny) - MainConsole.Instance.Output(string.Format(CultureInfo.InvariantCulture, "tsfind {0}: not found", assetId)); - } - - private void HandleTsVerify(string module, string[] args) - { - string scope = (args != null && args.Length >= 2) ? args[1] : "all"; - - List uniqueDatabases = BuildUniqueDatabases(); - - int supportedDatabases = 0; - int totalTablesChecked = 0; - int totalRows = 0; - int totalMissingIndexRows = 0; - int totalWrongIndexTypeRows = 0; - int totalOrphanIndexRows = 0; - int totalLegacyRowsWithIndex = 0; - - for (int i = 0; i < uniqueDatabases.Count; i++) - { - if (!(uniqueDatabases[i] is ITSAssetAdminData adminData)) - continue; - - supportedDatabases++; - - if (!adminData.TryVerifyAssets(scope, out TSAssetVerifyReport report, out string errorMessage)) - { - MainConsole.Instance.Output(string.Format("tsverify failed on database #{0}: {1}", i + 1, errorMessage)); - return; - } - - totalTablesChecked += report.TablesChecked; - totalRows += report.TotalRows; - totalMissingIndexRows += report.MissingIndexRows; - totalWrongIndexTypeRows += report.WrongIndexTypeRows; - totalOrphanIndexRows += report.OrphanIndexRows; - totalLegacyRowsWithIndex += report.LegacyRowsWithIndex; - } - - if (supportedDatabases == 0) - { - MainConsole.Instance.Output("tsverify is not supported by the configured asset data plugin(s)"); - return; - } - - MainConsole.Instance.Output( - string.Format( - CultureInfo.InvariantCulture, - "tsverify {0}: tables={1}, rows={2}, missing-index={3}, wrong-index-type={4}, orphan-index={5}, legacy-with-index={6}", - scope, - totalTablesChecked, - totalRows, - totalMissingIndexRows, - totalWrongIndexTypeRows, - totalOrphanIndexRows, - totalLegacyRowsWithIndex)); - } - - private void HandleTsReindex(string module, string[] args) - { - bool hasExplicitScope = args != null && args.Length >= 2 && !args[1].StartsWith("-", StringComparison.Ordinal); - string scope = hasExplicitScope ? args[1] : "all"; - int optionsStartIndex = hasExplicitScope ? 2 : 1; - - if (!TryParseAdminWriteOptions(args, optionsStartIndex, out TSAssetMoveOptions options, out string optionError)) - { - MainConsole.Instance.Output(optionError); - return; - } - - List uniqueDatabases = BuildUniqueDatabases(); - int supportedDatabases = 0; - - TSAssetReindexReport total = new TSAssetReindexReport - { - Scope = scope, - TablesProcessed = 0, - RowsScanned = 0, - IndexRowsUpserted = 0, - IndexRowsDeleted = 0 - }; - - for (int i = 0; i < uniqueDatabases.Count; i++) - { - if (!(uniqueDatabases[i] is ITSAssetAdminData adminData)) - continue; - - supportedDatabases++; - - if (!adminData.TryReindexAssets(scope, options, out TSAssetReindexReport report, out string errorMessage)) - { - MainConsole.Instance.Output(string.Format("tsreindex failed on database #{0}: {1}", i + 1, errorMessage)); - return; - } - - total.TablesProcessed += report.TablesProcessed; - total.RowsScanned += report.RowsScanned; - total.IndexRowsUpserted += report.IndexRowsUpserted; - total.IndexRowsDeleted += report.IndexRowsDeleted; - } - - if (supportedDatabases == 0) - { - MainConsole.Instance.Output("tsreindex is not supported by the configured asset data plugin(s)"); - return; - } - - MainConsole.Instance.Output( - string.Format( - CultureInfo.InvariantCulture, - "tsreindex {0}: tables={1}, scanned={2}, index-upserted={3}, index-deleted={4}", - scope, - total.TablesProcessed, - total.RowsScanned, - total.IndexRowsUpserted, - total.IndexRowsDeleted)); - } - - private void HandleTsCleanLegacy(string module, string[] args) - { - if (!TryParseAdminWriteOptions(args, 1, out TSAssetMoveOptions options, out string optionError)) - { - MainConsole.Instance.Output(optionError); - return; - } - - List uniqueDatabases = BuildUniqueDatabases(); - int supportedDatabases = 0; - - TSAssetReindexReport total = new TSAssetReindexReport - { - Scope = "assets", - TablesProcessed = 0, - RowsScanned = 0, - IndexRowsUpserted = 0, - IndexRowsDeleted = 0 - }; - - for (int i = 0; i < uniqueDatabases.Count; i++) - { - if (!(uniqueDatabases[i] is ITSAssetAdminData adminData)) - continue; - - supportedDatabases++; - - if (!adminData.TryCleanLegacyIndex(options, out TSAssetReindexReport report, out string errorMessage)) - { - MainConsole.Instance.Output(string.Format("tscleanlegacy failed on database #{0}: {1}", i + 1, errorMessage)); - return; - } - - total.TablesProcessed += report.TablesProcessed; - total.RowsScanned += report.RowsScanned; - total.IndexRowsUpserted += report.IndexRowsUpserted; - total.IndexRowsDeleted += report.IndexRowsDeleted; - } - - if (supportedDatabases == 0) - { - MainConsole.Instance.Output("tscleanlegacy is not supported by the configured asset data plugin(s)"); - return; - } - - MainConsole.Instance.Output( - string.Format( - CultureInfo.InvariantCulture, - "tscleanlegacy: tables={0}, scanned={1}, index-deleted={2}", - total.TablesProcessed, - total.RowsScanned, - total.IndexRowsDeleted)); - } - - private bool TryParseAdminWriteOptions(string[] args, int startIndex, out TSAssetMoveOptions options, out string error) - { - options = new TSAssetMoveOptions - { - ResetCheckpoint = false, - BatchSize = 0, - CommandTimeoutSeconds = 0 - }; - - error = string.Empty; - bool hasForce = false; - - if (args == null) - { - error = "Missing command arguments"; - return false; - } - - for (int argIndex = startIndex; argIndex < args.Length; argIndex++) - { - string flag = args[argIndex]; - - if (flag.Equals("--force", StringComparison.OrdinalIgnoreCase) || - flag.Equals("-f", StringComparison.OrdinalIgnoreCase)) - { - hasForce = true; - continue; - } - - if (flag.Equals("--reset", StringComparison.OrdinalIgnoreCase)) - { - options.ResetCheckpoint = true; - continue; - } - - if (flag.StartsWith("--batch=", StringComparison.OrdinalIgnoreCase)) - { - string value = flag.Substring("--batch=".Length).Trim(); - if (!int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out int batchSize) || batchSize <= 0) - { - error = "Invalid --batch value. Example: --batch=2000"; - return false; - } - - options.BatchSize = batchSize; - continue; - } - - if (flag.StartsWith("--timeout=", StringComparison.OrdinalIgnoreCase)) - { - string value = flag.Substring("--timeout=".Length).Trim(); - if (!int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out int timeoutSeconds) || timeoutSeconds <= 0) - { - error = "Invalid --timeout value. Example: --timeout=600"; - return false; - } - - options.CommandTimeoutSeconds = timeoutSeconds; - continue; - } - } - - if (!hasForce) - { - error = "Write command aborted: missing --force flag"; - return false; - } - - return true; - } - - private List BuildUniqueDatabases() - { - List uniqueDatabases = new List(); - for (int i = 0; i < m_probeOrder.Count; i++) - { - IAssetDataPlugin db = m_probeOrder[i]; - bool alreadyAdded = false; - - for (int j = 0; j < uniqueDatabases.Count; j++) - { - if (object.ReferenceEquals(uniqueDatabases[j], db)) - { - alreadyAdded = true; - break; - } - } - - if (!alreadyAdded) - uniqueDatabases.Add(db); - } - - return uniqueDatabases; - } - - private void HandleTsMoveInternal(string[] args, bool previewOnly) - { - if (args == null || args.Length < 3) - { - MainConsole.Instance.Output(previewOnly ? "Syntax: tsshowmove " : "Syntax: tsmove --force"); - MainConsole.Instance.Output("Examples: tsmove assets 7 | tsmove 7 assets | tsmove assets_7 assets"); - return; - } - - string from = args[1]; - string to = args[2]; - TSAssetMoveOptions moveOptions = new TSAssetMoveOptions - { - ResetCheckpoint = false, - BatchSize = 0, - CommandTimeoutSeconds = 0 - }; - - if (!previewOnly) - { - if (!TryParseAdminWriteOptions(args, 3, out moveOptions, out string optionError)) - { - MainConsole.Instance.Output(optionError.Replace("Write command", "tsmove")); - MainConsole.Instance.Output(string.Format(CultureInfo.InvariantCulture, "Preview first: tsshowmove {0} {1}", from, to)); - MainConsole.Instance.Output(string.Format(CultureInfo.InvariantCulture, "Execute move: tsmove {0} {1} --force", from, to)); - return; - } - } - - List uniqueDatabases = BuildUniqueDatabases(); - - int supportedDatabases = 0; - int totalCandidates = 0; - int totalAlreadyInTarget = 0; - int totalInserted = 0; - int totalDeletedFromSource = 0; - int totalIndexAffected = 0; - - for (int i = 0; i < uniqueDatabases.Count; i++) - { - if (!(uniqueDatabases[i] is ITSAssetAdminData adminData)) - continue; - - supportedDatabases++; - - bool ok; - TSAssetMoveReport report; - string errorMessage; - - if (previewOnly) - ok = adminData.TryPreviewMoveAssets(from, to, out report, out errorMessage); - else - ok = adminData.TryMoveAssets(from, to, moveOptions, out report, out errorMessage); - - if (!ok) - { - MainConsole.Instance.Output(string.Format("{0} failed on database #{1}: {2}", previewOnly ? "tsshowmove" : "tsmove", i + 1, errorMessage)); - return; - } - - totalCandidates += report.CandidateCount; - totalAlreadyInTarget += report.AlreadyInTargetCount; - totalInserted += report.InsertedCount; - totalDeletedFromSource += report.DeletedFromSourceCount; - totalIndexAffected += report.IndexAffectedCount; - } - - if (supportedDatabases == 0) - { - MainConsole.Instance.Output(string.Format("{0} is not supported by the configured asset data plugin(s)", previewOnly ? "tsshowmove" : "tsmove")); - return; - } - - MainConsole.Instance.Output( - string.Format( - CultureInfo.InvariantCulture, - "{0} {1} -> {2}: candidates={3}, inserted={4}, already-in-target={5}, deleted-from-source={6}, index-affected={7}", - previewOnly ? "tsshowmove" : "tsmove", - from, - to, - totalCandidates, - totalInserted, - totalAlreadyInTarget, - totalDeletedFromSource, - totalIndexAffected)); - } - } -} diff --git a/opensim/Services/AssetService/TSAssetConnector.cs b/opensim/Services/AssetService/TSAssetConnector.cs deleted file mode 100644 index 81e0fe9..0000000 --- a/opensim/Services/AssetService/TSAssetConnector.cs +++ /dev/null @@ -1,1182 +0,0 @@ -/* - * Copyright (c) Contributors, http://opensimulator.org/ - * See CONTRIBUTORS.TXT for a full list of copyright holders. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the OpenSimulator Project nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -using System; -using System.Collections.Generic; -using System.Collections.Concurrent; -using System.Globalization; -using System.Reflection; -using System.Threading; -using log4net; -using Nini.Config; -using OpenMetaverse; -using OpenSim.Data; -using OpenSim.Framework; -using OpenSim.Services.Base; -using OpenSim.Services.Interfaces; - -namespace OpenSim.Services.AssetService -{ - public class TSAssetConnector : ServiceBase, IAssetService - { - private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - private static readonly object m_commandLock = new object(); - private static bool m_commandsRegistered; - - private readonly Dictionary m_typeDatabases = new Dictionary(); - private readonly Dictionary m_connectionDatabases = new Dictionary(StringComparer.OrdinalIgnoreCase); - private readonly Dictionary m_assetLocationCache = new Dictionary(); - private readonly ConcurrentQueue m_fallbackMigrationQueue = new ConcurrentQueue(); - private readonly ConcurrentDictionary m_fallbackMigrationSet = new ConcurrentDictionary(); - private readonly object m_cacheLock = new object(); - private readonly HashSet m_allowedTypes = new HashSet(); - - private readonly List m_probeOrder = new List(); - - private IAssetDataPlugin m_defaultDatabase; - private IAssetLoader m_assetLoader; - private IAssetService m_fallbackService; - private Thread m_fallbackMigrationThread; - - private bool m_enableFallbackAutoMigration; - private bool m_enableFallbackAutoDelete; - private int m_migrationCheckIntervalMs = 60000; - private int m_migrationBatchSize = 25; - private int m_migrationLowTrafficMaxRequests = 3; - private int m_migrationQueueMax = 50000; - private int m_requestsInWindow; - - public TSAssetConnector(IConfigSource config) - : base(config) - { - IConfig assetConfig = config.Configs["AssetService"]; - if (assetConfig == null) - throw new Exception("No AssetService configuration"); - - IConfig tsConfig = config.Configs["TSAssetService"]; - IConfig dbConfig = config.Configs["DatabaseService"]; - - if (tsConfig == null || !tsConfig.GetBoolean("Enabled", false)) - throw new Exception("TSAssetConnector disabled. Set [TSAssetService] Enabled = true to enable."); - - string storageProvider = string.Empty; - string defaultConnectionString = string.Empty; - - if (tsConfig != null) - { - storageProvider = tsConfig.GetString("StorageProvider", string.Empty); - defaultConnectionString = tsConfig.GetString("ConnectionString", string.Empty); - } - - if (string.IsNullOrEmpty(storageProvider)) - storageProvider = assetConfig.GetString("StorageProvider", string.Empty); - - if (string.IsNullOrEmpty(defaultConnectionString)) - defaultConnectionString = assetConfig.GetString("ConnectionString", string.Empty); - - if (dbConfig != null) - { - if (string.IsNullOrEmpty(storageProvider)) - storageProvider = dbConfig.GetString("StorageProvider", string.Empty); - - if (string.IsNullOrEmpty(defaultConnectionString)) - defaultConnectionString = dbConfig.GetString("ConnectionString", string.Empty); - } - - if (string.IsNullOrEmpty(storageProvider)) - throw new Exception("No StorageProvider configured"); - - if (string.IsNullOrEmpty(defaultConnectionString)) - throw new Exception("Missing database connection string"); - - RegisterConsoleCommands(); - - ParseAllowedTypes(tsConfig); - - if (tsConfig != null) - { - m_enableFallbackAutoMigration = tsConfig.GetBoolean("EnableFallbackAutoMigration", false); - m_enableFallbackAutoDelete = tsConfig.GetBoolean("EnableFallbackAutoDelete", false); - m_migrationCheckIntervalMs = Math.Max(5000, tsConfig.GetInt("MigrationCheckIntervalSeconds", 60) * 1000); - m_migrationBatchSize = Math.Max(1, tsConfig.GetInt("MigrationBatchSize", 25)); - m_migrationLowTrafficMaxRequests = Math.Max(0, tsConfig.GetInt("MigrationLowTrafficMaxRequests", 3)); - m_migrationQueueMax = Math.Max(100, tsConfig.GetInt("MigrationQueueMax", 50000)); - } - - m_defaultDatabase = CreateAndInitDatabase(storageProvider, defaultConnectionString); - AddProbeDatabase(m_defaultDatabase); - - if (tsConfig != null) - { - ParseTypedDatabaseMappings(tsConfig.GetString("AssetDatabases", string.Empty), storageProvider); - ParseTypedDatabaseMappingsFromKeys(tsConfig, storageProvider); - } - - string loaderName = assetConfig.GetString("DefaultAssetLoader", string.Empty); - if (!string.IsNullOrEmpty(loaderName)) - { - m_assetLoader = LoadPlugin(loaderName); - if (m_assetLoader == null) - throw new Exception(string.Format("Asset loader could not be loaded from {0}", loaderName)); - - bool assetLoaderEnabled = assetConfig.GetBoolean("AssetLoaderEnabled", true); - if (assetLoaderEnabled) - { - string loaderArgs = assetConfig.GetString("AssetLoaderArgs", string.Empty); - m_log.InfoFormat("[TSASSET SERVICE]: Loading default asset set from {0}", loaderArgs); - - m_assetLoader.ForEachDefaultXmlAsset( - loaderArgs, - delegate(AssetBase a) - { - if (a == null) - return; - - if (Get(a.ID) == null) - Store(a); - }); - } - } - - string fallbackServiceName = assetConfig.GetString("FallbackService", string.Empty); - if (!string.IsNullOrEmpty(fallbackServiceName)) - { - object[] args = new object[] { config }; - m_fallbackService = LoadPlugin(fallbackServiceName, args); - if (m_fallbackService != null) - { - m_log.Info("[TSASSET SERVICE]: Fallback service loaded"); - - if (m_enableFallbackAutoMigration) - { - m_fallbackMigrationThread = new Thread(FallbackMigrationWorker); - m_fallbackMigrationThread.IsBackground = true; - m_fallbackMigrationThread.Name = "TSAssetFallbackMigration"; - m_fallbackMigrationThread.Start(); - m_log.Info("[TSASSET SERVICE]: Fallback auto-migration worker enabled"); - } - } - else - m_log.Error("[TSASSET SERVICE]: Failed to load fallback service"); - } - - m_log.InfoFormat("[TSASSET SERVICE]: Enabled with {0} type routes", m_typeDatabases.Count); - } - - public TSAssetConnector(IConfigSource config, string configName) - : this(config) - { - } - - public AssetBase Get(string id) - { - Interlocked.Increment(ref m_requestsInWindow); - - if (!UUID.TryParse(id, out UUID assetID)) - { - m_log.WarnFormat("[TSASSET SERVICE]: Could not parse requested asset id {0}", id); - return null; - } - - IAssetDataPlugin cachedDatabase = null; - lock (m_cacheLock) - { - m_assetLocationCache.TryGetValue(assetID, out cachedDatabase); - } - - if (cachedDatabase != null) - { - try - { - AssetBase cachedAsset = cachedDatabase.GetAsset(assetID); - if (cachedAsset != null) - return cachedAsset; - } - catch (Exception e) - { - m_log.WarnFormat("[TSASSET SERVICE]: Cached database lookup failed for asset {0}: {1}", assetID, e.Message); - } - } - - for (int i = 0; i < m_probeOrder.Count; i++) - { - IAssetDataPlugin db = m_probeOrder[i]; - - try - { - AssetBase asset = db.GetAsset(assetID); - if (asset != null) - { - lock (m_cacheLock) - { - m_assetLocationCache[assetID] = db; - } - return asset; - } - } - catch (Exception e) - { - m_log.WarnFormat("[TSASSET SERVICE]: Asset lookup failed for asset {0}: {1}", assetID, e.Message); - } - } - - if (m_fallbackService != null) - { - try - { - AssetBase fallbackAsset = m_fallbackService.Get(id); - if (fallbackAsset != null) - { - string storedId = Store(fallbackAsset); - bool storeOk = !storedId.Equals(UUID.Zero.ToString(), StringComparison.Ordinal); - - if (storeOk && m_enableFallbackAutoDelete) - TryDeleteFromFallback(id); - - if (m_enableFallbackAutoMigration && !storeOk) - EnqueueFallbackMigration(fallbackAsset); - - return fallbackAsset; - } - } - catch (Exception e) - { - m_log.WarnFormat("[TSASSET SERVICE]: Fallback lookup failed for asset {0}: {1}", id, e.Message); - } - } - - return null; - } - - public AssetBase Get(string id, string ForeignAssetService, bool StoreOnLocalGrid) - { - return Get(id); - } - - public AssetMetadata GetMetadata(string id) - { - AssetBase asset = Get(id); - return asset != null ? asset.Metadata : null; - } - - public byte[] GetData(string id) - { - AssetBase asset = Get(id); - return asset != null ? asset.Data : null; - } - - public AssetBase GetCached(string id) - { - return Get(id); - } - - public bool Get(string id, object sender, AssetRetrieved handler) - { - handler(id, sender, Get(id)); - return true; - } - - public void Get(string id, string ForeignAssetService, bool StoreOnLocalGrid, SimpleAssetRetrieved callBack) - { - callBack(Get(id)); - } - - public bool[] AssetsExist(string[] ids) - { - if (ids == null || ids.Length == 0) - return new bool[0]; - - bool[] results = new bool[ids.Length]; - UUID[] uuids = new UUID[ids.Length]; - bool[] valid = new bool[ids.Length]; - - for (int i = 0; i < ids.Length; i++) - { - if (UUID.TryParse(ids[i], out UUID parsed)) - { - uuids[i] = parsed; - valid[i] = true; - } - } - - for (int dbIndex = 0; dbIndex < m_probeOrder.Count; dbIndex++) - { - IAssetDataPlugin db = m_probeOrder[dbIndex]; - UUID[] query = BuildPendingUuidList(uuids, valid, results); - - if (query.Length == 0) - break; - - try - { - bool[] exists = db.AssetsExist(query); - MergeExistenceResults(query, exists, uuids, valid, results, db); - } - catch (Exception e) - { - m_log.WarnFormat("[TSASSET SERVICE]: AssetsExist failed on a database: {0}", e.Message); - } - } - - return results; - } - - public string Store(AssetBase asset) - { - if (asset == null) - return UUID.Zero.ToString(); - - IAssetDataPlugin db = ResolveDatabaseForType(asset.Type); - if (db == null) - { - m_log.WarnFormat("[TSASSET SERVICE]: No database resolved for asset {0}, type {1}", asset.FullID, asset.Type); - return UUID.Zero.ToString(); - } - - try - { - bool stored = db.StoreAsset(asset); - if (!stored) - return UUID.Zero.ToString(); - - lock (m_cacheLock) - { - m_assetLocationCache[asset.FullID] = db; - } - - return asset.ID; - } - catch (Exception e) - { - m_log.ErrorFormat("[TSASSET SERVICE]: Error storing asset {0}: {1}", asset.FullID, e.Message); - return UUID.Zero.ToString(); - } - } - - public bool UpdateContent(string id, byte[] data) - { - AssetBase asset = Get(id); - if (asset == null) - return false; - - asset.Data = data; - return Store(asset) != UUID.Zero.ToString(); - } - - public bool Delete(string id) - { - if (!UUID.TryParse(id, out UUID assetID)) - return false; - - IAssetDataPlugin cachedDatabase = null; - lock (m_cacheLock) - { - m_assetLocationCache.TryGetValue(assetID, out cachedDatabase); - } - - if (cachedDatabase != null) - { - try - { - bool deleted = cachedDatabase.Delete(id); - if (deleted) - { - lock (m_cacheLock) - { - m_assetLocationCache.Remove(assetID); - } - return true; - } - } - catch (Exception e) - { - m_log.WarnFormat("[TSASSET SERVICE]: Cached delete failed for asset {0}: {1}", assetID, e.Message); - } - } - - for (int i = 0; i < m_probeOrder.Count; i++) - { - try - { - if (m_probeOrder[i].Delete(id)) - { - lock (m_cacheLock) - { - m_assetLocationCache.Remove(assetID); - } - return true; - } - } - catch (Exception e) - { - m_log.WarnFormat("[TSASSET SERVICE]: Delete failed for asset {0}: {1}", assetID, e.Message); - } - } - - return false; - } - - private static UUID[] BuildPendingUuidList(UUID[] uuids, bool[] valid, bool[] results) - { - List pending = new List(uuids.Length); - for (int i = 0; i < uuids.Length; i++) - { - if (valid[i] && !results[i]) - pending.Add(uuids[i]); - } - - return pending.ToArray(); - } - - private void MergeExistenceResults(UUID[] query, bool[] exists, UUID[] sourceIds, bool[] valid, bool[] results, IAssetDataPlugin db) - { - if (exists == null) - return; - - int len = Math.Min(query.Length, exists.Length); - for (int i = 0; i < len; i++) - { - if (!exists[i]) - continue; - - UUID foundId = query[i]; - - for (int sourceIndex = 0; sourceIndex < sourceIds.Length; sourceIndex++) - { - if (valid[sourceIndex] && sourceIds[sourceIndex] == foundId) - { - results[sourceIndex] = true; - } - } - - lock (m_cacheLock) - { - m_assetLocationCache[foundId] = db; - } - } - } - - private void ParseAllowedTypes(IConfig tsConfig) - { - if (tsConfig == null) - return; - - string raw = tsConfig.GetString("TSAssetType", string.Empty); - if (string.IsNullOrWhiteSpace(raw)) - return; - - string[] tokens = raw.Split(new char[] { ',', ';', ' ', '\t', '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); - for (int i = 0; i < tokens.Length; i++) - { - if (sbyte.TryParse(tokens[i], NumberStyles.Integer, CultureInfo.InvariantCulture, out sbyte type)) - m_allowedTypes.Add(type); - else - m_log.WarnFormat("[TSASSET SERVICE]: Ignoring invalid TSAssetType entry '{0}'", tokens[i]); - } - } - - private void ParseTypedDatabaseMappings(string rawMappings, string storageProvider) - { - if (string.IsNullOrWhiteSpace(rawMappings)) - return; - - string[] lines = rawMappings.Split(new char[] { '\r', '\n', '|' }, StringSplitOptions.RemoveEmptyEntries); - for (int i = 0; i < lines.Length; i++) - { - string line = lines[i].Trim(); - if (string.IsNullOrEmpty(line)) - continue; - - line = line.Trim('"'); - if (line.EndsWith(";;", StringComparison.Ordinal)) - line = line.Substring(0, line.Length - 2); - else - line = line.TrimEnd(';'); - - int sep = line.IndexOf(':'); - if (sep <= 0 || sep == line.Length - 1) - continue; - - string typeToken = line.Substring(0, sep).Trim(); - string connectionString = line.Substring(sep + 1).Trim(); - - if (!sbyte.TryParse(typeToken, NumberStyles.Integer, CultureInfo.InvariantCulture, out sbyte assetType)) - { - m_log.WarnFormat("[TSASSET SERVICE]: Ignoring invalid asset type mapping '{0}'", typeToken); - continue; - } - - if (m_allowedTypes.Count > 0 && !m_allowedTypes.Contains(assetType)) - continue; - - if (string.IsNullOrEmpty(connectionString)) - { - m_log.WarnFormat("[TSASSET SERVICE]: Ignoring empty connection string for asset type {0}", assetType); - continue; - } - - IAssetDataPlugin db = GetOrCreateDatabase(storageProvider, connectionString); - m_typeDatabases[assetType] = db; - AddProbeDatabase(db); - } - } - - private void ParseTypedDatabaseMappingsFromKeys(IConfig tsConfig, string storageProvider) - { - if (tsConfig == null) - return; - - string[] keys = tsConfig.GetKeys(); - if (keys == null || keys.Length == 0) - return; - - for (int i = 0; i < keys.Length; i++) - { - string key = keys[i]; - if (string.IsNullOrWhiteSpace(key)) - continue; - - string prefix = null; - if (key.StartsWith("AssetDatabase_", StringComparison.OrdinalIgnoreCase)) - prefix = "AssetDatabase_"; - else if (key.StartsWith("AssetDatabase.", StringComparison.OrdinalIgnoreCase)) - prefix = "AssetDatabase."; - - if (prefix == null) - continue; - - string typeToken = key.Substring(prefix.Length).Trim(); - if (!sbyte.TryParse(typeToken, NumberStyles.Integer, CultureInfo.InvariantCulture, out sbyte assetType)) - { - m_log.WarnFormat("[TSASSET SERVICE]: Ignoring invalid asset type key '{0}'", key); - continue; - } - - if (m_allowedTypes.Count > 0 && !m_allowedTypes.Contains(assetType)) - continue; - - string connectionString = tsConfig.GetString(key, string.Empty).Trim(); - if (connectionString.Length >= 2 && - connectionString[0] == '"' && - connectionString[connectionString.Length - 1] == '"') - { - connectionString = connectionString.Substring(1, connectionString.Length - 2).Trim(); - } - - if (string.IsNullOrEmpty(connectionString)) - { - m_log.WarnFormat("[TSASSET SERVICE]: Ignoring empty connection string for asset type key '{0}'", key); - continue; - } - - IAssetDataPlugin db = GetOrCreateDatabase(storageProvider, connectionString); - m_typeDatabases[assetType] = db; - AddProbeDatabase(db); - } - } - - private IAssetDataPlugin ResolveDatabaseForType(sbyte type) - { - if (m_typeDatabases.TryGetValue(type, out IAssetDataPlugin db)) - return db; - - return m_defaultDatabase; - } - - private IAssetDataPlugin GetOrCreateDatabase(string storageProvider, string connectionString) - { - if (m_connectionDatabases.TryGetValue(connectionString, out IAssetDataPlugin existing)) - return existing; - - IAssetDataPlugin created = CreateAndInitDatabase(storageProvider, connectionString); - m_connectionDatabases[connectionString] = created; - return created; - } - - private IAssetDataPlugin CreateAndInitDatabase(string storageProvider, string connectionString) - { - IAssetDataPlugin database = LoadPlugin(storageProvider); - if (database == null) - throw new Exception(string.Format("Could not find a storage interface in the module {0}", storageProvider)); - - database.Initialise(connectionString); - return database; - } - - private void AddProbeDatabase(IAssetDataPlugin db) - { - for (int i = 0; i < m_probeOrder.Count; i++) - { - if (object.ReferenceEquals(m_probeOrder[i], db)) - return; - } - - m_probeOrder.Add(db); - } - - private void EnqueueFallbackMigration(AssetBase asset) - { - if (asset == null) - return; - - if (m_fallbackMigrationSet.Count >= m_migrationQueueMax) - return; - - if (m_fallbackMigrationSet.TryAdd(asset.FullID, 0)) - m_fallbackMigrationQueue.Enqueue(asset); - } - - private void FallbackMigrationWorker() - { - while (true) - { - Thread.Sleep(m_migrationCheckIntervalMs); - - int requests = Interlocked.Exchange(ref m_requestsInWindow, 0); - if (requests > m_migrationLowTrafficMaxRequests) - continue; - - int migrated = 0; - while (migrated < m_migrationBatchSize && m_fallbackMigrationQueue.TryDequeue(out AssetBase asset)) - { - try - { - string id = Store(asset); - if (!id.Equals(UUID.Zero.ToString(), StringComparison.Ordinal)) - { - if (m_enableFallbackAutoDelete) - TryDeleteFromFallback(asset.ID); - - migrated++; - } - } - catch (Exception e) - { - m_log.WarnFormat("[TSASSET SERVICE]: Auto-migration failed for asset {0}: {1}", asset != null ? asset.FullID.ToString() : "", e.Message); - } - finally - { - if (asset != null) - m_fallbackMigrationSet.TryRemove(asset.FullID, out _); - } - } - - if (migrated > 0) - m_log.InfoFormat("[TSASSET SERVICE]: Auto-migrated {0} fallback assets during low traffic", migrated); - } - } - - private void TryDeleteFromFallback(string id) - { - if (m_fallbackService == null || string.IsNullOrEmpty(id)) - return; - - try - { - if (!m_fallbackService.Delete(id)) - m_log.WarnFormat("[TSASSET SERVICE]: Fallback auto-delete failed for asset {0}", id); - } - catch (Exception e) - { - m_log.WarnFormat("[TSASSET SERVICE]: Fallback auto-delete exception for asset {0}: {1}", id, e.Message); - } - } - - private void RegisterConsoleCommands() - { - if (MainConsole.Instance == null) - return; - - lock (m_commandLock) - { - if (m_commandsRegistered) - return; - - MainConsole.Instance.Commands.AddCommand( - "tsasset", - false, - "tsmove", - "tsmove --force [--reset] [--batch=] [--timeout=]", - "Move tsasset rows between tables (requires --force). Supports resume/reset and batch/timeout overrides", - HandleTsMove); - - MainConsole.Instance.Commands.AddCommand( - "tsasset", - false, - "tsshowmove", - "tsshowmove ", - "Preview tsasset move counts without writing changes", - HandleTsShowMove); - - MainConsole.Instance.Commands.AddCommand( - "tsasset", - false, - "tsfind", - "tsfind ", - "Find tsasset table and index status for one asset id", - HandleTsFind); - - MainConsole.Instance.Commands.AddCommand( - "tsasset", - false, - "tsverify", - "tsverify [all|assets||assets_]", - "Verify tsasset table/index consistency", - HandleTsVerify); - - MainConsole.Instance.Commands.AddCommand( - "tsasset", - false, - "tsreindex", - "tsreindex [all|assets||assets_] --force [--batch=] [--timeout=]", - "Rebuild tsasset index entries for typed tables or clean legacy index entries", - HandleTsReindex); - - MainConsole.Instance.Commands.AddCommand( - "tsasset", - false, - "tscleanlegacy", - "tscleanlegacy --force [--batch=] [--timeout=]", - "Remove tsassets_index rows that still point to legacy assets table", - HandleTsCleanLegacy); - - m_commandsRegistered = true; - } - } - - private void HandleTsMove(string module, string[] args) - { - HandleTsMoveInternal(args, false); - } - - private void HandleTsShowMove(string module, string[] args) - { - HandleTsMoveInternal(args, true); - } - - private void HandleTsFind(string module, string[] args) - { - if (args == null || args.Length < 2) - { - MainConsole.Instance.Output("Syntax: tsfind "); - return; - } - - string assetId = args[1]; - List uniqueDatabases = BuildUniqueDatabases(); - - int supportedDatabases = 0; - bool foundAny = false; - - for (int i = 0; i < uniqueDatabases.Count; i++) - { - if (!(uniqueDatabases[i] is ITSAssetAdminData adminData)) - continue; - - supportedDatabases++; - - if (!adminData.TryFindAssetLocation(assetId, out TSAssetFindReport report, out string errorMessage)) - { - MainConsole.Instance.Output(string.Format("tsfind failed on database #{0}: {1}", i + 1, errorMessage)); - return; - } - - if (!report.Found) - continue; - - foundAny = true; - MainConsole.Instance.Output( - string.Format( - CultureInfo.InvariantCulture, - "tsfind {0}: table={1}, index={2}, index-type={3}", - report.AssetId, - string.IsNullOrEmpty(report.TableName) ? "" : report.TableName, - report.HasIndexEntry ? "yes" : "no", - report.HasIndexEntry ? report.IndexAssetType.ToString(CultureInfo.InvariantCulture) : "n/a")); - } - - if (supportedDatabases == 0) - { - MainConsole.Instance.Output("tsfind is not supported by the configured asset data plugin(s)"); - return; - } - - if (!foundAny) - MainConsole.Instance.Output(string.Format(CultureInfo.InvariantCulture, "tsfind {0}: not found", assetId)); - } - - private void HandleTsVerify(string module, string[] args) - { - string scope = (args != null && args.Length >= 2) ? args[1] : "all"; - - List uniqueDatabases = BuildUniqueDatabases(); - - int supportedDatabases = 0; - int totalTablesChecked = 0; - int totalRows = 0; - int totalMissingIndexRows = 0; - int totalWrongIndexTypeRows = 0; - int totalOrphanIndexRows = 0; - int totalLegacyRowsWithIndex = 0; - - for (int i = 0; i < uniqueDatabases.Count; i++) - { - if (!(uniqueDatabases[i] is ITSAssetAdminData adminData)) - continue; - - supportedDatabases++; - - if (!adminData.TryVerifyAssets(scope, out TSAssetVerifyReport report, out string errorMessage)) - { - MainConsole.Instance.Output(string.Format("tsverify failed on database #{0}: {1}", i + 1, errorMessage)); - return; - } - - totalTablesChecked += report.TablesChecked; - totalRows += report.TotalRows; - totalMissingIndexRows += report.MissingIndexRows; - totalWrongIndexTypeRows += report.WrongIndexTypeRows; - totalOrphanIndexRows += report.OrphanIndexRows; - totalLegacyRowsWithIndex += report.LegacyRowsWithIndex; - } - - if (supportedDatabases == 0) - { - MainConsole.Instance.Output("tsverify is not supported by the configured asset data plugin(s)"); - return; - } - - MainConsole.Instance.Output( - string.Format( - CultureInfo.InvariantCulture, - "tsverify {0}: tables={1}, rows={2}, missing-index={3}, wrong-index-type={4}, orphan-index={5}, legacy-with-index={6}", - scope, - totalTablesChecked, - totalRows, - totalMissingIndexRows, - totalWrongIndexTypeRows, - totalOrphanIndexRows, - totalLegacyRowsWithIndex)); - } - - private void HandleTsReindex(string module, string[] args) - { - bool hasExplicitScope = args != null && args.Length >= 2 && !args[1].StartsWith("-", StringComparison.Ordinal); - string scope = hasExplicitScope ? args[1] : "all"; - int optionsStartIndex = hasExplicitScope ? 2 : 1; - - if (!TryParseAdminWriteOptions(args, optionsStartIndex, out TSAssetMoveOptions options, out string optionError)) - { - MainConsole.Instance.Output(optionError); - return; - } - - List uniqueDatabases = BuildUniqueDatabases(); - int supportedDatabases = 0; - - TSAssetReindexReport total = new TSAssetReindexReport - { - Scope = scope, - TablesProcessed = 0, - RowsScanned = 0, - IndexRowsUpserted = 0, - IndexRowsDeleted = 0 - }; - - for (int i = 0; i < uniqueDatabases.Count; i++) - { - if (!(uniqueDatabases[i] is ITSAssetAdminData adminData)) - continue; - - supportedDatabases++; - - if (!adminData.TryReindexAssets(scope, options, out TSAssetReindexReport report, out string errorMessage)) - { - MainConsole.Instance.Output(string.Format("tsreindex failed on database #{0}: {1}", i + 1, errorMessage)); - return; - } - - total.TablesProcessed += report.TablesProcessed; - total.RowsScanned += report.RowsScanned; - total.IndexRowsUpserted += report.IndexRowsUpserted; - total.IndexRowsDeleted += report.IndexRowsDeleted; - } - - if (supportedDatabases == 0) - { - MainConsole.Instance.Output("tsreindex is not supported by the configured asset data plugin(s)"); - return; - } - - MainConsole.Instance.Output( - string.Format( - CultureInfo.InvariantCulture, - "tsreindex {0}: tables={1}, scanned={2}, index-upserted={3}, index-deleted={4}", - scope, - total.TablesProcessed, - total.RowsScanned, - total.IndexRowsUpserted, - total.IndexRowsDeleted)); - } - - private void HandleTsCleanLegacy(string module, string[] args) - { - if (!TryParseAdminWriteOptions(args, 1, out TSAssetMoveOptions options, out string optionError)) - { - MainConsole.Instance.Output(optionError); - return; - } - - List uniqueDatabases = BuildUniqueDatabases(); - int supportedDatabases = 0; - - TSAssetReindexReport total = new TSAssetReindexReport - { - Scope = "assets", - TablesProcessed = 0, - RowsScanned = 0, - IndexRowsUpserted = 0, - IndexRowsDeleted = 0 - }; - - for (int i = 0; i < uniqueDatabases.Count; i++) - { - if (!(uniqueDatabases[i] is ITSAssetAdminData adminData)) - continue; - - supportedDatabases++; - - if (!adminData.TryCleanLegacyIndex(options, out TSAssetReindexReport report, out string errorMessage)) - { - MainConsole.Instance.Output(string.Format("tscleanlegacy failed on database #{0}: {1}", i + 1, errorMessage)); - return; - } - - total.TablesProcessed += report.TablesProcessed; - total.RowsScanned += report.RowsScanned; - total.IndexRowsUpserted += report.IndexRowsUpserted; - total.IndexRowsDeleted += report.IndexRowsDeleted; - } - - if (supportedDatabases == 0) - { - MainConsole.Instance.Output("tscleanlegacy is not supported by the configured asset data plugin(s)"); - return; - } - - MainConsole.Instance.Output( - string.Format( - CultureInfo.InvariantCulture, - "tscleanlegacy: tables={0}, scanned={1}, index-deleted={2}", - total.TablesProcessed, - total.RowsScanned, - total.IndexRowsDeleted)); - } - - private bool TryParseAdminWriteOptions(string[] args, int startIndex, out TSAssetMoveOptions options, out string error) - { - options = new TSAssetMoveOptions - { - ResetCheckpoint = false, - BatchSize = 0, - CommandTimeoutSeconds = 0 - }; - - error = string.Empty; - bool hasForce = false; - - if (args == null) - { - error = "Missing command arguments"; - return false; - } - - for (int argIndex = startIndex; argIndex < args.Length; argIndex++) - { - string flag = args[argIndex]; - - if (flag.Equals("--force", StringComparison.OrdinalIgnoreCase) || - flag.Equals("-f", StringComparison.OrdinalIgnoreCase)) - { - hasForce = true; - continue; - } - - if (flag.Equals("--reset", StringComparison.OrdinalIgnoreCase)) - { - options.ResetCheckpoint = true; - continue; - } - - if (flag.StartsWith("--batch=", StringComparison.OrdinalIgnoreCase)) - { - string value = flag.Substring("--batch=".Length).Trim(); - if (!int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out int batchSize) || batchSize <= 0) - { - error = "Invalid --batch value. Example: --batch=2000"; - return false; - } - - options.BatchSize = batchSize; - continue; - } - - if (flag.StartsWith("--timeout=", StringComparison.OrdinalIgnoreCase)) - { - string value = flag.Substring("--timeout=".Length).Trim(); - if (!int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out int timeoutSeconds) || timeoutSeconds <= 0) - { - error = "Invalid --timeout value. Example: --timeout=600"; - return false; - } - - options.CommandTimeoutSeconds = timeoutSeconds; - continue; - } - } - - if (!hasForce) - { - error = "Write command aborted: missing --force flag"; - return false; - } - - return true; - } - - private List BuildUniqueDatabases() - { - List uniqueDatabases = new List(); - for (int i = 0; i < m_probeOrder.Count; i++) - { - IAssetDataPlugin db = m_probeOrder[i]; - bool alreadyAdded = false; - - for (int j = 0; j < uniqueDatabases.Count; j++) - { - if (object.ReferenceEquals(uniqueDatabases[j], db)) - { - alreadyAdded = true; - break; - } - } - - if (!alreadyAdded) - uniqueDatabases.Add(db); - } - - return uniqueDatabases; - } - - private void HandleTsMoveInternal(string[] args, bool previewOnly) - { - if (args == null || args.Length < 3) - { - MainConsole.Instance.Output(previewOnly ? "Syntax: tsshowmove " : "Syntax: tsmove --force"); - MainConsole.Instance.Output("Examples: tsmove assets 7 | tsmove 7 assets | tsmove assets_7 assets"); - return; - } - - string from = args[1]; - string to = args[2]; - TSAssetMoveOptions moveOptions = new TSAssetMoveOptions - { - ResetCheckpoint = false, - BatchSize = 0, - CommandTimeoutSeconds = 0 - }; - - if (!previewOnly) - { - if (!TryParseAdminWriteOptions(args, 3, out moveOptions, out string optionError)) - { - MainConsole.Instance.Output(optionError.Replace("Write command", "tsmove")); - MainConsole.Instance.Output(string.Format(CultureInfo.InvariantCulture, "Preview first: tsshowmove {0} {1}", from, to)); - MainConsole.Instance.Output(string.Format(CultureInfo.InvariantCulture, "Execute move: tsmove {0} {1} --force", from, to)); - return; - } - } - - List uniqueDatabases = BuildUniqueDatabases(); - - int supportedDatabases = 0; - int totalCandidates = 0; - int totalAlreadyInTarget = 0; - int totalInserted = 0; - int totalDeletedFromSource = 0; - int totalIndexAffected = 0; - - for (int i = 0; i < uniqueDatabases.Count; i++) - { - if (!(uniqueDatabases[i] is ITSAssetAdminData adminData)) - continue; - - supportedDatabases++; - - bool ok; - TSAssetMoveReport report; - string errorMessage; - - if (previewOnly) - ok = adminData.TryPreviewMoveAssets(from, to, out report, out errorMessage); - else - ok = adminData.TryMoveAssets(from, to, moveOptions, out report, out errorMessage); - - if (!ok) - { - MainConsole.Instance.Output(string.Format("{0} failed on database #{1}: {2}", previewOnly ? "tsshowmove" : "tsmove", i + 1, errorMessage)); - return; - } - - totalCandidates += report.CandidateCount; - totalAlreadyInTarget += report.AlreadyInTargetCount; - totalInserted += report.InsertedCount; - totalDeletedFromSource += report.DeletedFromSourceCount; - totalIndexAffected += report.IndexAffectedCount; - } - - if (supportedDatabases == 0) - { - MainConsole.Instance.Output(string.Format("{0} is not supported by the configured asset data plugin(s)", previewOnly ? "tsshowmove" : "tsmove")); - return; - } - - MainConsole.Instance.Output( - string.Format( - CultureInfo.InvariantCulture, - "{0} {1} -> {2}: candidates={3}, inserted={4}, already-in-target={5}, deleted-from-source={6}, index-affected={7}", - previewOnly ? "tsshowmove" : "tsmove", - from, - to, - totalCandidates, - totalInserted, - totalAlreadyInTarget, - totalDeletedFromSource, - totalIndexAffected)); - } - } -} diff --git a/opensim/bin/Robust.HG.ini.example b/opensim/bin/Robust.HG.ini.example deleted file mode 100644 index 02085df..0000000 --- a/opensim/bin/Robust.HG.ini.example +++ /dev/null @@ -1,929 +0,0 @@ -; * Run -; * $ Robust.exe -inifile Robust.HG.ini -; * - -; * Configurations for enabling HG1.5 -; * -; * HG1.5 handlers are: OpenSim.Server.Handlers.dll:GatekeeperService -; * OpenSim.Server.Handlers.dll:UserAgentService -; * Additional OpenSim.Server.Handlers.dll:AssetServiceConnector and -; * OpenSim.Server.Handlers.dll:XInventoryInConnector -; * are started in port 8002, outside the firewall -; * -; ** -; * -; * The Const section allows us to define some basic information that we -; * will use throughout our configuration. We will provide examples for -; * setting the base url of the Robust server and the public and private ports -; * it uses. Changing the values of the constants will set the operating -; * parameters thoughout the configuration. Other constants that may prove -; * to be useful may be added to the followin section. They may be -; * referenced anywhere in the configuration by using ${Const|Name}. One -; * such use is providing a base path for setting locations that Robust -; * uses to write data. -; * -[Const] - ; The domain or IP of the Robust server. - BaseHostname = "127.0.0.1" - - ; The http URL of the Robust server. - BaseURL = "http://${Const|BaseHostname}" - - ; The https URL of the Robust server. - ; Use this if you have the SSL enabled. - ; BaseURL = "https://${Const|BaseHostname}" - - ; The public port of the Robust server - PublicPort = "8002" - - ; The private port of the Robust server - PrivatePort = "8003" - -; * The startup section lists all the connectors to start up in this server -; * instance. This may be only one, or it may be the entire server suite. -; * Multiple connectors should be separated by commas. -; * -; * These are the IN connectors the server uses, the in connectors -; * read this config file and load the needed service and database connectors -; * -; * The full syntax of a connector string is: -; * [[@]/][:] -; * -[Startup] - ; Place to create a PID file - ; If no path if specified then a PID file is not created. - ; PIDFile = "/tmp/Robust.exe.pid" - - ; Plugin Registry Location - ; Set path to directory for plugin registry. Information - ; about the registered repositories and installed plugins - ; will be stored here - ; The Robust.exe process must have R/W access to the location - RegistryLocation = "." - - ; Modular configurations - ; Set path to directory for modular ini files... - ; The Robust.exe process must have R/W access to the location - ConfigDirectory = "robust-include" - - ; Console commands can be saved to a file, so the command history persists after a restart. (default is true) - ConsoleHistoryFileEnabled = true - - ; The history file can be just a filename (relative to OpenSim's bin/ directory - ; or it can be a full path to somewhere else. (default is OpenSimConsoleHistory.txt in bin/) - ConsoleHistoryFile = "RobustConsoleHistory.txt" - - ; How many lines of command history should we keep? (default is 100) - ConsoleHistoryFileLines = 100 - - ; Time stamp commands in history file (default false) - ; ConsoleHistoryTimeStamp = false - - ;; SSL selfsigned certificate settings. - ; Enable selfsigned certificate creation for local and external use. When set to true, will create a folder SSL\ and 2 sub folders SSL\ssl\ and SSL\src\. - ; Next creates and store an RSA private key in SSL\src\ and the derived selfsigned certificate in SSL\ssl\ folder. - ; Is also possible to renew the certificate on every server restart if CertRenewOnStartup is set to true. - EnableRobustSelfsignedCertSupport = false - - ;Renew the selfsigned certificate on every server startup ? - RobustCertRenewOnStartup = false - - ;; Certificate options: - ; Set the certificate file name. the output files extensions are CertFileName.p12 and RobustCertFileName.pfx. This must be different than CertFileName in OpenSim.ini - RobustCertFileName = "Robust" - - ; Set the certificate password. - RobustCertPassword = "mycertpass" - - ; The certificate host name (CN). - RobustCertHostName = ${Const|BaseHostname} - - ; The certificate host IP. - RobustCertHostIp = "127.0.0.1" - - ; peers SSL certificate validation options - ; you can allow selfsigned certificates or no official CA with next option set to true - NoVerifyCertChain = true - ; you can also bypass the hostname or domain verification - NoVerifyCertHostname = true - ; having both options true does provide encryption but with low security - ; set both true if you don't care to use SSL, they are needed to contact regions or grids that do use it. - - -[ServiceList] - AssetServiceConnector = "${Const|PrivatePort}/OpenSim.Server.Handlers.dll:AssetServiceConnector" - InventoryInConnector = "${Const|PrivatePort}/OpenSim.Server.Handlers.dll:XInventoryInConnector" - ;; Uncomment if you have set up Freeswitch (see [FreeswitchService] below) - ;VoiceConnector = "8004/OpenSim.Server.Handlers.dll:FreeswitchServerConnector" - GridServiceConnector = "${Const|PrivatePort}/OpenSim.Server.Handlers.dll:GridServiceConnector" - GridInfoServerInConnector = "${Const|PublicPort}/OpenSim.Server.Handlers.dll:GridInfoServerInConnector" - AuthenticationServiceConnector = "${Const|PrivatePort}/OpenSim.Server.Handlers.dll:AuthenticationServiceConnector" - OpenIdServerConnector = "${Const|PublicPort}/OpenSim.Server.Handlers.dll:OpenIdServerConnector" - AvatarServiceConnector = "${Const|PrivatePort}/OpenSim.Server.Handlers.dll:AvatarServiceConnector" - LLLoginServiceInConnector = "${Const|PublicPort}/OpenSim.Server.Handlers.dll:LLLoginServiceInConnector" - PresenceServiceConnector = "${Const|PrivatePort}/OpenSim.Server.Handlers.dll:PresenceServiceConnector" - UserAccountServiceConnector = "${Const|PrivatePort}/OpenSim.Server.Handlers.dll:UserAccountServiceConnector" - GridUserServiceConnector = "${Const|PrivatePort}/OpenSim.Server.Handlers.dll:GridUserServiceConnector" - AgentPreferencesServiceConnector = "${Const|PrivatePort}/OpenSim.Server.Handlers.dll:AgentPreferencesServiceConnector" - FriendsServiceConnector = "${Const|PrivatePort}/OpenSim.Server.Handlers.dll:FriendsServiceConnector" - MapAddServiceConnector = "${Const|PrivatePort}/OpenSim.Server.Handlers.dll:MapAddServiceConnector" - MapGetServiceConnector = "${Const|PublicPort}/OpenSim.Server.Handlers.dll:MapGetServiceConnector" - ;; Uncomment this if you want offline IM to work - ; OfflineIMServiceConnector = "${Const|PrivatePort}/OpenSim.Addons.OfflineIM.dll:OfflineIMServiceRobustConnector" - ;; Uncomment this if you want Groups V2 to work - ; GroupsServiceConnector = "${Const|PrivatePort}/OpenSim.Addons.Groups.dll:GroupsServiceRobustConnector" - ;; Uncomment to provide bakes caching - ; BakedTextureService = "${Const|PrivatePort}/OpenSim.Server.Handlers.dll:XBakesConnector" - - ;; Uncomment for UserProfiles see [UserProfilesService] to configure... - ; UserProfilesServiceConnector = "${Const|PublicPort}/OpenSim.Server.Handlers.dll:UserProfilesConnector" - - ;; Uncomment if you want to have centralized estate data - ; EstateDataService = "${Const|PrivatePort}/OpenSim.Server.Handlers.dll:EstateDataRobustConnector" - - MuteListConnector = "${Const|PrivatePort}/OpenSim.Server.Handlers.dll:MuteListServiceConnector" - - ;; Additions for Hypergrid - - GatekeeperServiceInConnector = "${Const|PublicPort}/OpenSim.Server.Handlers.dll:GatekeeperServiceInConnector" - UserAgentServerConnector = "${Const|PublicPort}/OpenSim.Server.Handlers.dll:UserAgentServerConnector" - HeloServiceInConnector = "${Const|PublicPort}/OpenSim.Server.Handlers.dll:HeloServiceInConnector" - HGFriendsServerConnector = "${Const|PublicPort}/OpenSim.Server.Handlers.dll:HGFriendsServerConnector" - InstantMessageServerConnector = "${Const|PublicPort}/OpenSim.Server.Handlers.dll:InstantMessageServerConnector" - HGInventoryServiceConnector = "HGInventoryService@${Const|PublicPort}/OpenSim.Server.Handlers.dll:XInventoryInConnector" - HGAssetServiceConnector = "HGAssetService@${Const|PublicPort}/OpenSim.Server.Handlers.dll:AssetServiceConnector" - ;; Uncomment this if you want Groups V2, HG to work - ; HGGroupsServiceConnector = "${Const|PublicPort}/OpenSim.Addons.Groups.dll:HGGroupsServiceRobustConnector" - -; * This is common for all services, it's the network setup for the entire -; * server instance, if none is specified above -; * -[Network] - port = ${Const|PrivatePort} - - ; HTTPS for "Out of band" management applications such as the remote admin - ; module. May specify https_main = True to make the main http server - ; use https or "False" to make the main server HTTP - ; https_main = False - ; - ; Create https_listener = "True" will create a listener on the port - ; specified. Provide the path to your server certificate along with it's - ; password - ; https_listener = False - ; - ; Set our listener to this port - ; https_port = 0 - ; - ; Path to X509 certificate - ; cert_path = "path/to/cert.p12" - ; - ; Password for cert - ; cert_pass = "password" - - ;; The follow 3 variables are for HTTP Basic Authentication for the Robust services. - ;; Use this if your central services in port ${Const|PrivatePort} need to be accessible on the Internet - ;; but you want to protect them from unauthorized access. - ; AuthType = "BasicHttpAuthentication" - ; HttpAuthUsername = "some_username" - ; HttpAuthPassword = "some_password" - ;; - ;; AuthType above can be overriden in any of the service sections below by - ; AuthType = "None" - ;; This is useful in cases where you want to protect most of the services, - ;; but unprotect individual services. Username and Password can also be - ;; overriden if you want to use different credentials for the different services. - ;; Hypergrid services are not affected by this; they are publicly available - ;; by design. - - ;; By default, scripts are not allowed to call private services via llHttpRequest() - ;; Such calls are detected by the X-SecondLife-Shared HTTP header - ;; If you allow such calls you must be sure that they are restricted to very trusted scripters - ;; (remember scripts can also be in visiting avatar attachments). - ;; This can be overriden in individual private service sections if necessary - AllowllHTTPRequestIn = false - - ; * The following are for the remote console - ; * They have no effect for the local or basic console types - ; * Leave commented to disable logins to the console - ;ConsoleUser = Test - ;ConsolePass = secret - ;ConsolePort = 0 - - -[Hypergrid] - ;# {HomeURI} {Hypergrid} {The Home URL of this grid} {} - ;; This is the address of the external robust server that - ;; runs the UserAgentsService, possibly this server. - ;; For example http://myworld.com:8002 - ;; This is a default that can be overwritten in some sections. - ; HomeURI = "${Const|BaseURL}:${Const|PublicPort}" - - ;# {GatekeeperURI} {Hypergrid} {The URL of the gatekeeper of this grid} {} - ;; This is the address of the external robust server - ;; that runs the Gatekeeper service, possibly this server. - ;; For example http://myworld.com:8002 - ;; This is a default that can be overwritten in some sections. - ; GatekeeperURI = "${Const|BaseURL}:${Const|PublicPort}" - - ;# {GatekeeperURIAlias} {Hypergrid} {alternative hostnames (FQDN), or IPs of the gatekeeper of this grid and port (default 80 or 443 if alias starts with https://)} {} - ;; comma separated list, - ;; this is to allow this grid to identify this as references to itself - ;; entries can be unsecure url (host:port) if using ssl, direct login url if different, old grid url, etc - ; GatekeeperURIAlias = "login.osgrid.org" - -[AccessControl] - ;# {AllowedClients} {} {Bar (|) separated list of allowed clients} {} - ;; Bar (|) separated list of viewers which may gain access to the regions. - ;; One can use a substring of the viewer name to enable only certain - ;; versions - ;; Example: Agent uses the viewer "Imprudence 1.3.2.0" - ;; - "Imprudence" has access - ;; - "Imprudence 1.3" has access - ;; - "Imprudence 1.3.1" has no access - ; AllowedClients = "" - - ;# {DeniedClients} {} {Bar (|) separated list of denied clients} {} - ;; Bar (|) separated list of viewers which may not gain access to the regions. - ;; One can use a Substring of the viewer name to disable only certain - ;; versions - ;; Example: Agent uses the viewer "Imprudence 1.3.2.0" - ;; - "Imprudence" has no access - ;; - "Imprudence 1.3" has no access - ;; - "Imprudence 1.3.1" has access - ; DeniedClients = "" - -[DatabaseService] - ; PGSQL - ; Uncomment these lines if you want to use PGSQL storage - ; Change the connection string to your db details - ;StorageProvider = "OpenSim.Data.PGSQL.dll" - ;ConnectionString = "Data Source=localhost;Database=opensim;User Id=opensim; password=***;" - - ; MySQL - ; Uncomment these lines if you want to use MySQL storage - ; Change the connection string to your db details - ; If using MySQL 8.0.4 or later, check that default-authentication-plugin=mysql_native_password - ; rather than caching_sha2_password is set in /etc/mysql/mysql.conf.d/mysqld.cnf (not applicable to MariaDB). - StorageProvider = "OpenSim.Data.MySQL.dll" - ConnectionString = "Data Source=localhost;Database=robust;User ID=opensim;Password=opensim123;Old Guids=true;SslMode=None;" - - -; * As an example, the below configuration precisely mimicks the legacy -; * asset server. It is read by the asset IN connector (defined above) -; * and it then loads the OUT connector (a local database module). That, -; * in turn, reads the asset loader and database connection information -; * -[AssetService] - - ;; Choose an asset service (Only one option should be enabled) - ;LocalServiceModule = "OpenSim.Services.AssetService.dll:TSAssetConnector" - LocalServiceModule = "OpenSim.Services.AssetService.dll:AssetService" - ;LocalServiceModule = "OpenSim.Services.FSAssetService.dll:FSAssetConnector" - - ;; FSAssets Directories. Base directory, where final asset files are stored and Spool directory for temp files - ;; These directories must be on the same physical filesystem - ;BaseDirectory = "./fsassets/data" - ;SpoolDirectory = "./fsassets/tmp" - - ;; FSAssets only: if running several instances, only one can run some services, so others need to be set as secondary - ;; for secondary instances uncoment this line - ;SecondaryInstance = true - - ;; FallbackService is only needed when TSAssetConnector is enabled - ;FallbackService = "OpenSim.Services.AssetService.dll:AssetService" - - ;; How many days since last updating the access time before its updated again by FSAssets when accessing an asset - ;; Reduces DB calls if asset is requested often. Default value 0 will always update access time - ;DaysBetweenAccessTimeUpdates = 30 - - ;; Should FSAssets print read/write stats to the robust console, default is true - ;ShowConsoleStats = true - - ;; Legacy provider used by FallbackService (optional but recommended) - StorageProvider = "OpenSim.Data.MySQL.dll:MySQLAssetData" - ConnectionString = "Data Source=localhost;Database=robust;User ID=opensim;Password=opensim123;Old Guids=true;SslMode=None;" - - ;; FSAssets uses a folder structure to reduce file access times. By default that structure is 00/00/00 - ;; Setting this to true will switch to an older 000/000 format, which can cause issues with long file lists - ;UseOsgridFormat = false - - ;; The following are common to both the default asset service and FSAsset service - - ;; Common asset service options - DefaultAssetLoader = "OpenSim.Framework.AssetLoader.Filesystem.dll" - AssetLoaderArgs = "./assets/AssetSets.xml" - - ; Allow maptile assets to remotely deleted by remote calls to the asset service. - ; There is no harm in having this as false - it just means that historical maptile assets are not deleted. - ; This only applies to maptiles served via the version 1 viewer mechanisms - ; Default is false - AllowRemoteDelete = false - - ; Allow all assets to be remotely deleted. - ; Only set this to true if you are operating a grid where you control all calls to the asset service - ; (where a necessary condition is that you control all simulators) and you need this for admin purposes. - ; If set to true, AllowRemoteDelete = true is required as well. - ; Default is false. - AllowRemoteDeleteAllTypes = false - -[TSAssetService] - - ;; Master switch for TS asset service. Must be true to activate TSAssetConnector. - Enabled = false - - ;; TS provider (explicit class avoids wrong provider selection) - StorageProvider = "OpenSim.Data.MySQL.dll:MySQLtsAssetData" - ConnectionString = "Data Source=localhost;Database=robust;User ID=opensim;Password=opensim123;Old Guids=true;SslMode=None;" - - ;; Optional type allowlist. Empty means all asset types. - TSAssetType = "" - - ;; Fallback hits are always copied immediately into TS storage (standard behavior) - ;; If true, failed immediate copies are queued and retried in background only when traffic is low - ;; If false, failed immediate copies are not retried by background worker - EnableFallbackAutoMigration = false - ;; If true, fallback records are deleted after successful copy into TS storage - EnableFallbackAutoDelete = false - MigrationCheckIntervalSeconds = 60 - MigrationLowTrafficMaxRequests = 3 - MigrationBatchSize = 25 - MigrationQueueMax = 50000 - - ;; Per-type DB routing (editierbar): nur Kern-Typen. - ;; Alle anderen Typen nutzen automatisch die Default-ConnectionString oben. - AssetDatabase_-2 = "Data Source=localhost;Database=robust;User ID=opensim;Password=opensim123;Old Guids=true;SslMode=None;" - AssetDatabase_0 = "Data Source=localhost;Database=robust;User ID=opensim;Password=opensim123;Old Guids=true;SslMode=None;" - AssetDatabase_1 = "Data Source=localhost;Database=robust;User ID=opensim;Password=opensim123;Old Guids=true;SslMode=None;" - AssetDatabase_3 = "Data Source=localhost;Database=robust;User ID=opensim;Password=opensim123;Old Guids=true;SslMode=None;" - AssetDatabase_5 = "Data Source=localhost;Database=robust;User ID=opensim;Password=opensim123;Old Guids=true;SslMode=None;" - AssetDatabase_6 = "Data Source=localhost;Database=robust;User ID=opensim;Password=opensim123;Old Guids=true;SslMode=None;" - AssetDatabase_7 = "Data Source=localhost;Database=robust;User ID=opensim;Password=opensim123;Old Guids=true;SslMode=None;" - AssetDatabase_10 = "Data Source=localhost;Database=robust;User ID=opensim;Password=opensim123;Old Guids=true;SslMode=None;" - AssetDatabase_13 = "Data Source=localhost;Database=robust;User ID=opensim;Password=opensim123;Old Guids=true;SslMode=None;" - AssetDatabase_20 = "Data Source=localhost;Database=robust;User ID=opensim;Password=opensim123;Old Guids=true;SslMode=None;" - AssetDatabase_21 = "Data Source=localhost;Database=robust;User ID=opensim;Password=opensim123;Old Guids=true;SslMode=None;" - AssetDatabase_49 = "Data Source=localhost;Database=robust;User ID=opensim;Password=opensim123;Old Guids=true;SslMode=None;" - AssetDatabase_56 = "Data Source=localhost;Database=robust;User ID=opensim;Password=opensim123;Old Guids=true;SslMode=None;" - AssetDatabase_57 = "Data Source=localhost;Database=robust;User ID=opensim;Password=opensim123;Old Guids=true;SslMode=None;" - -; * This configuration loads the inventory server modules. It duplicates -; * the function of the legacy inventory server -; * -[InventoryService] - LocalServiceModule = "OpenSim.Services.InventoryService.dll:XInventoryService" - - ; Will calls to purge folders (empty trash) and immediately delete/update items or folders (not move to trash first) succeed? - ; If this is set to false then some other arrangement must be made to perform these operations if necessary. - AllowDelete = true - - -; * This is the new style grid service. -; * "Realm" is the table that is used for user lookup. -; * It defaults to "regions", which uses the legacy tables -; * -[GridService] - LocalServiceModule = "OpenSim.Services.GridService.dll:GridService" - - ; Realm = "regions" - ; AllowDuplicateNames = "" - - ;; Perform distance check for the creation of a linked region - ; Check4096 = "True" - - ;; Needed to display non-default map tile images for linked regions - AssetService = "OpenSim.Services.AssetService.dll:AssetService" - - ;; Directory for map tile images of linked regions - ; MapTileDirectory = "./maptiles" - - ;; Next, we can specify properties of regions, including default and fallback regions - ;; The syntax is: Region_ = "" - ;; or: Region_ = "" - ;; where can be DefaultRegion, DefaultHGRegion, FallbackRegion, NoDirectLogin, Persistent, LockedOut, Reservation, NoMove, Authenticate - ;; - ;; DefaultRegion If a local login cannot be placed in the required region (e.g. home region does not exist, avatar is not allowed entry, etc.) - ;; then this region becomes the destination. Only the first online default region will be used. If no DefaultHGRegion - ;; is specified then this will also be used as the region for hypergrid connections that require it (commonly because they have not specified - ;; an explicit region. - ;; - ;; DefaultHGRegion If an avatar connecting via the hypergrid does not specify a region, then they are placed here. Only the first online - ;; region will be used. - ;; - ;; FallbackRegion If the DefaultRegion is not available for a local login, then any FallbackRegions are tried instead. These are tried in the - ;; order specified. This only applies to local logins at this time, not Hypergrid connections. - ;; - ;; NoDirectLogin A hypergrid user cannot directly connect to this region. This does not apply to local logins. - ;; - ;; Persistent When the simulator is shutdown, the region is signalled as offline but left registered on the grid. - ;; - ;; Example specification: - ; Region_Welcome_Area = "DefaultRegion, DefaultHGRegion" - ; (replace spaces with underscore) - - ;; Allow Hyperlinks to be created at the console - HypergridLinker = true - - ;; Allow supporting viewers to export content - ;; Set to false to prevent export - ExportSupported = true - - ;; If you have this set under [Hypergrid], no need to set it here, leave it commented - ; GatekeeperURI = "${Const|BaseURL}:${Const|PublicPort}" - - -; * This is the configuration for the freeswitch server in grid mode -[FreeswitchService] - LocalServiceModule = "OpenSim.Services.FreeswitchService.dll:FreeswitchService" - - ;; The IP address of your FreeSWITCH server. - ;; This address must be reachable by viewers. - ; ServerAddress = 127.0.0.1 - - ;; The following configuration parameters are optional - - ;; By default, this is the same as the ServerAddress - ; Realm = 127.0.0.1 - - ;; By default, this is the same as the ServerAddress on port 5060 - ; SIPProxy = 127.0.0.1:5060 - - ;; Default is 5000ms - ; DefaultTimeout = 5000 - - ;; The dial plan context. Default is "default" - ; Context = default - - ;; Currently unused - ; UserName = freeswitch - - ;; Currently unused - ; Password = password - - ;; The following parameters are for STUN = Simple Traversal of UDP through NATs - ;; See http://wiki.freeswitch.org/wiki/NAT_Traversal - ;; stun.freeswitch.org is not guaranteed to be running so use it in - ;; production at your own risk - ; EchoServer = 127.0.0.1 - ; EchoPort = 50505 - ; AttemptSTUN = false - - -; * This is the new style authentication service. Currently, only MySQL -; * is implemented. -; * -[AuthenticationService] - ; for the server connector - LocalServiceModule = "OpenSim.Services.AuthenticationService.dll:PasswordAuthenticationService" - ; Realm = "auth" - - ;; Allow the service to process HTTP getauthinfo calls. - ;; Default is false. - ; AllowGetAuthInfo = false - - ;; Allow the service to process HTTP setauthinfo calls. - ;; Default is false. - ; AllowSetAuthInfo = false - - ;; Allow the service to process HTTP setpassword calls. - ;; Default is false. - ; AllowSetPassword = false - - -[OpenIdService] - ; for the server connector - AuthenticationServiceModule = "OpenSim.Services.AuthenticationService.dll:PasswordAuthenticationService" - UserAccountServiceModule = "OpenSim.Services.UserAccountService.dll:UserAccountService" - - -; * This is the new style user service. -; * "Realm" is the table that is used for user lookup. -; * It defaults to "UserAccounts", which uses the new style. -; * Realm = "users" will use the legacy tables as an authentication source -; * -[UserAccountService] - ; for the server connector - LocalServiceModule = "OpenSim.Services.UserAccountService.dll:UserAccountService" - ; Realm = "UserAccounts" - - ; These are for creating new accounts by the service - AuthenticationService = "OpenSim.Services.AuthenticationService.dll:PasswordAuthenticationService" - PresenceService = "OpenSim.Services.PresenceService.dll:PresenceService" - GridService = "OpenSim.Services.GridService.dll:GridService" - InventoryService = "OpenSim.Services.InventoryService.dll:XInventoryService" - AvatarService = "OpenSim.Services.AvatarService.dll:AvatarService" - GridUserService = "OpenSim.Services.UserAccountService.dll:GridUserService" - - ;; This switch creates the minimum set of body parts and avatar entries for a viewer 2 - ;; to show a default "Ruth" avatar rather than a cloud for a newly created user. - ;; Default is false - CreateDefaultAvatarEntries = true - - ;; Allow the service to process HTTP createuser calls. - ;; Default is false. - ; AllowCreateUser = false - - ;; Allow the service to process HTTP setaccount calls. - ;; Default is false. - ; AllowSetAccount = false - - -[GridUserService] - ; for the server connector - LocalServiceModule = "OpenSim.Services.UserAccountService.dll:GridUserService" - - -[AgentPreferencesService] - ; for the server connector - LocalServiceModule = "OpenSim.Services.UserAccountService.dll:AgentPreferencesService" - - -[PresenceService] - ; for the server connector - LocalServiceModule = "OpenSim.Services.PresenceService.dll:PresenceService" - -[AvatarService] - ; for the server connector - LocalServiceModule = "OpenSim.Services.AvatarService.dll:AvatarService" - - -[FriendsService] - ; for the server connector - LocalServiceModule = "OpenSim.Services.FriendsService.dll:FriendsService" - -[EstateService] - LocalServiceModule = "OpenSim.Services.EstateService.dll:EstateDataService" - -[LibraryService] - LibraryName = "OpenSim Library" - DefaultLibrary = "./inventory/Libraries.xml" - - -[LoginService] - ; for the server connector - LocalServiceModule = "OpenSim.Services.LLLoginService.dll:LLLoginService" - ; for the service - UserAccountService = "OpenSim.Services.UserAccountService.dll:UserAccountService" - GridUserService = "OpenSim.Services.UserAccountService.dll:GridUserService" - AuthenticationService = "OpenSim.Services.AuthenticationService.dll:PasswordAuthenticationService" - InventoryService = "OpenSim.Services.InventoryService.dll:XInventoryService" - AvatarService = "OpenSim.Services.AvatarService.dll:AvatarService" - PresenceService = "OpenSim.Services.PresenceService.dll:PresenceService" - GridService = "OpenSim.Services.GridService.dll:GridService" - SimulationService ="OpenSim.Services.Connectors.dll:SimulationServiceConnector" - LibraryService = "OpenSim.Services.InventoryService.dll:LibraryService" - FriendsService = "OpenSim.Services.FriendsService.dll:FriendsService" - ; The minimum user level required for a user to be able to login. 0 by default - ; If you disable a particular user's account then you can set their login level below this number. - ; You can also change this level from the console though these changes will not be persisted. - ; MinLoginLevel = 0 - - ;; for hypergrid - UserAgentService = "OpenSim.Services.HypergridService.dll:UserAgentService" - - ; This inventory service will be used to initialize the user's inventory - HGInventoryServicePlugin = "HGInventoryService@OpenSim.Services.HypergridService.dll:HGSuitcaseInventoryService" - ; NOTE: HGInventoryServiceConstructorArg is deprecated. For now it will work, but see above - ; for the correct method if passing additional arguments. - ;; end hypergrid - - ; Ask co-operative viewers to use a different currency name - ;Currency = "" - - ;; Set minimum fee to publish classified - ; ClassifiedFee = 0 - - WelcomeMessage = "Welcome, Avatar!" - AllowRemoteSetLoginLevel = "false" - - ; For V2 map - MapTileURL = "${Const|BaseURL}:${Const|PublicPort}/"; - - ; Url to search service - ; SearchURL = "${Const|BaseURL}:${Const|PublicPort}/"; - - ; For V3 destination guide - ; DestinationGuide = "${Const|BaseURL}/guide" - - ; For V3 avatar picker (( work in progress )) - ; AvatarPicker = "${Const|BaseURL}/avatars" - - ; If you run this login server behind a proxy, set this to true - ; HasProxy = false - - ; Defaults for the users, if none is specified in the useraccounts table entry (ServiceURLs) - ;; If you have GatekeeperURI set under [Hypergrid], no need to set it here, leave it commented - ; GatekeeperURI = "${Const|BaseURL}:${Const|PublicPort}" - - SRV_HomeURI = "${Const|BaseURL}:${Const|PublicPort}" - SRV_InventoryServerURI = "${Const|BaseURL}:${Const|PublicPort}" - SRV_AssetServerURI = "${Const|BaseURL}:${Const|PublicPort}" - SRV_ProfileServerURI = "${Const|BaseURL}:${Const|PublicPort}" - SRV_FriendsServerURI = "${Const|BaseURL}:${Const|PublicPort}" - SRV_IMServerURI = "${Const|BaseURL}:${Const|PublicPort}" - SRV_GroupsServerURI = "${Const|BaseURL}:${Const|PublicPort}" - - ;# {DSTZone} {} {Override Daylight Saving Time rules} {* none local} "America/Los_Angeles;Pacific Standard Time" - ;; Viewers do not receive timezone information from the server - almost all (?) default to Pacific Standard Time - ;; However, they do rely on the server to tell them whether it's Daylight Saving Time or not. - ;; Hence, calculating DST based on a different timezone can result in a misleading viewer display and inconsistencies between grids. - ;; By default, this setting uses various timezone names to calculate DST with regards to the viewer's standard PST. - ;; Options are - ;; "none" no DST - ;; "local" use the server's only timezone to calculate DST. This is previous OpenSimulator behaviour. - ;; "America/Los_Angeles;Pacific Standard Time" use these timezone names to look up Daylight savings. - ;; 'America/Los_Angeles' is used on Linux/Mac systems whilst 'Pacific Standard Time' is used on Windows - DSTZone = "America/Los_Angeles;Pacific Standard Time" - - ;; If the region requested at login is not found and there are no default or fallback regions - ;; online or defined in section [GridService], try to send user to any region online - ;; this similar to legacy (was disabled on 0.9.2.0) - ;; you should set this to false and define regions with Default and possible Fallback flags - ;; With this option set to true, users maybe sent to regions they where not supposed to be, or even know about - AllowLoginFallbackToAnyRegion = false - - ;Basic Login Service Dos Protection Tweaks - ;; - ;; Some Grids/Users use a transparent proxy that makes use of the X-Forwarded-For HTTP Header, If you do, set this to true - ;; If you set this to true and you don't have a transparent proxy, it may allow attackers to put random things in the X-Forwarded-For header to - ;; get around this basic DOS protection. - ;DOSAllowXForwardedForHeader = false - ;; - ;; The protector adds up requests during this rolling period of time, default 10 seconds - ;DOSRequestTimeFrameMS = 10000 - ;; - ;; The amount of requests in the above timeframe from the same endpoint that triggers protection - ;DOSMaxRequestsInTimeFrame = 5 - ;; - ;; The amount of time that a specific endpoint is blocked. Default 2 minutes. - ;DOSForgiveClientAfterMS = 120000 - ;; - ;; To turn off basic dos protection, set the DOSMaxRequestsInTimeFrame to 0. - - ;; Allow banning via hashed MAC must be set in both [GatekeeperService] and [LoginService] - ;DeniedMacs = "YOURLONGMACTRSING ANOTHERMAC" - -[MapImageService] - LocalServiceModule = "OpenSim.Services.MapImageService.dll:MapImageService" - - ; Set this if you want to change the default - ; TilesStoragePath = "maptiles" - ; - ; If for some reason you have the AddMapTile service outside the firewall (e.g. ${Const|PublicPort}), - ; you may want to set this. Otherwise, don't set it, because it's already protected. - ; GridService = "OpenSim.Services.GridService.dll:GridService" - ; - ; Additionally, if you run this server behind a proxy, set this to true - ; HasProxy = false - - -[GridInfoService] - ; These settings are used to return information on a get_grid_info call. - ; Client launcher scripts and third-party clients make use of this to - ; autoconfigure the client and to provide a nice user experience. If you - ; want to facilitate that, you should configure the settings here according - ; to your grid or standalone setup. - ; - ; See http://opensimulator.org/wiki/GridInfo - - ; login uri: for grid this is the login server URI - login = ${Const|BaseURL}:${Const|PublicPort}/ - - ; long grid name: the long name of your grid - gridname = "the lost continent of hippo" - - ; short grid name: the short name of your grid - gridnick = "hippogrid" - - ; login page: optional: if it exists it will be used to tell the client to use - ; this as splash page - ;welcome = ${Const|BaseURL}/welcome - - ; helper uri: optional: if it exists it will be used to tell the client to use - ; this for all economy related things - ;economy = ${Const|BaseURL}/economy - - ; web page of grid: optional: page providing further information about your grid - ;about = ${Const|BaseURL}/about - - ; account creation: optional: page providing further information about obtaining - ; a user account on your grid - ;register = ${Const|BaseURL}/register - - ; help: optional: page providing further assistance for users of your grid - ;help = ${Const|BaseURL}/help - - ; password help: optional: page providing password assistance for users of your grid - ;password = ${Const|BaseURL}/password - - ; HG address of the gatekeeper, if you have one - ; this is the entry point for all the regions of the world - ; gatekeeper = ${Const|BaseURL}:${Const|PublicPort}/ - - ; a http page for grid status - ;GridStatus = ${Const|BaseURL}:${Const|PublicPort}/GridStatus - ; a RSS page for grid status - ;GridStatusRSS = ${Const|BaseURL}:${Const|PublicPort}/GridStatusRSS - - ; optional web page for profiles - ;[AGENT_NAME] will be converted to Firstname.LastName by viewers - ; web_profile_url = http://webprofilesurl:ItsPort?name=[AGENT_NAME] - - ; http REST handler /get_grid_stats may output some grid statistics This can be turned off here - ;DisableStatsEndpoint = true - -[GatekeeperService] - LocalServiceModule = "OpenSim.Services.HypergridService.dll:GatekeeperService" - ;; for the service - UserAccountService = "OpenSim.Services.UserAccountService.dll:UserAccountService" - UserAgentService = "OpenSim.Services.HypergridService.dll:UserAgentService" - PresenceService = "OpenSim.Services.PresenceService.dll:PresenceService" - GridUserService = "OpenSim.Services.UserAccountService.dll:GridUserService" - GridService = "OpenSim.Services.GridService.dll:GridService" - AuthenticationService = "OpenSim.Services.Connectors.dll:AuthenticationServicesConnector" - SimulationService ="OpenSim.Services.Connectors.dll:SimulationServiceConnector" - ; how does the outside world reach me? This acts as public key too. - ;; If you have GatekeeperURI set under [Hypergrid], no need to set it here, leave it commented - ; ExternalName = "${Const|BaseURL}:${Const|PublicPort}" - - ; Does this grid allow incoming links to any region in it? - ; If false, HG TPs happen only to the Default regions specified in [GridService] section - AllowTeleportsToAnyRegion = true - - ; If you run this gatekeeper server behind a proxy, set this to true - ; HasProxy = false - - ;; Are foreign visitors allowed? - ;ForeignAgentsAllowed = true - ;; - ;; If ForeignAgentsAllowed is true, make exceptions using AllowExcept. - ;; Leave blank or commented for no exceptions. - ; AllowExcept = "http://griefer.com:8002, http://enemy.com:8002" - ;; - ;; If ForeignAgentsAllowed is false, make exceptions using DisallowExcept - ;; Leave blank or commented for no exceptions. - ; DisallowExcept = "http://myfriendgrid.com:8002, http://myboss.com:8002" - - ;; Allow banning via hashed MAC must be set in both [GatekeeperService] and [LoginService] - ;DeniedMacs = "YOURLONGMACTRSING ANOTHERMAC" - -[UserAgentService] - LocalServiceModule = "OpenSim.Services.HypergridService.dll:UserAgentService" - ;; for the service - GridUserService = "OpenSim.Services.UserAccountService.dll:GridUserService" - GridService = "OpenSim.Services.GridService.dll:GridService" - GatekeeperService = "OpenSim.Services.HypergridService.dll:GatekeeperService" - PresenceService = "OpenSim.Services.PresenceService.dll:PresenceService" - FriendsService = "OpenSim.Services.FriendsService.dll:FriendsService" - UserAccountService = "OpenSim.Services.UserAccountService.dll:UserAccountService" - - ; If you run this user agent server behind a proxy, set this to true - ; HasProxy = false - - ;; If you separate the UserAgentService from the LoginService, set this to - ;; the IP address of the machine where your LoginService is - ;LoginServerIP = "127.0.0.1" - - ; User level required to be contacted from other grids - ;LevelOutsideContacts = 0 - - ;; Restrictions on destinations of local users. - ;; Are local users allowed to visit other grids? - ;; What user level? Use variables of this forrm: - ;; ForeignTripsAllowed_Level_ = true | false - ;; (the default is true) - ;; For example: - ; ForeignTripsAllowed_Level_0 = false - ; ForeignTripsAllowed_Level_200 = true ; true is default, no need to say it - ;; - ;; If ForeignTripsAllowed is false, make exceptions using DisallowExcept - ;; Leave blank or commented for no exceptions. - ; DisallowExcept_Level_0 = "http://myothergrid.com:8002, http://boss.com:8002" - ;; - ;; If ForeignTripsAllowed is true, make exceptions using AllowExcept. - ;; Leave blank or commented for no exceptions. - ; AllowExcept_Level_200 = "http://griefer.com:8002, http://enemy.com:8002" - - ;; This variable controls what is exposed to profiles of local users - ;; as seen from outside of this grid. Leave it uncommented for exposing - ;; UserTitle, UserFlags and the creation date. Uncomment and change to False - ;; to block this info from being exposed. - ; ShowUserDetailsInHGProfile = True - - -; * The interface that local users get when they are in other grids. -; * This restricts the inventory operations while in other grids. -; * Still not completely safe, especially if users perform inventory operations -; * while in those grids. The more the user accesses his/her inventory, the more -; * those simulators will know about the user's inventory. -; * -[HGInventoryService] - ; For the InventoryServiceInConnector - LocalServiceModule = "OpenSim.Services.HypergridService.dll:HGSuitcaseInventoryService" - ;; alternatives: - ;; HG1.5, more permissive, not recommended, but still supported - ;LocalServiceModule = "OpenSim.Services.HypergridService.dll:HGInventoryService" - ;; HG1.0, totally permissive, not recommended, but OK for grids with 100% trust - ;LocalServiceModule = "OpenSim.Services.InventoryService.dll:XInventoryService" - - UserAccountsService = "OpenSim.Services.UserAccountService.dll:UserAccountService" - AvatarService = "OpenSim.Services.AvatarService.dll:AvatarService" - - ; HGInventoryService is a public-facing inventory service that allows users to - ; interact with their suitcase folder when on a foreign grid. This reuses the general inventory service connector. - ; Hence, if the user has set up authentication in [Network] to protect their private services - ; make sure it is not set here. - AuthType = None - - ;; Can overwrite the default in [Hypergrid], but probably shouldn't - ; HomeURI = "${Const|BaseURL}:${Const|PublicPort}" - - -; * The interface that local users get when they are in other grids. -; * This restricts the access that the rest of the world has to -; * the assets of this world. -; * -[HGAssetService] - ;; Use the second option if you have FSAsset service enabled - LocalServiceModule = "OpenSim.Services.HypergridService.dll:HGAssetService" - ;LocalServiceModule = "OpenSim.Services.HypergridService.dll:HGFSAssetService" - - UserAccountsService = "OpenSim.Services.UserAccountService.dll:UserAccountService" - - ; HGAssetService is a public-facing service that allows users to - ; read and create assets when on another grid. This reuses the general asset service connector. - ; Hence, if the user has set up authentication in [Network] to protect their private services - ; make sure it is overriden for this public service. - AuthType = None - - ;; Can overwrite the default in [Hypergrid], but probably shouldn't - ; HomeURI = "${Const|BaseURL}:${Const|PublicPort}" - - ;; The asset types that this grid can export to / import from other grids. - ;; Comma separated. - ;; Valid values are all the asset types in OpenMetaverse.AssetType, namely: - ;; Unknown, Texture, Sound, CallingCard, Landmark, Clothing, Object, Notecard, LSLText, - ;; LSLBytecode, TextureTGA, Bodypart, SoundWAV, ImageTGA, ImageJPEG, Animation, Gesture, Mesh - ;; - ;; Leave blank or commented if you don't want to apply any restrictions. - ;; A more strict, but still reasonable, policy may be to disallow the exchange - ;; of scripts, like so: - ; DisallowExport ="LSLText" - ; DisallowImport ="LSLBytecode" - - -[HGFriendsService] - LocalServiceModule = "OpenSim.Services.HypergridService.dll:HGFriendsService" - UserAgentService = "OpenSim.Services.HypergridService.dll:UserAgentService" - FriendsService = "OpenSim.Services.FriendsService.dll:FriendsService" - UserAccountService = "OpenSim.Services.UserAccountService.dll:UserAccountService" - GridService = "OpenSim.Services.GridService.dll:GridService" - PresenceService = "OpenSim.Services.PresenceService.dll:PresenceService" - - -[HGInstantMessageService] - LocalServiceModule = "OpenSim.Services.HypergridService.dll:HGInstantMessageService" - GridService = "OpenSim.Services.GridService.dll:GridService" - PresenceService = "OpenSim.Services.PresenceService.dll:PresenceService" - UserAgentService = "OpenSim.Services.HypergridService.dll:UserAgentService" - ; This should always be true in the Robust config - InGatekeeper = True - - -[Messaging] - ; OfflineIM - OfflineIMService = "OpenSim.Addons.OfflineIM.dll:OfflineIMService" - ; maximum number of offline ims per user, default 25 - ;MaxOfflineIMs = 25; - - -[Groups] - ;; for the HG Groups service - OfflineIMService = "OpenSim.Addons.OfflineIM.dll:OfflineIMService" - UserAccountService = "OpenSim.Services.UserAccountService.dll:UserAccountService" - - ;; What is the HomeURI of users associated with this grid? - ;; Can overwrite the default in [Hypergrid], but probably shouldn't - ; HomeURI = "${Const|BaseURL}:${Const|PublicPort}" - ;; end hypergrid - - ;; Sets the maximum number of groups an agent may join - ; MaxAgentGroups = 42 - - -[UserProfilesService] - LocalServiceModule = "OpenSim.Services.UserProfilesService.dll:UserProfilesService" - Enabled = false - ;; Configure this for separate profiles database - ;; ConnectionString = "Data Source=localhost;Database=opensim;User ID=opensim;Password=*****;Old Guids=true;" - ;; Realm = UserProfiles - UserAccountService = OpenSim.Services.UserAccountService.dll:UserAccountService - AuthenticationServiceModule = "OpenSim.Services.AuthenticationService.dll:PasswordAuthenticationService" - - -[BakedTextureService] - LocalServiceModule = "OpenSim.Server.Handlers.dll:XBakes" - ;; This directory must be writable by the user ROBUST runs as. It will be created automatically. - BaseDirectory = "./bakes" - -[MuteListService] - LocalServiceModule = "OpenSim.Services.MuteListService.dll:MuteListService"