From 537fa44118fcc52abbe826f16c2d1adf30b1fe22 Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Thu, 19 Mar 2026 21:03:14 -0700 Subject: [PATCH 1/5] adding retries to hotreload of server type in test --- .../HotReload/ConfigurationHotReloadTests.cs | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs b/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs index 18c19cbd7a..9bbf19e2bc 100644 --- a/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs +++ b/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs @@ -780,7 +780,9 @@ await WaitForConditionAsync( succeedConfigLog = _writer.ToString(); } - HttpResponseMessage restResult = await _testClient.GetAsync("/rest/Book"); + // After hot-reload, the engine may still be re-initializing metadata providers. + // Poll the REST endpoint to allow time for the engine to become fully ready. + HttpResponseMessage restResult = await WaitForRestEndpointAsync("/rest/Book", HttpStatusCode.OK); // Assert Assert.IsTrue(failedConfigLog.Contains(HOT_RELOAD_FAILURE_MESSAGE)); @@ -838,7 +840,9 @@ await WaitForConditionAsync( succeedConfigLog = _writer.ToString(); } - HttpResponseMessage restResult = await _testClient.GetAsync("/rest/Book"); + // After hot-reload, the engine may still be re-initializing metadata providers. + // Poll the REST endpoint to allow time for the engine to become fully ready. + HttpResponseMessage restResult = await WaitForRestEndpointAsync("/rest/Book", HttpStatusCode.OK); // Assert Assert.IsTrue(failedConfigLog.Contains(HOT_RELOAD_FAILURE_MESSAGE)); @@ -974,4 +978,31 @@ private static async Task WaitForConditionAsync(Func condition, TimeSpan t throw new TimeoutException("The condition was not met within the timeout period."); } + + /// + /// Polls a REST endpoint until it returns the expected status code. + /// After a successful hot-reload, the engine may still be re-initializing + /// metadata providers, so an immediate request can intermittently fail. + /// + private static async Task WaitForRestEndpointAsync( + string requestUri, + HttpStatusCode expectedStatus, + int maxRetries = 5, + int delayMilliseconds = 1000) + { + HttpResponseMessage response = null; + for (int attempt = 1; attempt <= maxRetries; attempt++) + { + response = await _testClient.GetAsync(requestUri); + if (response.StatusCode == expectedStatus) + { + return response; + } + + Console.WriteLine($"REST {requestUri} returned {response.StatusCode} on attempt {attempt}/{maxRetries}, retrying..."); + await Task.Delay(delayMilliseconds); + } + + return response; + } } From 962c80c41e036508ffed3e3835bbb704bb5016d9 Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Thu, 19 Mar 2026 21:56:06 -0700 Subject: [PATCH 2/5] add some more retries --- .../HotReload/ConfigurationHotReloadTests.cs | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs b/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs index 9bbf19e2bc..6035a85ea0 100644 --- a/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs +++ b/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs @@ -331,19 +331,40 @@ await WaitForConditionAsync( HttpResponseMessage badPathRestResult = await _testClient.GetAsync($"rest/Book"); HttpResponseMessage badPathGQLResult = await _testClient.SendAsync(request); - HttpResponseMessage result = await _testClient.GetAsync($"{restPath}/Book"); + // After hot-reload, the engine may still be re-initializing metadata providers. + // Poll the REST endpoint to allow time for the engine to become fully ready. + HttpResponseMessage result = await WaitForRestEndpointAsync($"{restPath}/Book", HttpStatusCode.OK); string reloadRestContent = await result.Content.ReadAsStringAsync(); - JsonElement reloadGQLContents = await GraphQLRequestExecutor.PostGraphQLRequestAsync( - _testClient, - _configProvider, - GQL_QUERY_NAME, - GQL_QUERY); + + // Retry GraphQL request because metadata re-initialization happens asynchronously + // after the "Validated hot-reloaded configuration file" message. + JsonElement reloadGQLContents = default; + bool querySucceeded = false; + for (int attempt = 1; attempt <= 5; attempt++) + { + reloadGQLContents = await GraphQLRequestExecutor.PostGraphQLRequestAsync( + _testClient, + _configProvider, + GQL_QUERY_NAME, + GQL_QUERY); + + if (reloadGQLContents.ValueKind == JsonValueKind.Object && + reloadGQLContents.TryGetProperty("items", out _)) + { + querySucceeded = true; + break; + } + + Console.WriteLine($"GraphQL query returned {reloadGQLContents.ValueKind} on attempt {attempt}/5, retrying..."); + await Task.Delay(1000); + } // Assert // Old paths are not found. Assert.AreEqual(HttpStatusCode.BadRequest, badPathRestResult.StatusCode); Assert.AreEqual(HttpStatusCode.NotFound, badPathGQLResult.StatusCode); // Hot reloaded paths return correct response. + Assert.IsTrue(querySucceeded, "GraphQL query did not return valid results after hot-reload."); Assert.IsTrue(SqlTestHelper.JsonStringsDeepEqual(restBookContents, reloadRestContent)); SqlTestHelper.PerformTestEqualJsonStrings(_bookDBOContents, reloadGQLContents.GetProperty("items").ToString()); } From b616124c5d6f0f2a4c1a7572d1332d96d8e6c583 Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Thu, 19 Mar 2026 23:09:17 -0700 Subject: [PATCH 3/5] address comments --- .../HotReload/ConfigurationHotReloadTests.cs | 67 +++++++++++++------ 1 file changed, 46 insertions(+), 21 deletions(-) diff --git a/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs b/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs index 6035a85ea0..235c24e71b 100644 --- a/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs +++ b/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs @@ -338,24 +338,34 @@ await WaitForConditionAsync( // Retry GraphQL request because metadata re-initialization happens asynchronously // after the "Validated hot-reloaded configuration file" message. + // PostGraphQLRequestAsync can also throw (e.g. JsonException) if the server returns + // a non-JSON error response during re-initialization, so we catch and retry. JsonElement reloadGQLContents = default; bool querySucceeded = false; for (int attempt = 1; attempt <= 5; attempt++) { - reloadGQLContents = await GraphQLRequestExecutor.PostGraphQLRequestAsync( - _testClient, - _configProvider, - GQL_QUERY_NAME, - GQL_QUERY); - - if (reloadGQLContents.ValueKind == JsonValueKind.Object && - reloadGQLContents.TryGetProperty("items", out _)) + try + { + reloadGQLContents = await GraphQLRequestExecutor.PostGraphQLRequestAsync( + _testClient, + _configProvider, + GQL_QUERY_NAME, + GQL_QUERY); + + if (reloadGQLContents.ValueKind == JsonValueKind.Object && + reloadGQLContents.TryGetProperty("items", out _)) + { + querySucceeded = true; + break; + } + + Console.WriteLine($"GraphQL query returned {reloadGQLContents.ValueKind} on attempt {attempt}/5, retrying..."); + } + catch (Exception ex) when (ex is JsonException || ex is HttpRequestException) { - querySucceeded = true; - break; + Console.WriteLine($"GraphQL request threw {ex.GetType().Name} on attempt {attempt}/5: {ex.Message}"); } - Console.WriteLine($"GraphQL query returned {reloadGQLContents.ValueKind} on attempt {attempt}/5, retrying..."); await Task.Delay(1000); } @@ -696,17 +706,24 @@ await WaitForConditionAsync( bool querySucceeded = false; for (int attempt = 1; attempt <= 10; attempt++) { - reloadGQLContents = await GraphQLRequestExecutor.PostGraphQLRequestAsync( - _testClient, - _configProvider, - GQL_QUERY_NAME, - GQL_QUERY); - - if (reloadGQLContents.ValueKind == JsonValueKind.Object && - reloadGQLContents.TryGetProperty("items", out _)) + try { - querySucceeded = true; - break; + reloadGQLContents = await GraphQLRequestExecutor.PostGraphQLRequestAsync( + _testClient, + _configProvider, + GQL_QUERY_NAME, + GQL_QUERY); + + if (reloadGQLContents.ValueKind == JsonValueKind.Object && + reloadGQLContents.TryGetProperty("items", out _)) + { + querySucceeded = true; + break; + } + } + catch (Exception ex) when (ex is JsonException || ex is HttpRequestException) + { + Console.WriteLine($"GraphQL request threw {ex.GetType().Name} on attempt {attempt}/10: {ex.Message}"); } await Task.Delay(1000); @@ -1021,9 +1038,17 @@ private static async Task WaitForRestEndpointAsync( } Console.WriteLine($"REST {requestUri} returned {response.StatusCode} on attempt {attempt}/{maxRetries}, retrying..."); + + // Dispose unsuccessful responses to avoid leaking connections/sockets. + if (attempt < maxRetries) + { + response.Dispose(); + } + await Task.Delay(delayMilliseconds); } + // Return the last response (undisposed) so the caller can inspect/assert on it. return response; } } From 188d1795fec125e51319e7bccdde42d7049610da Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Fri, 20 Mar 2026 01:41:17 -0700 Subject: [PATCH 4/5] factor out helper --- .../HotReload/ConfigurationHotReloadTests.cs | 111 ++++++++---------- 1 file changed, 49 insertions(+), 62 deletions(-) diff --git a/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs b/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs index 235c24e71b..6e613db149 100644 --- a/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs +++ b/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs @@ -336,38 +336,8 @@ await WaitForConditionAsync( HttpResponseMessage result = await WaitForRestEndpointAsync($"{restPath}/Book", HttpStatusCode.OK); string reloadRestContent = await result.Content.ReadAsStringAsync(); - // Retry GraphQL request because metadata re-initialization happens asynchronously - // after the "Validated hot-reloaded configuration file" message. - // PostGraphQLRequestAsync can also throw (e.g. JsonException) if the server returns - // a non-JSON error response during re-initialization, so we catch and retry. - JsonElement reloadGQLContents = default; - bool querySucceeded = false; - for (int attempt = 1; attempt <= 5; attempt++) - { - try - { - reloadGQLContents = await GraphQLRequestExecutor.PostGraphQLRequestAsync( - _testClient, - _configProvider, - GQL_QUERY_NAME, - GQL_QUERY); - - if (reloadGQLContents.ValueKind == JsonValueKind.Object && - reloadGQLContents.TryGetProperty("items", out _)) - { - querySucceeded = true; - break; - } - - Console.WriteLine($"GraphQL query returned {reloadGQLContents.ValueKind} on attempt {attempt}/5, retrying..."); - } - catch (Exception ex) when (ex is JsonException || ex is HttpRequestException) - { - Console.WriteLine($"GraphQL request threw {ex.GetType().Name} on attempt {attempt}/5: {ex.Message}"); - } - - await Task.Delay(1000); - } + // Poll the GraphQL endpoint to allow time for the engine to become fully ready. + (bool querySucceeded, JsonElement reloadGQLContents) = await WaitForGraphQLEndpointAsync(GQL_QUERY_NAME, GQL_QUERY); // Assert // Old paths are not found. @@ -698,36 +668,8 @@ await WaitForConditionAsync( RuntimeConfig updatedRuntimeConfig = _configProvider.GetConfig(); MsSqlOptions actualSessionContext = updatedRuntimeConfig.DataSource.GetTypedOptions(); - // Retry GraphQL request because metadata re-initialization happens asynchronously - // after the "Validated hot-reloaded configuration file" message. The metadata provider - // factory clears and re-initializes providers on the hot-reload thread, so requests - // arriving before that completes will fail with "Initialization of metadata incomplete." - JsonElement reloadGQLContents = default; - bool querySucceeded = false; - for (int attempt = 1; attempt <= 10; attempt++) - { - try - { - reloadGQLContents = await GraphQLRequestExecutor.PostGraphQLRequestAsync( - _testClient, - _configProvider, - GQL_QUERY_NAME, - GQL_QUERY); - - if (reloadGQLContents.ValueKind == JsonValueKind.Object && - reloadGQLContents.TryGetProperty("items", out _)) - { - querySucceeded = true; - break; - } - } - catch (Exception ex) when (ex is JsonException || ex is HttpRequestException) - { - Console.WriteLine($"GraphQL request threw {ex.GetType().Name} on attempt {attempt}/10: {ex.Message}"); - } - - await Task.Delay(1000); - } + // Poll the GraphQL endpoint to allow time for the engine to become fully ready. + (bool querySucceeded, JsonElement reloadGQLContents) = await WaitForGraphQLEndpointAsync(GQL_QUERY_NAME, GQL_QUERY, maxRetries: 10); // Assert Assert.IsTrue(querySucceeded, "GraphQL query did not return valid results after hot-reload. Metadata initialization may not have completed."); @@ -1051,4 +993,49 @@ private static async Task WaitForRestEndpointAsync( // Return the last response (undisposed) so the caller can inspect/assert on it. return response; } + + /// + /// Polls a GraphQL endpoint until it returns a valid response containing + /// the expected property. After a successful hot-reload, the engine may + /// still be re-initializing metadata providers, so an immediate request + /// can intermittently fail. PostGraphQLRequestAsync can also throw + /// (e.g. JsonException) if the server returns a non-JSON error response + /// during re-initialization. + /// + private static async Task<(bool Success, JsonElement Result)> WaitForGraphQLEndpointAsync( + string queryName, + string query, + string expectedProperty = "items", + int maxRetries = 5, + int delayMilliseconds = 1000) + { + JsonElement result = default; + for (int attempt = 1; attempt <= maxRetries; attempt++) + { + try + { + result = await GraphQLRequestExecutor.PostGraphQLRequestAsync( + _testClient, + _configProvider, + queryName, + query); + + if (result.ValueKind == JsonValueKind.Object && + result.TryGetProperty(expectedProperty, out _)) + { + return (true, result); + } + + Console.WriteLine($"GraphQL query returned {result.ValueKind} on attempt {attempt}/{maxRetries}, retrying..."); + } + catch (Exception ex) when (ex is JsonException || ex is HttpRequestException) + { + Console.WriteLine($"GraphQL request threw {ex.GetType().Name} on attempt {attempt}/{maxRetries}: {ex.Message}"); + } + + await Task.Delay(delayMilliseconds); + } + + return (false, result); + } } From 00a93c729394af1062a21d357ea8031b35c7567b Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Fri, 20 Mar 2026 19:54:48 -0700 Subject: [PATCH 5/5] add using to caller side for disposal --- .../Configuration/HotReload/ConfigurationHotReloadTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs b/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs index 6e613db149..1f98d32fac 100644 --- a/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs +++ b/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs @@ -333,7 +333,7 @@ await WaitForConditionAsync( // After hot-reload, the engine may still be re-initializing metadata providers. // Poll the REST endpoint to allow time for the engine to become fully ready. - HttpResponseMessage result = await WaitForRestEndpointAsync($"{restPath}/Book", HttpStatusCode.OK); + using HttpResponseMessage result = await WaitForRestEndpointAsync($"{restPath}/Book", HttpStatusCode.OK); string reloadRestContent = await result.Content.ReadAsStringAsync(); // Poll the GraphQL endpoint to allow time for the engine to become fully ready. @@ -762,7 +762,7 @@ await WaitForConditionAsync( // After hot-reload, the engine may still be re-initializing metadata providers. // Poll the REST endpoint to allow time for the engine to become fully ready. - HttpResponseMessage restResult = await WaitForRestEndpointAsync("/rest/Book", HttpStatusCode.OK); + using HttpResponseMessage restResult = await WaitForRestEndpointAsync("/rest/Book", HttpStatusCode.OK); // Assert Assert.IsTrue(failedConfigLog.Contains(HOT_RELOAD_FAILURE_MESSAGE)); @@ -822,7 +822,7 @@ await WaitForConditionAsync( // After hot-reload, the engine may still be re-initializing metadata providers. // Poll the REST endpoint to allow time for the engine to become fully ready. - HttpResponseMessage restResult = await WaitForRestEndpointAsync("/rest/Book", HttpStatusCode.OK); + using HttpResponseMessage restResult = await WaitForRestEndpointAsync("/rest/Book", HttpStatusCode.OK); // Assert Assert.IsTrue(failedConfigLog.Contains(HOT_RELOAD_FAILURE_MESSAGE));