diff --git a/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs b/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs index 18c19cbd7a..1f98d32fac 100644 --- a/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs +++ b/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs @@ -331,19 +331,20 @@ await WaitForConditionAsync( HttpResponseMessage badPathRestResult = await _testClient.GetAsync($"rest/Book"); HttpResponseMessage badPathGQLResult = await _testClient.SendAsync(request); - HttpResponseMessage result = await _testClient.GetAsync($"{restPath}/Book"); + // After hot-reload, the engine may still be re-initializing metadata providers. + // Poll the REST endpoint to allow time for the engine to become fully ready. + using HttpResponseMessage result = await WaitForRestEndpointAsync($"{restPath}/Book", HttpStatusCode.OK); string reloadRestContent = await result.Content.ReadAsStringAsync(); - JsonElement reloadGQLContents = await GraphQLRequestExecutor.PostGraphQLRequestAsync( - _testClient, - _configProvider, - GQL_QUERY_NAME, - GQL_QUERY); + + // Poll the GraphQL endpoint to allow time for the engine to become fully ready. + (bool querySucceeded, JsonElement reloadGQLContents) = await WaitForGraphQLEndpointAsync(GQL_QUERY_NAME, GQL_QUERY); // Assert // Old paths are not found. Assert.AreEqual(HttpStatusCode.BadRequest, badPathRestResult.StatusCode); Assert.AreEqual(HttpStatusCode.NotFound, badPathGQLResult.StatusCode); // Hot reloaded paths return correct response. + Assert.IsTrue(querySucceeded, "GraphQL query did not return valid results after hot-reload."); Assert.IsTrue(SqlTestHelper.JsonStringsDeepEqual(restBookContents, reloadRestContent)); SqlTestHelper.PerformTestEqualJsonStrings(_bookDBOContents, reloadGQLContents.GetProperty("items").ToString()); } @@ -667,29 +668,8 @@ await WaitForConditionAsync( RuntimeConfig updatedRuntimeConfig = _configProvider.GetConfig(); MsSqlOptions actualSessionContext = updatedRuntimeConfig.DataSource.GetTypedOptions(); - // Retry GraphQL request because metadata re-initialization happens asynchronously - // after the "Validated hot-reloaded configuration file" message. The metadata provider - // factory clears and re-initializes providers on the hot-reload thread, so requests - // arriving before that completes will fail with "Initialization of metadata incomplete." - JsonElement reloadGQLContents = default; - bool querySucceeded = false; - for (int attempt = 1; attempt <= 10; attempt++) - { - reloadGQLContents = await GraphQLRequestExecutor.PostGraphQLRequestAsync( - _testClient, - _configProvider, - GQL_QUERY_NAME, - GQL_QUERY); - - if (reloadGQLContents.ValueKind == JsonValueKind.Object && - reloadGQLContents.TryGetProperty("items", out _)) - { - querySucceeded = true; - break; - } - - await Task.Delay(1000); - } + // Poll the GraphQL endpoint to allow time for the engine to become fully ready. + (bool querySucceeded, JsonElement reloadGQLContents) = await WaitForGraphQLEndpointAsync(GQL_QUERY_NAME, GQL_QUERY, maxRetries: 10); // Assert Assert.IsTrue(querySucceeded, "GraphQL query did not return valid results after hot-reload. Metadata initialization may not have completed."); @@ -780,7 +760,9 @@ await WaitForConditionAsync( succeedConfigLog = _writer.ToString(); } - HttpResponseMessage restResult = await _testClient.GetAsync("/rest/Book"); + // After hot-reload, the engine may still be re-initializing metadata providers. + // Poll the REST endpoint to allow time for the engine to become fully ready. + using HttpResponseMessage restResult = await WaitForRestEndpointAsync("/rest/Book", HttpStatusCode.OK); // Assert Assert.IsTrue(failedConfigLog.Contains(HOT_RELOAD_FAILURE_MESSAGE)); @@ -838,7 +820,9 @@ await WaitForConditionAsync( succeedConfigLog = _writer.ToString(); } - HttpResponseMessage restResult = await _testClient.GetAsync("/rest/Book"); + // After hot-reload, the engine may still be re-initializing metadata providers. + // Poll the REST endpoint to allow time for the engine to become fully ready. + using HttpResponseMessage restResult = await WaitForRestEndpointAsync("/rest/Book", HttpStatusCode.OK); // Assert Assert.IsTrue(failedConfigLog.Contains(HOT_RELOAD_FAILURE_MESSAGE)); @@ -974,4 +958,84 @@ private static async Task WaitForConditionAsync(Func condition, TimeSpan t throw new TimeoutException("The condition was not met within the timeout period."); } + + /// + /// Polls a REST endpoint until it returns the expected status code. + /// After a successful hot-reload, the engine may still be re-initializing + /// metadata providers, so an immediate request can intermittently fail. + /// + private static async Task WaitForRestEndpointAsync( + string requestUri, + HttpStatusCode expectedStatus, + int maxRetries = 5, + int delayMilliseconds = 1000) + { + HttpResponseMessage response = null; + for (int attempt = 1; attempt <= maxRetries; attempt++) + { + response = await _testClient.GetAsync(requestUri); + if (response.StatusCode == expectedStatus) + { + return response; + } + + Console.WriteLine($"REST {requestUri} returned {response.StatusCode} on attempt {attempt}/{maxRetries}, retrying..."); + + // Dispose unsuccessful responses to avoid leaking connections/sockets. + if (attempt < maxRetries) + { + response.Dispose(); + } + + await Task.Delay(delayMilliseconds); + } + + // Return the last response (undisposed) so the caller can inspect/assert on it. + return response; + } + + /// + /// Polls a GraphQL endpoint until it returns a valid response containing + /// the expected property. After a successful hot-reload, the engine may + /// still be re-initializing metadata providers, so an immediate request + /// can intermittently fail. PostGraphQLRequestAsync can also throw + /// (e.g. JsonException) if the server returns a non-JSON error response + /// during re-initialization. + /// + private static async Task<(bool Success, JsonElement Result)> WaitForGraphQLEndpointAsync( + string queryName, + string query, + string expectedProperty = "items", + int maxRetries = 5, + int delayMilliseconds = 1000) + { + JsonElement result = default; + for (int attempt = 1; attempt <= maxRetries; attempt++) + { + try + { + result = await GraphQLRequestExecutor.PostGraphQLRequestAsync( + _testClient, + _configProvider, + queryName, + query); + + if (result.ValueKind == JsonValueKind.Object && + result.TryGetProperty(expectedProperty, out _)) + { + return (true, result); + } + + Console.WriteLine($"GraphQL query returned {result.ValueKind} on attempt {attempt}/{maxRetries}, retrying..."); + } + catch (Exception ex) when (ex is JsonException || ex is HttpRequestException) + { + Console.WriteLine($"GraphQL request threw {ex.GetType().Name} on attempt {attempt}/{maxRetries}: {ex.Message}"); + } + + await Task.Delay(delayMilliseconds); + } + + return (false, result); + } }