areller
6 years ago
committed by
GitHub
GitHub
49 changed files with 2337 additions and 70 deletions
@ -0,0 +1,18 @@ |
|||
// Licensed to the .NET Foundation under one or more agreements.
|
|||
// The .NET Foundation licenses this file to you under the MIT license.
|
|||
// See the LICENSE file in the project root for more information.
|
|||
|
|||
using System.Collections.Generic; |
|||
using System.ComponentModel.DataAnnotations; |
|||
using System.Net.Http.Headers; |
|||
|
|||
namespace Microsoft.Tye.ConfigModel |
|||
{ |
|||
public class ConfigHttpProber |
|||
{ |
|||
[Required] public string Path { get; set; } = default!; |
|||
public int? Port { get; set; } |
|||
public string? Protocol { get; set; } |
|||
public List<KeyValuePair<string, object>> Headers { get; set; } = new List<KeyValuePair<string, object>>(); |
|||
} |
|||
} |
|||
@ -0,0 +1,16 @@ |
|||
// Licensed to the .NET Foundation under one or more agreements.
|
|||
// The .NET Foundation licenses this file to you under the MIT license.
|
|||
// See the LICENSE file in the project root for more information.
|
|||
|
|||
namespace Microsoft.Tye.ConfigModel |
|||
{ |
|||
public class ConfigProbe |
|||
{ |
|||
public ConfigHttpProber? Http { get; set; } |
|||
public int InitialDelay { get; set; } = 0; |
|||
public int Period { get; set; } = 1; |
|||
public int Timeout { get; set; } = 1; |
|||
public int SuccessThreshold { get; set; } = 1; |
|||
public int FailureThreshold { get; set; } = 3; |
|||
} |
|||
} |
|||
@ -0,0 +1,16 @@ |
|||
// Licensed to the .NET Foundation under one or more agreements.
|
|||
// The .NET Foundation licenses this file to you under the MIT license.
|
|||
// See the LICENSE file in the project root for more information.
|
|||
|
|||
using System.Collections.Generic; |
|||
|
|||
namespace Microsoft.Tye |
|||
{ |
|||
public class HttpProberBuilder |
|||
{ |
|||
public string Path { get; set; } = default!; |
|||
public int? Port { get; set; } |
|||
public string? Protocol { get; set; } |
|||
public List<KeyValuePair<string, object>> Headers { get; set; } = new List<KeyValuePair<string, object>>(); |
|||
} |
|||
} |
|||
@ -0,0 +1,16 @@ |
|||
// Licensed to the .NET Foundation under one or more agreements.
|
|||
// The .NET Foundation licenses this file to you under the MIT license.
|
|||
// See the LICENSE file in the project root for more information.
|
|||
|
|||
namespace Microsoft.Tye |
|||
{ |
|||
public class ProbeBuilder |
|||
{ |
|||
public HttpProberBuilder? Http { get; set; } |
|||
public int InitialDelay { get; set; } |
|||
public int Period { get; set; } |
|||
public int Timeout { get; set; } |
|||
public int SuccessThreshold { get; set; } |
|||
public int FailureThreshold { get; set; } |
|||
} |
|||
} |
|||
@ -0,0 +1,17 @@ |
|||
// Licensed to the .NET Foundation under one or more agreements.
|
|||
// The .NET Foundation licenses this file to you under the MIT license.
|
|||
// See the LICENSE file in the project root for more information.
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
|
|||
namespace Microsoft.Tye.Hosting.Model |
|||
{ |
|||
public class HttpProber |
|||
{ |
|||
public string Path { get; set; } = default!; |
|||
public int? Port { get; set; } |
|||
public string? Protocol { get; set; } |
|||
public List<KeyValuePair<string, object>> Headers { get; set; } = new List<KeyValuePair<string, object>>(); |
|||
} |
|||
} |
|||
@ -0,0 +1,18 @@ |
|||
// Licensed to the .NET Foundation under one or more agreements.
|
|||
// The .NET Foundation licenses this file to you under the MIT license.
|
|||
// See the LICENSE file in the project root for more information.
|
|||
|
|||
using System; |
|||
|
|||
namespace Microsoft.Tye.Hosting.Model |
|||
{ |
|||
public class Probe |
|||
{ |
|||
public HttpProber? Http { get; set; } |
|||
public TimeSpan InitialDelay { get; set; } |
|||
public TimeSpan Period { get; set; } |
|||
public TimeSpan Timeout { get; set; } |
|||
public int SuccessThreshold { get; set; } |
|||
public int FailureThreshold { get; set; } |
|||
} |
|||
} |
|||
@ -0,0 +1,13 @@ |
|||
// Licensed to the .NET Foundation under one or more agreements.
|
|||
// The .NET Foundation licenses this file to you under the MIT license.
|
|||
// See the LICENSE file in the project root for more information.
|
|||
|
|||
namespace Microsoft.Tye.Hosting.Model |
|||
{ |
|||
public class ReplicaBinding |
|||
{ |
|||
public int Port { get; set; } |
|||
public int ExternalPort { get; set; } |
|||
public string? Protocol { get; set; } |
|||
} |
|||
} |
|||
@ -0,0 +1,455 @@ |
|||
// Licensed to the .NET Foundation under one or more agreements.
|
|||
// The .NET Foundation licenses this file to you under the MIT license.
|
|||
// See the LICENSE file in the project root for more information.
|
|||
|
|||
using System; |
|||
using System.Collections.Concurrent; |
|||
using System.Linq; |
|||
using System.Net.Http; |
|||
using System.Reactive.Subjects; |
|||
using System.Threading; |
|||
using System.Threading.Tasks; |
|||
using Microsoft.Extensions.Logging; |
|||
using Microsoft.Tye.Hosting.Model; |
|||
|
|||
namespace Microsoft.Tye.Hosting |
|||
{ |
|||
public class ReplicaMonitor : IApplicationProcessor |
|||
{ |
|||
private ILogger _logger; |
|||
private ConcurrentDictionary<string, ReplicaMonitorState> _states; |
|||
|
|||
public ReplicaMonitor(ILogger logger) |
|||
{ |
|||
_logger = logger; |
|||
_states = new ConcurrentDictionary<string, ReplicaMonitorState>(); |
|||
} |
|||
|
|||
public Task StartAsync(Application application) |
|||
{ |
|||
foreach (var service in application.Services.Values) |
|||
{ |
|||
service.Items[typeof(Subscription)] = service.ReplicaEvents.Subscribe(OnReplicaChanged); |
|||
} |
|||
|
|||
return Task.CompletedTask; |
|||
} |
|||
|
|||
public Task StopAsync(Application application) |
|||
{ |
|||
foreach (var service in application.Services.Values) |
|||
{ |
|||
if (service.Items.TryGetValue(typeof(Subscription), out var item) && item is IDisposable disposable) |
|||
{ |
|||
disposable.Dispose(); |
|||
} |
|||
} |
|||
|
|||
return Task.CompletedTask; |
|||
} |
|||
|
|||
private void OnReplicaChanged(ReplicaEvent replicaEvent) |
|||
{ |
|||
switch (replicaEvent.State) |
|||
{ |
|||
case ReplicaState.Started: |
|||
_states.TryAdd(replicaEvent.Replica.Name, new ReplicaMonitorState(replicaEvent.Replica, _logger)); |
|||
break; |
|||
case ReplicaState.Stopped: |
|||
if (_states.TryRemove(replicaEvent.Replica.Name, out var stateToDispose)) |
|||
{ |
|||
stateToDispose.Dispose(); |
|||
} |
|||
break; |
|||
default: |
|||
if (_states.TryGetValue(replicaEvent.Replica.Name, out var state)) |
|||
{ |
|||
state.Update(replicaEvent); |
|||
} |
|||
break; |
|||
} |
|||
} |
|||
|
|||
private class Subscription |
|||
{ |
|||
} |
|||
|
|||
private class ReplicaMonitorState : IDisposable |
|||
{ |
|||
private ReplicaStatus _replica; |
|||
private ILogger _logger; |
|||
|
|||
private Prober? _livenessProber; |
|||
private Prober? _readinessProber; |
|||
private IDisposable? _livenessProberObserver; |
|||
private IDisposable? _readinessProberObserver; |
|||
|
|||
private ReplicaState _currentState; |
|||
private DateTime _lastStateChange; |
|||
private object _stateChangeLocker; |
|||
|
|||
public ReplicaMonitorState(ReplicaStatus replica, ILogger logger) |
|||
{ |
|||
_replica = replica; |
|||
_logger = logger; |
|||
|
|||
_currentState = ReplicaState.Started; |
|||
_lastStateChange = DateTime.Now; |
|||
_stateChangeLocker = new object(); |
|||
|
|||
Init(); |
|||
} |
|||
|
|||
private void Init() |
|||
{ |
|||
var serviceDesc = _replica.Service.Description; |
|||
if (serviceDesc.Liveness == null && serviceDesc.Readiness == null) |
|||
{ |
|||
MoveToReady(); |
|||
} |
|||
else if (serviceDesc.Liveness == null) |
|||
{ |
|||
MoveToHealthy(from: ReplicaState.Started); |
|||
StartReadinessProbe(serviceDesc.Readiness!); |
|||
} |
|||
else if (serviceDesc.Readiness == null) |
|||
{ |
|||
StartLivenessProbe(serviceDesc.Liveness, moveToOnSuccess: ReplicaState.Ready); |
|||
} |
|||
else |
|||
{ |
|||
StartLivenessProbe(serviceDesc.Liveness); |
|||
StartReadinessProbe(serviceDesc.Readiness); |
|||
} |
|||
} |
|||
|
|||
private void StartLivenessProbe(Probe probe, ReplicaState moveToOnSuccess = ReplicaState.Healthy) |
|||
{ |
|||
// currently only HTTP is available
|
|||
if (probe.Http == null) |
|||
{ |
|||
_logger.LogWarning("Cannot start probing replica {name} because probe configuration is not set", _replica.Name); |
|||
return; |
|||
} |
|||
|
|||
_livenessProber = new HttpProber(_replica, "liveness", probe, probe.Http, _logger); |
|||
|
|||
var failureThreshold = probe.FailureThreshold; |
|||
var failures = 0; |
|||
var dead = false; |
|||
|
|||
_livenessProberObserver = _livenessProber.ProbeResults.Subscribe(entry => |
|||
{ |
|||
if (dead) |
|||
{ |
|||
return; |
|||
} |
|||
|
|||
if (entry) |
|||
{ |
|||
// Reset failures count on success
|
|||
failures = 0; |
|||
} |
|||
|
|||
(var currentState, _) = ReadCurrentState(); |
|||
var failuresPastThreshold = failures >= failureThreshold; |
|||
switch ((entry, currentState, moveToOnSuccess, failuresPastThreshold)) |
|||
{ |
|||
case (false, _, _, true): |
|||
dead = true; |
|||
Kill(); |
|||
break; |
|||
case (false, _, _, false): |
|||
Interlocked.Increment(ref failures); |
|||
break; |
|||
case (true, ReplicaState.Started, ReplicaState.Ready, _): |
|||
case (true, ReplicaState.Healthy, ReplicaState.Ready, _): |
|||
MoveToReady(); |
|||
break; |
|||
case (true, ReplicaState.Started, ReplicaState.Healthy, _): |
|||
MoveToHealthy(from: ReplicaState.Started); |
|||
break; |
|||
} |
|||
}); |
|||
|
|||
_livenessProber.Start(); |
|||
} |
|||
|
|||
private void StartReadinessProbe(Probe probe) |
|||
{ |
|||
// currently only HTTP is available
|
|||
if (probe.Http == null) |
|||
{ |
|||
_logger.LogWarning("Cannot start probing replica {name} because probe configuration is not set", _replica.Name); |
|||
return; |
|||
} |
|||
|
|||
_readinessProber = new HttpProber(_replica, "readiness", probe, probe.Http, _logger); |
|||
|
|||
var successThreshold = probe.SuccessThreshold; |
|||
var failureThreshold = probe.FailureThreshold; |
|||
|
|||
var successes = 0; |
|||
var failures = 0; |
|||
|
|||
_readinessProberObserver = _readinessProber.ProbeResults.Subscribe(entry => |
|||
{ |
|||
if (entry) |
|||
{ |
|||
// Reset failures count on success
|
|||
failures = 0; |
|||
} |
|||
else |
|||
{ |
|||
// Reset successes count on failure
|
|||
successes = 0; |
|||
} |
|||
|
|||
(var currentState, _) = ReadCurrentState(); |
|||
var successesPastThreshold = successes >= successThreshold; |
|||
var failuresPastThreshold = failures >= failureThreshold; |
|||
switch ((entry, currentState, failuresPastThreshold, successesPastThreshold)) |
|||
{ |
|||
case (false, ReplicaState.Ready, true, _): |
|||
MoveToHealthy(from: ReplicaState.Ready); |
|||
break; |
|||
case (false, ReplicaState.Ready, false, _): |
|||
Interlocked.Increment(ref failures); |
|||
break; |
|||
case (true, ReplicaState.Healthy, _, true): |
|||
MoveToReady(); |
|||
break; |
|||
case (true, ReplicaState.Healthy, _, false): |
|||
Interlocked.Increment(ref successes); |
|||
break; |
|||
} |
|||
}); |
|||
|
|||
_readinessProber.Start(); |
|||
} |
|||
|
|||
private void MoveToHealthy(ReplicaState from) |
|||
{ |
|||
_logger.LogDebug("Replica {name} is moving to an healthy state", _replica.Name); |
|||
ChangeState(ReplicaState.Healthy); |
|||
} |
|||
|
|||
private void MoveToReady() |
|||
{ |
|||
_logger.LogDebug("Replica {name} is moving to a ready state", _replica.Name); |
|||
ChangeState(ReplicaState.Ready); |
|||
} |
|||
|
|||
private void Kill() |
|||
{ |
|||
_logger.LogDebug("Killing replica {name} because it has failed the liveness probe", _replica.Name); |
|||
|
|||
// it is assumed that a `Started` replica should have an initialized stopping token source
|
|||
_replica.StoppingTokenSource!.Cancel(); |
|||
} |
|||
|
|||
private void ChangeState(ReplicaState state) |
|||
{ |
|||
_replica.Service.ReplicaEvents.OnNext(new ReplicaEvent(state, _replica)); |
|||
lock (_stateChangeLocker) |
|||
{ |
|||
_currentState = state; |
|||
_lastStateChange = DateTime.Now; |
|||
} |
|||
} |
|||
|
|||
private (ReplicaState state, DateTime lastChanged) ReadCurrentState() |
|||
{ |
|||
lock (_stateChangeLocker) |
|||
{ |
|||
return (_currentState, _lastStateChange); |
|||
} |
|||
} |
|||
|
|||
public void Update(ReplicaEvent replicaEvent) |
|||
{ |
|||
} |
|||
|
|||
public void Dispose() |
|||
{ |
|||
_livenessProber?.Dispose(); |
|||
_readinessProber?.Dispose(); |
|||
_livenessProberObserver?.Dispose(); |
|||
_readinessProberObserver?.Dispose(); |
|||
} |
|||
} |
|||
|
|||
private abstract class Prober : IDisposable |
|||
{ |
|||
protected Prober() |
|||
{ |
|||
ProbeResults = new Subject<bool>(); |
|||
} |
|||
|
|||
public Subject<bool> ProbeResults { get; } |
|||
|
|||
public abstract void Start(); |
|||
|
|||
public abstract void Dispose(); |
|||
} |
|||
|
|||
private class HttpProber : Prober |
|||
{ |
|||
private static HttpClient _httpClient; |
|||
|
|||
static HttpProber() |
|||
{ |
|||
_httpClient = new HttpClient(); |
|||
} |
|||
|
|||
private ReplicaStatus _replica; |
|||
private ReplicaBinding? _selectedBinding; |
|||
private string _probeName; |
|||
private Probe _probe; |
|||
private Model.HttpProber _httpProberSettings; |
|||
|
|||
private Timer _probeTimer; |
|||
private CancellationTokenSource _cts; |
|||
|
|||
private ILogger _logger; |
|||
|
|||
private bool _lastStatus; |
|||
|
|||
public HttpProber(ReplicaStatus replica, string probeName, Probe probe, Model.HttpProber httpProberSettings, ILogger logger) |
|||
: base() |
|||
{ |
|||
_replica = replica; |
|||
_selectedBinding = null; |
|||
_probeName = probeName; |
|||
_probe = probe; |
|||
_httpProberSettings = httpProberSettings; |
|||
|
|||
_probeTimer = new Timer(DoProbe, null, Timeout.Infinite, Timeout.Infinite); |
|||
_cts = new CancellationTokenSource(); |
|||
|
|||
_logger = logger; |
|||
|
|||
_lastStatus = true; |
|||
} |
|||
|
|||
private void DoProbe(object? state) |
|||
{ |
|||
_probeTimer.Change(Timeout.Infinite, Timeout.Infinite); |
|||
_ = DoProbeAsync(); |
|||
} |
|||
|
|||
private async Task DoProbeAsync() |
|||
{ |
|||
if (_cts.Token.IsCancellationRequested) |
|||
{ |
|||
return; |
|||
} |
|||
|
|||
try |
|||
{ |
|||
var protocol = _selectedBinding!.Protocol; |
|||
var address = $"{protocol}://localhost:{_selectedBinding.Port}{_httpProberSettings.Path}"; |
|||
|
|||
using var timeoutCts = new CancellationTokenSource(_probe.Timeout); |
|||
var req = new HttpRequestMessage(HttpMethod.Get, address); |
|||
foreach (var header in _httpProberSettings.Headers) |
|||
{ |
|||
req.Headers.Add(header.Key, header.Value.ToString()); |
|||
} |
|||
|
|||
var res = await _httpClient.SendAsync(req, timeoutCts.Token); |
|||
if (!res.IsSuccessStatusCode) |
|||
{ |
|||
ShowWarning($"Replica {_replica.Name} failed http probe at address '{_httpProberSettings.Path}' due to a failed status ({res.StatusCode})"); |
|||
Send(false); |
|||
return; |
|||
} |
|||
|
|||
Send(true); |
|||
} |
|||
catch (HttpRequestException ex) |
|||
{ |
|||
ShowWarning($"Replica {_replica.Name} failed http probe at address '{_httpProberSettings.Path}' due to an http exception", ex); |
|||
Send(false); |
|||
} |
|||
catch (TaskCanceledException) |
|||
{ |
|||
ShowWarning($"Replica {_replica.Name} failed http probe at address '{_httpProberSettings.Path}' due to timeout"); |
|||
Send(false); |
|||
} |
|||
finally |
|||
{ |
|||
try |
|||
{ |
|||
_probeTimer.Change(_probe.Period, Timeout.InfiniteTimeSpan); |
|||
} |
|||
catch (ObjectDisposedException) |
|||
{ |
|||
} |
|||
} |
|||
} |
|||
|
|||
private void Send(bool status) |
|||
{ |
|||
ProbeResults.OnNext(status); |
|||
_lastStatus = status; |
|||
} |
|||
|
|||
private void ShowWarning(string message, Exception? ex = null) |
|||
{ |
|||
if (!_lastStatus) |
|||
{ |
|||
return; |
|||
} |
|||
|
|||
if (ex != null) |
|||
{ |
|||
_logger.LogWarning(ex, message); |
|||
} |
|||
else |
|||
{ |
|||
_logger.LogWarning(message); |
|||
} |
|||
} |
|||
|
|||
public override void Start() |
|||
{ |
|||
// the logic that selects the binding for the probing depends on the protocol and port fields that were provided in the probe
|
|||
// if neither port nor protocol were provided, we select the first binding
|
|||
// if just port was provided, we select the first binding with that port
|
|||
// if just protocol was provided, we select the first binding with that protocol (http/https)
|
|||
// if both port and protocol were provided, we select the first binding with that port and protocol
|
|||
Func<ReplicaBinding, bool> bindingClosure = (_httpProberSettings.Port.HasValue, _httpProberSettings.Protocol != null) switch |
|||
{ |
|||
(false, false) => _ => true, |
|||
(true, false) => r => r.ExternalPort == _httpProberSettings.Port!.Value, |
|||
(false, true) => r => r.Protocol == _httpProberSettings.Protocol!, |
|||
(true, true) => r => r.ExternalPort == _httpProberSettings.Port!.Value && r.Protocol == _httpProberSettings.Protocol! |
|||
}; |
|||
|
|||
var selectedBindings = _replica.Bindings.Where(bindingClosure); |
|||
if (selectedBindings.Count() == 0) |
|||
{ |
|||
_logger.LogWarning($"No suitable binding was found for replica {_replica.Name} for probe '{_probeName}'"); |
|||
return; |
|||
} |
|||
|
|||
_selectedBinding = selectedBindings.First(); |
|||
|
|||
try |
|||
{ |
|||
_probeTimer.Change(_probe.InitialDelay, Timeout.InfiniteTimeSpan); |
|||
} |
|||
catch (ObjectDisposedException) |
|||
{ |
|||
} |
|||
} |
|||
|
|||
public override void Dispose() |
|||
{ |
|||
_cts.Cancel(); |
|||
_probeTimer.Dispose(); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,453 @@ |
|||
// Licensed to the .NET Foundation under one or more agreements.
|
|||
// The .NET Foundation licenses this file to you under the MIT license.
|
|||
// See the LICENSE file in the project root for more information.
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.IO; |
|||
using System.Linq; |
|||
using System.Net; |
|||
using System.Net.Http; |
|||
using System.Text.Json; |
|||
using System.Text.Json.Serialization; |
|||
using System.Threading.Tasks; |
|||
using Microsoft.Tye; |
|||
using Microsoft.Tye.Hosting; |
|||
using Microsoft.Tye.Hosting.Model; |
|||
using Test.Infrastructure; |
|||
using Xunit; |
|||
using Xunit.Abstractions; |
|||
using static Test.Infrastructure.TestHelpers; |
|||
|
|||
namespace E2ETest |
|||
{ |
|||
public class HealthCheckTests |
|||
{ |
|||
private readonly ITestOutputHelper _output; |
|||
private readonly TestOutputLogEventSink _sink; |
|||
private readonly JsonSerializerOptions _options; |
|||
|
|||
private static readonly ReplicaState?[] startedOrHigher = new ReplicaState?[] { ReplicaState.Started, ReplicaState.Healthy, ReplicaState.Ready }; |
|||
private static readonly ReplicaState?[] stoppedOrLower = new ReplicaState?[] { ReplicaState.Stopped, ReplicaState.Removed }; |
|||
|
|||
private static HttpClient _client; |
|||
|
|||
static HealthCheckTests() |
|||
{ |
|||
var handler = new HttpClientHandler |
|||
{ |
|||
ServerCertificateCustomValidationCallback = (a, b, c, d) => true, |
|||
AllowAutoRedirect = false |
|||
}; |
|||
|
|||
_client = new HttpClient(new RetryHandler(handler)); |
|||
} |
|||
|
|||
public HealthCheckTests(ITestOutputHelper output) |
|||
{ |
|||
_output = output; |
|||
_sink = new TestOutputLogEventSink(output); |
|||
|
|||
_options = new JsonSerializerOptions() |
|||
{ |
|||
PropertyNameCaseInsensitive = true, |
|||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase, |
|||
WriteIndented = true, |
|||
}; |
|||
|
|||
_options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase)); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task ServiceWithoutLivenessReadinessShouldDefaultToReadyTests() |
|||
{ |
|||
using var projectDirectory = CopyTestProjectDirectory("health-checks"); |
|||
|
|||
var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-none.yaml")); |
|||
var outputContext = new OutputContext(_sink, Verbosity.Debug); |
|||
var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); |
|||
|
|||
await using var host = new TyeHost(application.ToHostingApplication(), new HostOptions()) |
|||
{ |
|||
Sink = _sink, |
|||
}; |
|||
|
|||
await StartHostAndWaitForReplicasToStart(host, new[] { "health-none" }, ReplicaState.Ready); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task ServiceWithoutLivenessShouldDefaultToHealthyTests() |
|||
{ |
|||
using var projectDirectory = CopyTestProjectDirectory("health-checks"); |
|||
|
|||
var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-readiness.yaml")); |
|||
var outputContext = new OutputContext(_sink, Verbosity.Debug); |
|||
var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); |
|||
|
|||
await using var host = new TyeHost(application.ToHostingApplication(), new HostOptions()) |
|||
{ |
|||
Sink = _sink, |
|||
}; |
|||
|
|||
await StartHostAndWaitForReplicasToStart(host, new[] { "health-readiness" }, ReplicaState.Healthy); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task ServicWithoutLivenessShouldBecomeReadyWhenReadyTests() |
|||
{ |
|||
using var projectDirectory = CopyTestProjectDirectory("health-checks"); |
|||
|
|||
var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-readiness.yaml")); |
|||
var outputContext = new OutputContext(_sink, Verbosity.Debug); |
|||
var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); |
|||
|
|||
await using var host = new TyeHost(application.ToHostingApplication(), new HostOptions()) |
|||
{ |
|||
Sink = _sink, |
|||
}; |
|||
|
|||
await StartHostAndWaitForReplicasToStart(host, new[] { "health-readiness" }, ReplicaState.Healthy); |
|||
|
|||
var replicas = host.Application.Services["health-readiness"].Replicas.Select(r => r.Value).ToList(); |
|||
Assert.True(await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Ready, replicas.Count, replicas.Select(r => r.Name).ToHashSet(), null, TimeSpan.Zero, async _ => |
|||
{ |
|||
await Task.WhenAll(replicas.Select(r => SetHealthyReadyInReplica(r, ready: true))); |
|||
})); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task ServiceWithoutReadinessShouldDefaultToReadyWhenHealthyTests() |
|||
{ |
|||
using var projectDirectory = CopyTestProjectDirectory("health-checks"); |
|||
|
|||
var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-liveness.yaml")); |
|||
var outputContext = new OutputContext(_sink, Verbosity.Debug); |
|||
var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); |
|||
|
|||
await using var host = new TyeHost(application.ToHostingApplication(), new HostOptions()) |
|||
{ |
|||
Sink = _sink, |
|||
}; |
|||
|
|||
await StartHostAndWaitForReplicasToStart(host, new[] { "health-liveness" }, ReplicaState.Started); |
|||
|
|||
var replicasToBecomeReady = host.Application.Services["health-liveness"].Replicas.Select(r => r.Value); |
|||
var replicasNamesToBecomeReady = replicasToBecomeReady.Select(r => r.Name).ToHashSet(); |
|||
Assert.True(await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Ready, replicasNamesToBecomeReady.Count, replicasNamesToBecomeReady, null, TimeSpan.Zero, async _ => |
|||
{ |
|||
await Task.WhenAll(replicasToBecomeReady.Select(r => SetHealthyReadyInReplica(r, healthy: true))); |
|||
})); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task ReadyServiceShouldBecomeHealthyWhenReadinessFailsTests() |
|||
{ |
|||
using var projectDirectory = CopyTestProjectDirectory("health-checks"); |
|||
|
|||
var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-all.yaml")); |
|||
var outputContext = new OutputContext(_sink, Verbosity.Debug); |
|||
var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); |
|||
|
|||
await using var host = new TyeHost(application.ToHostingApplication(), new HostOptions()) |
|||
{ |
|||
Sink = _sink, |
|||
}; |
|||
|
|||
SetReplicasInitialState(host, true, true); |
|||
|
|||
await StartHostAndWaitForReplicasToStart(host, new[] { "health-all" }, ReplicaState.Ready); |
|||
|
|||
var replicasToBecomeReady = host.Application.Services["health-all"].Replicas.Select(r => r.Value).ToList(); |
|||
var replicasNamesToBecomeReady = replicasToBecomeReady.Select(r => r.Name).ToHashSet(); |
|||
|
|||
var randomReplica = replicasToBecomeReady[new Random().Next(0, replicasToBecomeReady.Count)]; |
|||
|
|||
Assert.True(await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Healthy, 1, new[] { randomReplica.Name }.ToHashSet(), replicasNamesToBecomeReady.Where(r => r != randomReplica.Name).ToHashSet(), TimeSpan.FromSeconds(1), async _ => |
|||
{ |
|||
await SetHealthyReadyInReplica(randomReplica, ready: false); |
|||
})); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task ReadyServiceShouldRestartWhenLivenessFailsTests() |
|||
{ |
|||
using var projectDirectory = CopyTestProjectDirectory("health-checks"); |
|||
|
|||
var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-all.yaml")); |
|||
var outputContext = new OutputContext(_sink, Verbosity.Debug); |
|||
var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); |
|||
|
|||
await using var host = new TyeHost(application.ToHostingApplication(), new HostOptions()) |
|||
{ |
|||
Sink = _sink, |
|||
}; |
|||
|
|||
SetReplicasInitialState(host, true, true); |
|||
|
|||
await StartHostAndWaitForReplicasToStart(host, new[] { "health-all" }, ReplicaState.Ready); |
|||
|
|||
var replicasToBecomeReady = host.Application.Services["health-all"].Replicas.Select(r => r.Value).ToList(); |
|||
var replicasNamesToBecomeReady = replicasToBecomeReady.Select(r => r.Name).ToHashSet(); |
|||
|
|||
var randomReplica = replicasToBecomeReady[new Random().Next(0, replicasToBecomeReady.Count)]; |
|||
|
|||
Assert.True(await DoOperationAndWaitForReplicasToRestart(host, new[] { randomReplica.Name }.ToHashSet(), replicasNamesToBecomeReady.Where(r => r != randomReplica.Name).ToHashSet(), TimeSpan.FromSeconds(1), async _ => |
|||
{ |
|||
await SetHealthyReadyInReplica(randomReplica, healthy: false); |
|||
})); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task ProbeShouldRespectTimeoutTests() |
|||
{ |
|||
using var projectDirectory = CopyTestProjectDirectory("health-checks"); |
|||
|
|||
var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-all.yaml")); |
|||
var outputContext = new OutputContext(_sink, Verbosity.Debug); |
|||
var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); |
|||
|
|||
await using var host = new TyeHost(application.ToHostingApplication(), new HostOptions()) |
|||
{ |
|||
Sink = _sink, |
|||
}; |
|||
|
|||
SetReplicasInitialState(host, true, true); |
|||
|
|||
await StartHostAndWaitForReplicasToStart(host, new[] { "health-all" }, ReplicaState.Ready); |
|||
|
|||
var replicasToBecomeReady = host.Application.Services["health-all"].Replicas.Select(r => r.Value).ToList(); |
|||
var replicasNamesToBecomeReady = replicasToBecomeReady.Select(r => r.Name).ToHashSet(); |
|||
|
|||
var randomNumber = new Random().Next(0, replicasToBecomeReady.Count); |
|||
var randomReplica1 = replicasToBecomeReady[randomNumber]; |
|||
var randomReplica2 = replicasToBecomeReady[(randomNumber + 1) % replicasToBecomeReady.Count]; |
|||
|
|||
Assert.True(await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Healthy, 1, new[] { randomReplica1.Name }.ToHashSet(), replicasNamesToBecomeReady.Where(r => r != randomReplica1.Name).ToHashSet(), TimeSpan.FromSeconds(2), async _ => |
|||
{ |
|||
await Task.WhenAll(new[] |
|||
{ |
|||
SetHealthyReadyInReplica(randomReplica1, readyDelay: 2), |
|||
SetHealthyReadyInReplica(randomReplica2, readyDelay: 1) |
|||
}); |
|||
})); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task ProxyShouldNotProxyToNonReadyReplicasTests() |
|||
{ |
|||
using var projectDirectory = CopyTestProjectDirectory("health-checks"); |
|||
|
|||
var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-proxy.yaml")); |
|||
var outputContext = new OutputContext(_sink, Verbosity.Debug); |
|||
var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); |
|||
|
|||
await using var host = new TyeHost(application.ToHostingApplication(), new HostOptions()) |
|||
{ |
|||
Sink = _sink, |
|||
}; |
|||
|
|||
SetReplicasInitialState(host, true, true); |
|||
|
|||
await StartHostAndWaitForReplicasToStart(host, new[] { "health-proxy" }, ReplicaState.Ready); |
|||
|
|||
var replicasToBecomeReady = host.Application.Services["health-proxy"].Replicas.Select(r => r.Value).ToList(); |
|||
|
|||
// we assume that proxy will continue sending http request to the same replica
|
|||
var randomReplicaPortRes1 = await _client.GetAsync($"http://localhost:{host.Application.Services["health-proxy"].Description.Bindings.First().Port}/ports"); |
|||
var randomReplicaPort1 = JsonSerializer.Deserialize<int[]>(await randomReplicaPortRes1.Content.ReadAsStringAsync())[0]; |
|||
var randomReplica1 = replicasToBecomeReady.First(r => r.Bindings.Any(b => b.Port == randomReplicaPort1)); |
|||
|
|||
await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Healthy, 1, new[] { randomReplica1.Name }.ToHashSet(), null, TimeSpan.Zero, async _ => |
|||
{ |
|||
await SetHealthyReadyInReplica(randomReplica1, ready: false); |
|||
}); |
|||
|
|||
var randomReplicaPortRes2 = await _client.GetAsync($"http://localhost:{host.Application.Services["health-proxy"].Description.Bindings.First().Port}/ports"); |
|||
var randomReplicaPort2 = JsonSerializer.Deserialize<int[]>(await randomReplicaPortRes2.Content.ReadAsStringAsync())[0]; |
|||
var randomReplica2 = replicasToBecomeReady.First(r => r.Bindings.Any(b => b.Port == randomReplicaPort2)); |
|||
|
|||
Assert.NotEqual(randomReplicaPort1, randomReplicaPort2); |
|||
|
|||
await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Healthy, 1, new[] { randomReplica2.Name }.ToHashSet(), null, TimeSpan.Zero, async _ => |
|||
{ |
|||
await SetHealthyReadyInReplica(randomReplica2, ready: false); |
|||
}); |
|||
|
|||
try |
|||
{ |
|||
var resShouldFail = await _client.GetAsync($"http://localhost:{host.Application.Services["health-proxy"].Description.Bindings.First().Port}/ports"); |
|||
Assert.False(resShouldFail.IsSuccessStatusCode); |
|||
} |
|||
catch (HttpRequestException) |
|||
{ |
|||
} |
|||
|
|||
await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Ready, 1, new[] { randomReplica2.Name }.ToHashSet(), null, TimeSpan.Zero, async _ => |
|||
{ |
|||
await SetHealthyReadyInReplica(randomReplica2, ready: true); |
|||
}); |
|||
|
|||
var randomReplicaPortRes3 = await _client.GetAsync($"http://localhost:{host.Application.Services["health-proxy"].Description.Bindings.First().Port}/ports"); |
|||
var randomReplicaPort3 = JsonSerializer.Deserialize<int[]>(await randomReplicaPortRes3.Content.ReadAsStringAsync())[0]; |
|||
|
|||
Assert.Equal(randomReplicaPort3, randomReplicaPort2); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task IngressShouldNotProxyToNonReadyReplicasTests() |
|||
{ |
|||
using var projectDirectory = CopyTestProjectDirectory("health-checks"); |
|||
|
|||
var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-ingress.yaml")); |
|||
var outputContext = new OutputContext(_sink, Verbosity.Debug); |
|||
var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); |
|||
|
|||
await using var host = new TyeHost(application.ToHostingApplication(), new HostOptions()) |
|||
{ |
|||
Sink = _sink, |
|||
}; |
|||
|
|||
SetReplicasInitialState(host, true, true); |
|||
|
|||
await StartHostAndWaitForReplicasToStart(host, new[] { "health-ingress-svc" }, ReplicaState.Ready); |
|||
|
|||
var replicasToBecomeReady = host.Application.Services["health-ingress-svc"].Replicas.Select(r => r.Value).ToList(); |
|||
var ingressBinding = host.Application.Services.First(s => s.Value.Description.RunInfo is IngressRunInfo).Value.Description.Bindings.First(); |
|||
var uniqueIdUrl = $"{ingressBinding.Protocol}://localhost:{ingressBinding.Port}/api/id"; |
|||
|
|||
var uniqueIds = await ProbeNumberOfUniqueReplicas(uniqueIdUrl); |
|||
Assert.Equal(2, uniqueIds); |
|||
|
|||
var firstReplica = replicasToBecomeReady.First(); |
|||
var secondReplica = replicasToBecomeReady.Skip(1).First(); |
|||
|
|||
await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Healthy, 1, new[] { firstReplica.Name }.ToHashSet(), null, TimeSpan.Zero, async _ => |
|||
{ |
|||
await SetHealthyReadyInReplica(firstReplica, ready: false); |
|||
}); |
|||
|
|||
uniqueIds = await ProbeNumberOfUniqueReplicas(uniqueIdUrl); |
|||
Assert.Equal(1, uniqueIds); |
|||
|
|||
await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Healthy, 1, new[] { secondReplica.Name }.ToHashSet(), null, TimeSpan.Zero, async _ => |
|||
{ |
|||
await SetHealthyReadyInReplica(secondReplica, ready: false); |
|||
}); |
|||
|
|||
var res = await _client.GetAsync(uniqueIdUrl); |
|||
Assert.Equal(HttpStatusCode.BadGateway, res.StatusCode); |
|||
|
|||
await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Ready, 2, new[] { firstReplica.Name, secondReplica.Name }.ToHashSet(), null, TimeSpan.Zero, async _ => |
|||
{ |
|||
await SetHealthyReadyInReplica(firstReplica, ready: true); |
|||
await SetHealthyReadyInReplica(secondReplica, ready: true); |
|||
}); |
|||
|
|||
uniqueIds = await ProbeNumberOfUniqueReplicas(uniqueIdUrl); |
|||
Assert.Equal(2, uniqueIds); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task HeadersTests() |
|||
{ |
|||
using var projectDirectory = CopyTestProjectDirectory("health-checks"); |
|||
|
|||
var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-all.yaml")); |
|||
var outputContext = new OutputContext(_sink, Verbosity.Debug); |
|||
var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); |
|||
|
|||
await using var host = new TyeHost(application.ToHostingApplication(), new HostOptions()) |
|||
{ |
|||
Sink = _sink, |
|||
}; |
|||
|
|||
SetReplicasInitialState(host, true, true); |
|||
|
|||
await StartHostAndWaitForReplicasToStart(host, new[] { "health-all" }, ReplicaState.Ready); |
|||
|
|||
var res = await _client.GetAsync($"http://localhost:{host.Application.Services["health-all"].Description.Bindings.First().Port}/livenessHeaders"); |
|||
Assert.True(res.IsSuccessStatusCode); |
|||
|
|||
var headers = JsonSerializer.Deserialize<Dictionary<string, string>>(await res.Content.ReadAsStringAsync()); |
|||
Assert.Equal("value1", headers["name1"]); |
|||
Assert.Equal("value2", headers["name2"]); |
|||
|
|||
res = await _client.GetAsync($"http://localhost:{host.Application.Services["health-all"].Description.Bindings.First().Port}/readinessHeaders"); |
|||
Assert.True(res.IsSuccessStatusCode); |
|||
|
|||
headers = JsonSerializer.Deserialize<Dictionary<string, string>>(await res.Content.ReadAsStringAsync()); |
|||
Assert.Equal("value3", headers["name3"]); |
|||
Assert.Equal("value4", headers["name4"]); |
|||
} |
|||
|
|||
private async Task SetHealthyReadyInReplica(ReplicaStatus replica, bool? healthy = null, bool? ready = null, int? healthyDelay = null, int? readyDelay = null) |
|||
{ |
|||
var query = new List<string>(); |
|||
|
|||
if (healthy.HasValue) |
|||
{ |
|||
query.Add("healthy=" + healthy); |
|||
} |
|||
|
|||
if (ready.HasValue) |
|||
{ |
|||
query.Add("ready=" + ready); |
|||
} |
|||
|
|||
if (healthyDelay.HasValue) |
|||
{ |
|||
query.Add("healthyDelay=" + healthyDelay.Value); |
|||
} |
|||
|
|||
if (readyDelay.HasValue) |
|||
{ |
|||
query.Add("readyDelay=" + readyDelay.Value); |
|||
} |
|||
|
|||
await _client.GetAsync($"http://localhost:{replica.Ports.First()}/set?" + string.Join("&", query)); |
|||
} |
|||
|
|||
private async Task<int> ProbeNumberOfUniqueReplicas(string url) |
|||
{ |
|||
// this assumes roundrobin
|
|||
var unique = new HashSet<string>(); |
|||
|
|||
string? id = null; |
|||
while (id == null || unique.Add(id)) |
|||
{ |
|||
var res = await _client.GetAsync(url); |
|||
id = await res.Content.ReadAsStringAsync(); |
|||
} |
|||
|
|||
return unique.Count; |
|||
} |
|||
|
|||
private void SetReplicasInitialState(TyeHost host, bool? healthy, bool? ready, string[]? services = null) |
|||
{ |
|||
if (services == null) |
|||
{ |
|||
services = host.Application.Services.Select(s => s.Key).ToArray(); |
|||
} |
|||
else |
|||
{ |
|||
if (services.Any(s => !host.Application.Services.ContainsKey(s))) |
|||
{ |
|||
throw new ArgumentException($"not all services given in {nameof(services)} exist"); |
|||
} |
|||
} |
|||
|
|||
foreach (var service in services) |
|||
{ |
|||
if (healthy.HasValue) |
|||
{ |
|||
host.Application.Services[service].Description.Configuration.Add(new EnvironmentVariable("healthy") { Value = "true" }); |
|||
} |
|||
|
|||
if (ready.HasValue) |
|||
{ |
|||
host.Application.Services[service].Description.Configuration.Add(new EnvironmentVariable("ready") { Value = "true" }); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,143 @@ |
|||
// Licensed to the .NET Foundation under one or more agreements.
|
|||
// The .NET Foundation licenses this file to you under the MIT license.
|
|||
// See the LICENSE file in the project root for more information.
|
|||
|
|||
using System; |
|||
using System.IO; |
|||
using System.Linq; |
|||
using System.Net.Http; |
|||
using System.Text.Json; |
|||
using System.Text.Json.Serialization; |
|||
using System.Threading.Tasks; |
|||
using Microsoft.Tye; |
|||
using Microsoft.Tye.Hosting; |
|||
using Microsoft.Tye.Hosting.Model; |
|||
using Microsoft.Tye.Hosting.Model.V1; |
|||
using Test.Infrastructure; |
|||
using Xunit; |
|||
using Xunit.Abstractions; |
|||
using static Test.Infrastructure.TestHelpers; |
|||
|
|||
namespace E2ETest |
|||
{ |
|||
public class ReplicaStoppingTests |
|||
{ |
|||
private readonly ITestOutputHelper _output; |
|||
private readonly TestOutputLogEventSink _sink; |
|||
private readonly JsonSerializerOptions _options; |
|||
|
|||
private static readonly ReplicaState?[] startedOrHigher = new ReplicaState?[] { ReplicaState.Started, ReplicaState.Healthy, ReplicaState.Ready }; |
|||
private static readonly ReplicaState?[] stoppedOrLower = new ReplicaState?[] { ReplicaState.Stopped, ReplicaState.Removed }; |
|||
|
|||
public ReplicaStoppingTests(ITestOutputHelper output) |
|||
{ |
|||
_output = output; |
|||
_sink = new TestOutputLogEventSink(output); |
|||
|
|||
_options = new JsonSerializerOptions() |
|||
{ |
|||
PropertyNameCaseInsensitive = true, |
|||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase, |
|||
WriteIndented = true, |
|||
}; |
|||
|
|||
_options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase)); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task MultiProjectStoppingTests() |
|||
{ |
|||
using var projectDirectory = CopyTestProjectDirectory("health-checks"); |
|||
|
|||
var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-none.yaml")); |
|||
var outputContext = new OutputContext(_sink, Verbosity.Debug); |
|||
var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); |
|||
|
|||
await RunHostingApplication(application, new HostOptions(), async (host, uri) => |
|||
{ |
|||
var replicaToStop = host.Application.Services["health-none"].Replicas.First(); |
|||
Assert.Contains(replicaToStop.Value.State, startedOrHigher); |
|||
|
|||
var replicasToRestart = new[] { replicaToStop.Key }; |
|||
var restOfReplicas = host.Application.Services.SelectMany(s => s.Value.Replicas).Select(r => r.Value.Name).Where(r => r != replicaToStop.Key).ToArray(); |
|||
|
|||
Assert.True(await DoOperationAndWaitForReplicasToRestart(host, replicasToRestart.ToHashSet(), restOfReplicas.ToHashSet(), TimeSpan.FromSeconds(1), _ => |
|||
{ |
|||
replicaToStop.Value.StoppingTokenSource!.Cancel(); |
|||
return Task.CompletedTask; |
|||
})); |
|||
|
|||
Assert.Contains(replicaToStop.Value.State, stoppedOrLower); |
|||
Assert.True(host.Application.Services.SelectMany(s => s.Value.Replicas).All(r => startedOrHigher.Contains(r.Value.State))); |
|||
}); |
|||
} |
|||
|
|||
[ConditionalFact] |
|||
[SkipIfDockerNotRunning] |
|||
public async Task MultiProjectDockerStoppingTests() |
|||
{ |
|||
using var projectDirectory = CopyTestProjectDirectory("multi-project"); |
|||
|
|||
var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye.yaml")); |
|||
var outputContext = new OutputContext(_sink, Verbosity.Debug); |
|||
var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); |
|||
|
|||
await RunHostingApplication(application, new HostOptions() { Docker = true }, async (host, uri) => |
|||
{ |
|||
var replicaToStop = host.Application.Services["frontend"].Replicas.First(); |
|||
Assert.Contains(replicaToStop.Value.State, startedOrHigher); |
|||
|
|||
var replicasToRestart = new[] { replicaToStop.Key }; |
|||
var restOfReplicas = host.Application.Services.SelectMany(s => s.Value.Replicas).Select(r => r.Value.Name).Where(r => r != replicaToStop.Key).ToArray(); |
|||
|
|||
Assert.True(await DoOperationAndWaitForReplicasToRestart(host, replicasToRestart.ToHashSet(), restOfReplicas.ToHashSet(), TimeSpan.FromSeconds(1), _ => |
|||
{ |
|||
replicaToStop.Value.StoppingTokenSource!.Cancel(); |
|||
return Task.CompletedTask; |
|||
})); |
|||
|
|||
Assert.Contains(replicaToStop.Value.State, startedOrHigher); // when a container restarts, it assumes the same replica
|
|||
Assert.True(host.Application.Services.SelectMany(s => s.Value.Replicas).All(r => startedOrHigher.Contains(r.Value.State))); |
|||
}); |
|||
} |
|||
|
|||
private async Task RunHostingApplication(ApplicationBuilder application, HostOptions options, Func<TyeHost, Uri, Task> execute) |
|||
{ |
|||
await using var host = new TyeHost(application.ToHostingApplication(), options) |
|||
{ |
|||
Sink = _sink, |
|||
}; |
|||
|
|||
try |
|||
{ |
|||
await StartHostAndWaitForReplicasToStart(host); |
|||
|
|||
var uri = new Uri(host.DashboardWebApplication!.Addresses.First()); |
|||
|
|||
await execute(host, uri!); |
|||
} |
|||
finally |
|||
{ |
|||
if (host.DashboardWebApplication != null) |
|||
{ |
|||
var uri = new Uri(host.DashboardWebApplication!.Addresses.First()); |
|||
|
|||
using var client = new HttpClient(); |
|||
|
|||
foreach (var s in host.Application.Services.Values) |
|||
{ |
|||
var logs = await client.GetStringAsync(new Uri(uri, $"/api/v1/logs/{s.Description.Name}")); |
|||
|
|||
_output.WriteLine($"Logs for service: {s.Description.Name}"); |
|||
_output.WriteLine(logs); |
|||
|
|||
var description = await client.GetStringAsync(new Uri(uri, $"/api/v1/services/{s.Description.Name}")); |
|||
|
|||
_output.WriteLine($"Service defintion: {s.Description.Name}"); |
|||
_output.WriteLine(description); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,76 @@ |
|||
kind: Deployment |
|||
apiVersion: apps/v1 |
|||
metadata: |
|||
name: health-all |
|||
labels: |
|||
app.kubernetes.io/name: 'health-all' |
|||
app.kubernetes.io/part-of: 'health-checks' |
|||
spec: |
|||
replicas: 3 |
|||
selector: |
|||
matchLabels: |
|||
app.kubernetes.io/name: health-all |
|||
template: |
|||
metadata: |
|||
labels: |
|||
app.kubernetes.io/name: 'health-all' |
|||
app.kubernetes.io/part-of: 'health-checks' |
|||
spec: |
|||
containers: |
|||
- name: health-all |
|||
image: health-all:1.0.0 |
|||
imagePullPolicy: Always |
|||
env: |
|||
- name: ASPNETCORE_URLS |
|||
value: 'http://*:8004' |
|||
- name: PORT |
|||
value: '8004' |
|||
ports: |
|||
- containerPort: 8004 |
|||
livenessProbe: |
|||
httpGet: |
|||
path: /healthy |
|||
port: 8004 |
|||
scheme: HTTP |
|||
httpHeaders: |
|||
- name: name1 |
|||
value: value1 |
|||
- name: name2 |
|||
value: value2 |
|||
initialDelaySeconds: 5 |
|||
periodSeconds: 1 |
|||
successThreshold: 1 |
|||
failureThreshold: 1 |
|||
readinessProbe: |
|||
httpGet: |
|||
path: /ready |
|||
port: 8004 |
|||
scheme: HTTP |
|||
httpHeaders: |
|||
- name: name3 |
|||
value: value3 |
|||
- name: name4 |
|||
value: value4 |
|||
initialDelaySeconds: 5 |
|||
periodSeconds: 1 |
|||
successThreshold: 1 |
|||
failureThreshold: 1 |
|||
... |
|||
--- |
|||
kind: Service |
|||
apiVersion: v1 |
|||
metadata: |
|||
name: health-all |
|||
labels: |
|||
app.kubernetes.io/name: 'health-all' |
|||
app.kubernetes.io/part-of: 'health-checks' |
|||
spec: |
|||
selector: |
|||
app.kubernetes.io/name: health-all |
|||
type: ClusterIP |
|||
ports: |
|||
- name: http |
|||
protocol: TCP |
|||
port: 8004 |
|||
targetPort: 8004 |
|||
... |
|||
@ -0,0 +1,32 @@ |
|||
// Licensed to the .NET Foundation under one or more agreements.
|
|||
// The .NET Foundation licenses this file to you under the MIT license.
|
|||
// See the LICENSE file in the project root for more information.
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Linq; |
|||
using System.Threading.Tasks; |
|||
using Microsoft.AspNetCore.Hosting; |
|||
using Microsoft.AspNetCore.Server.Kestrel.Core; |
|||
using Microsoft.Extensions.Configuration; |
|||
using Microsoft.Extensions.Hosting; |
|||
using Microsoft.Extensions.Logging; |
|||
|
|||
namespace api |
|||
{ |
|||
public class Program |
|||
{ |
|||
public static void Main(string[] args) |
|||
{ |
|||
CreateHostBuilder(args).Build().Run(); |
|||
} |
|||
|
|||
public static IHostBuilder CreateHostBuilder(string[] args) => |
|||
Host.CreateDefaultBuilder(args) |
|||
.ConfigureWebHostDefaults(web => |
|||
{ |
|||
web.UseStartup<Startup>() |
|||
.ConfigureKestrel(options => {}); |
|||
}); |
|||
} |
|||
} |
|||
@ -0,0 +1,30 @@ |
|||
{ |
|||
"$schema": "http://json.schemastore.org/launchsettings.json", |
|||
"iisSettings": { |
|||
"windowsAuthentication": false, |
|||
"anonymousAuthentication": true, |
|||
"iisExpress": { |
|||
"applicationUrl": "http://localhost:39065", |
|||
"sslPort": 44337 |
|||
} |
|||
}, |
|||
"profiles": { |
|||
"IIS Express": { |
|||
"commandName": "IISExpress", |
|||
"launchBrowser": true, |
|||
"launchUrl": "weatherforecast", |
|||
"environmentVariables": { |
|||
"ASPNETCORE_ENVIRONMENT": "Development" |
|||
} |
|||
}, |
|||
"api": { |
|||
"commandName": "Project", |
|||
"launchBrowser": true, |
|||
"launchUrl": "weatherforecast", |
|||
"applicationUrl": "https://localhost:5001;http://localhost:5000", |
|||
"environmentVariables": { |
|||
"ASPNETCORE_ENVIRONMENT": "Development" |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,166 @@ |
|||
// Licensed to the .NET Foundation under one or more agreements.
|
|||
// The .NET Foundation licenses this file to you under the MIT license.
|
|||
// See the LICENSE file in the project root for more information.
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Linq; |
|||
using System.Text; |
|||
using System.Text.Json; |
|||
using System.Threading.Tasks; |
|||
using Microsoft.AspNetCore.Builder; |
|||
using Microsoft.AspNetCore.Hosting; |
|||
using Microsoft.AspNetCore.Http; |
|||
using Microsoft.AspNetCore.HttpsPolicy; |
|||
using Microsoft.AspNetCore.Mvc; |
|||
using Microsoft.Extensions.Configuration; |
|||
using Microsoft.Extensions.DependencyInjection; |
|||
using Microsoft.Extensions.Hosting; |
|||
using Microsoft.Extensions.Logging; |
|||
|
|||
namespace api |
|||
{ |
|||
public class Startup |
|||
{ |
|||
private static string _randomId = Guid.NewGuid().ToString(); |
|||
|
|||
private static bool _healthy = false; |
|||
private static bool _ready = false; |
|||
|
|||
private static int _healthyDelay = 0; |
|||
private static int _readyDelay = 0; |
|||
|
|||
private static Dictionary<string, string> _livenessHeaders; |
|||
private static Dictionary<string, string> _readinessHeaders; |
|||
|
|||
private static int[] _ports; |
|||
|
|||
private static object _locker = new object(); |
|||
|
|||
public Startup(IConfiguration configuration) |
|||
{ |
|||
Configuration = configuration; |
|||
|
|||
var healthyEnv = Environment.GetEnvironmentVariable("healthy"); |
|||
var readyEnv = Environment.GetEnvironmentVariable("ready"); |
|||
|
|||
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("PORT"))) |
|||
{ |
|||
var portParts = Environment.GetEnvironmentVariable("PORT").Split(';'); |
|||
_ports = portParts.Select(p => int.Parse(p)).ToArray(); |
|||
} |
|||
|
|||
if (healthyEnv != null) |
|||
{ |
|||
_healthy = bool.Parse(healthyEnv); |
|||
} |
|||
|
|||
if (readyEnv != null) |
|||
{ |
|||
_ready = bool.Parse(readyEnv); |
|||
} |
|||
} |
|||
|
|||
public IConfiguration Configuration { get; } |
|||
|
|||
// This method gets called by the runtime. Use this method to add services to the container.
|
|||
public void ConfigureServices(IServiceCollection services) |
|||
{ |
|||
services.AddControllers(); |
|||
} |
|||
|
|||
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
|
|||
public void Configure(IApplicationBuilder app, IWebHostEnvironment env) |
|||
{ |
|||
if (env.IsDevelopment()) |
|||
{ |
|||
app.UseDeveloperExceptionPage(); |
|||
} |
|||
|
|||
app.UseRouting(); |
|||
|
|||
app.UseAuthorization(); |
|||
|
|||
app.UseEndpoints(endpoints => |
|||
{ |
|||
endpoints.MapGet("/", async ctx => |
|||
{ |
|||
await ctx.Response.WriteAsync("Hello"); |
|||
}); |
|||
|
|||
endpoints.MapGet("/ports", async ctx => |
|||
{ |
|||
await ctx.Response.WriteAsync(JsonSerializer.Serialize(_ports)); |
|||
}); |
|||
|
|||
endpoints.MapGet("/id", async ctx => |
|||
{ |
|||
await ctx.Response.WriteAsync(_randomId); |
|||
}); |
|||
|
|||
endpoints.MapGet("/healthy", async ctx => |
|||
{ |
|||
if (_healthyDelay != 0) |
|||
{ |
|||
await Task.Delay(TimeSpan.FromSeconds(_healthyDelay)); |
|||
} |
|||
|
|||
_livenessHeaders = ctx.Request.Headers.ToDictionary(h => h.Key, h => h.Value.ToString()); |
|||
|
|||
ctx.Response.StatusCode = _healthy ? 200 : 500; |
|||
await ctx.Response.WriteAsync(ctx.Response.StatusCode.ToString()); |
|||
}); |
|||
|
|||
endpoints.MapGet("/ready", async ctx => |
|||
{ |
|||
if (_readyDelay != 0) |
|||
{ |
|||
await Task.Delay(TimeSpan.FromSeconds(_readyDelay)); |
|||
} |
|||
|
|||
_readinessHeaders = ctx.Request.Headers.ToDictionary(h => h.Key, h => h.Value.ToString()); |
|||
|
|||
ctx.Response.StatusCode = _ready ? 200 : 500; |
|||
await ctx.Response.WriteAsync(ctx.Response.StatusCode.ToString()); |
|||
}); |
|||
|
|||
// Should be technically POST/PUT, but it's just for tests...
|
|||
endpoints.MapGet("/set", async ctx => |
|||
{ |
|||
var query = ctx.Request.Query.ToDictionary(kv => kv.Key).ToDictionary(kv => kv.Key, kv => kv.Value.Value.First()); |
|||
if (query.ContainsKey("healthy")) |
|||
{ |
|||
_healthy = bool.Parse(query["healthy"]); |
|||
} |
|||
|
|||
if (query.ContainsKey("ready")) |
|||
{ |
|||
_ready = bool.Parse(query["ready"]); |
|||
} |
|||
|
|||
if (query.ContainsKey("healthyDelay")) |
|||
{ |
|||
_healthyDelay = int.Parse(query["healthyDelay"]); |
|||
} |
|||
|
|||
if (query.ContainsKey("readyDelay")) |
|||
{ |
|||
_readyDelay = int.Parse(query["readyDelay"]); |
|||
} |
|||
|
|||
await ctx.Response.WriteAsync(_randomId); |
|||
}); |
|||
|
|||
endpoints.MapGet("/livenessHeaders", async ctx => |
|||
{ |
|||
await ctx.Response.WriteAsync(JsonSerializer.Serialize(_livenessHeaders)); |
|||
}); |
|||
|
|||
endpoints.MapGet("/readinessHeaders", async ctx => |
|||
{ |
|||
await ctx.Response.WriteAsync(JsonSerializer.Serialize(_readinessHeaders)); |
|||
}); |
|||
}); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,8 @@ |
|||
<Project Sdk="Microsoft.NET.Sdk.Web"> |
|||
|
|||
<PropertyGroup> |
|||
<TargetFramework>netcoreapp3.1</TargetFramework> |
|||
</PropertyGroup> |
|||
|
|||
|
|||
</Project> |
|||
@ -0,0 +1,9 @@ |
|||
{ |
|||
"Logging": { |
|||
"LogLevel": { |
|||
"Default": "Information", |
|||
"Microsoft": "Warning", |
|||
"Microsoft.Hosting.Lifetime": "Information" |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,10 @@ |
|||
{ |
|||
"Logging": { |
|||
"LogLevel": { |
|||
"Default": "Information", |
|||
"Microsoft": "Warning", |
|||
"Microsoft.Hosting.Lifetime": "Information" |
|||
} |
|||
}, |
|||
"AllowedHosts": "*" |
|||
} |
|||
@ -0,0 +1,34 @@ |
|||
|
|||
Microsoft Visual Studio Solution File, Format Version 12.00 |
|||
# Visual Studio 15 |
|||
VisualStudioVersion = 15.0.26124.0 |
|||
MinimumVisualStudioVersion = 15.0.26124.0 |
|||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "api", "api\api.csproj", "{239E0475-3113-4C1C-A517-38AD9188B65D}" |
|||
EndProject |
|||
Global |
|||
GlobalSection(SolutionConfigurationPlatforms) = preSolution |
|||
Debug|Any CPU = Debug|Any CPU |
|||
Debug|x64 = Debug|x64 |
|||
Debug|x86 = Debug|x86 |
|||
Release|Any CPU = Release|Any CPU |
|||
Release|x64 = Release|x64 |
|||
Release|x86 = Release|x86 |
|||
EndGlobalSection |
|||
GlobalSection(SolutionProperties) = preSolution |
|||
HideSolutionNode = FALSE |
|||
EndGlobalSection |
|||
GlobalSection(ProjectConfigurationPlatforms) = postSolution |
|||
{239E0475-3113-4C1C-A517-38AD9188B65D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU |
|||
{239E0475-3113-4C1C-A517-38AD9188B65D}.Debug|Any CPU.Build.0 = Debug|Any CPU |
|||
{239E0475-3113-4C1C-A517-38AD9188B65D}.Debug|x64.ActiveCfg = Debug|Any CPU |
|||
{239E0475-3113-4C1C-A517-38AD9188B65D}.Debug|x64.Build.0 = Debug|Any CPU |
|||
{239E0475-3113-4C1C-A517-38AD9188B65D}.Debug|x86.ActiveCfg = Debug|Any CPU |
|||
{239E0475-3113-4C1C-A517-38AD9188B65D}.Debug|x86.Build.0 = Debug|Any CPU |
|||
{239E0475-3113-4C1C-A517-38AD9188B65D}.Release|Any CPU.ActiveCfg = Release|Any CPU |
|||
{239E0475-3113-4C1C-A517-38AD9188B65D}.Release|Any CPU.Build.0 = Release|Any CPU |
|||
{239E0475-3113-4C1C-A517-38AD9188B65D}.Release|x64.ActiveCfg = Release|Any CPU |
|||
{239E0475-3113-4C1C-A517-38AD9188B65D}.Release|x64.Build.0 = Release|Any CPU |
|||
{239E0475-3113-4C1C-A517-38AD9188B65D}.Release|x86.ActiveCfg = Release|Any CPU |
|||
{239E0475-3113-4C1C-A517-38AD9188B65D}.Release|x86.Build.0 = Release|Any CPU |
|||
EndGlobalSection |
|||
EndGlobal |
|||
@ -0,0 +1,36 @@ |
|||
# tye application configuration file |
|||
# read all about it at https://github.com/dotnet/tye |
|||
name: health-checks |
|||
services: |
|||
- name: health-all |
|||
project: api/api.csproj |
|||
replicas: 3 |
|||
bindings: |
|||
- port: 8004 |
|||
liveness: |
|||
http: |
|||
path: /healthy |
|||
headers: |
|||
- name: name1 |
|||
value: value1 |
|||
- name: name2 |
|||
value: value2 |
|||
initialDelay: 5 |
|||
period: 1 |
|||
timeout: 1 |
|||
successThreshold: 1 |
|||
failureThreshold: 1 |
|||
readiness: |
|||
http: |
|||
path: /ready |
|||
headers: |
|||
- name: name3 |
|||
value: value3 |
|||
- name: name4 |
|||
value: value4 |
|||
initialDelay: 5 |
|||
period: 1 |
|||
timeout: 2 |
|||
successThreshold: 1 |
|||
failureThreshold: 1 |
|||
|
|||
@ -0,0 +1,30 @@ |
|||
# tye application configuration file |
|||
# read all about it at https://github.com/dotnet/tye |
|||
name: health-checks |
|||
ingress: |
|||
- name: ingress |
|||
bindings: |
|||
- port: 8006 |
|||
rules: |
|||
- path: /api |
|||
service: health-ingress-svc |
|||
services: |
|||
- name: health-ingress-svc |
|||
project: api/api.csproj |
|||
replicas: 2 |
|||
liveness: |
|||
http: |
|||
path: /healthy |
|||
initialDelay: 5 |
|||
period: 1 |
|||
timeout: 1 |
|||
successThreshold: 1 |
|||
failureThreshold: 1 |
|||
readiness: |
|||
http: |
|||
path: /ready |
|||
initialDelay: 5 |
|||
period: 1 |
|||
timeout: 1 |
|||
successThreshold: 1 |
|||
failureThreshold: 1 |
|||
@ -0,0 +1,17 @@ |
|||
# tye application configuration file |
|||
# read all about it at https://github.com/dotnet/tye |
|||
name: health-checks |
|||
services: |
|||
- name: health-liveness |
|||
project: api/api.csproj |
|||
replicas: 3 |
|||
bindings: |
|||
- port: 8002 |
|||
liveness: |
|||
http: |
|||
path: /healthy |
|||
initialDelay: 5 |
|||
period: 1 |
|||
timeout: 1 |
|||
successThreshold: 1 |
|||
failureThreshold: 1 |
|||
@ -0,0 +1,10 @@ |
|||
# tye application configuration file |
|||
# read all about it at https://github.com/dotnet/tye |
|||
name: health-checks |
|||
services: |
|||
- name: health-none |
|||
project: api/api.csproj |
|||
replicas: 3 |
|||
bindings: |
|||
- port: 8001 |
|||
|
|||
@ -0,0 +1,26 @@ |
|||
# tye application configuration file |
|||
# read all about it at https://github.com/dotnet/tye |
|||
name: health-checks |
|||
services: |
|||
- name: health-proxy |
|||
project: api/api.csproj |
|||
replicas: 2 |
|||
bindings: |
|||
- port: 8005 |
|||
liveness: |
|||
http: |
|||
path: /healthy |
|||
initialDelay: 5 |
|||
period: 1 |
|||
timeout: 1 |
|||
successThreshold: 1 |
|||
failureThreshold: 1 |
|||
readiness: |
|||
http: |
|||
path: /ready |
|||
initialDelay: 5 |
|||
period: 1 |
|||
timeout: 1 |
|||
successThreshold: 1 |
|||
failureThreshold: 1 |
|||
|
|||
@ -0,0 +1,17 @@ |
|||
# tye application configuration file |
|||
# read all about it at https://github.com/dotnet/tye |
|||
name: health-checks |
|||
services: |
|||
- name: health-readiness |
|||
project: api/api.csproj |
|||
replicas: 3 |
|||
bindings: |
|||
- port: 8003 |
|||
readiness: |
|||
http: |
|||
path: /ready |
|||
initialDelay: 5 |
|||
period: 1 |
|||
timeout: 1 |
|||
successThreshold: 1 |
|||
failureThreshold: 1 |
|||
Loading…
Reference in new issue