Dump endpoint (#882)

* Dump endpoint.

* Catch timer exceptions.

* Fix dockerfile.

Dockerfile

@@ -32,6 +32,12 @@ RUN dotnet test --no-restore --filter Category!=Dependencies
 # Publish
 RUN dotnet publish --no-restore src/Squidex/Squidex.csproj --output /build/ --configuration Release -p:version=$SQUIDEX__VERSION
 
+# Install tools
+RUN dotnet tool install --tool-path /tools dotnet-counters \
+ && dotnet tool install --tool-path /tools dotnet-dump \
+ && dotnet tool install --tool-path /tools dotnet-gcdump \
+ && dotnet tool install --tool-path /tools dotnet-trace
+
 
 #
 # Stage 2, Build Frontend
@@ -42,10 +48,8 @@ WORKDIR /src
 
 ENV CONTINUOUS_INTEGRATION=1
 
-# Copy Node project files.
+# Copy Node project files and patches
 COPY frontend/package*.json /tmp/
-
-# Copy patches for broken npm packages
 COPY frontend/patches /tmp/patches
 
 # Install Node packages 
@@ -66,20 +70,31 @@ RUN cp -a build /build/
 #
 FROM mcr.microsoft.com/dotnet/aspnet:6.0.0-bullseye-slim
 
-# Curl for debugging and libc-dev for Protobuf
+# Curl for debugging and libc-dev for protobuf
 RUN apt-get update \
  && apt-get install -y curl libc-dev
 
-# Default AspNetCore directory
+# Default tool directory
+WORKDIR /tools
+
+# Copy tools from backend build stage.
+COPY --from=backend /tools .
+
+# Default app directory
 WORKDIR /app
 
-# Copy from backend build stages
+# Copy backend files
 COPY --from=backend /build/ .
 
-# Copy from backend build stages to webserver folder
+# Copy frontend files to backend folder.
 COPY --from=frontend /build/ wwwroot/build/
 
 EXPOSE 80
 EXPOSE 11111
 
+ENV DIAGNOSTICS__GCDUMPTOOL=/TOOLS/DOTNET-GCDUMP
+ENV DIAGNOSTICS__DUMPTOOL=/TOOLS/DOTNET-DUMP
+ENV DIAGNOSTICS__TRACETOOL=/TOOLS/DOTNET-TRACE
+ENV DIAGNOSTICS__COUNTERSTOOL=/TOOLS/DOTNET-COUNTERS
+
 ENTRYPOINT ["dotnet", "Squidex.dll"]

backend/src/Squidex.Infrastructure/Dump/Dumper.cs

@@ -0,0 +1,157 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System.Diagnostics;
+using Microsoft.Extensions.Options;
+using Squidex.Assets;
+using Squidex.Hosting;
+
+namespace Squidex.Infrastructure.Dump
+{
+    public sealed class Dumper : IInitializable
+    {
+        private static readonly TimeSpan DefaultTimeout = TimeSpan.FromMinutes(30);
+        private readonly DumperOptions options;
+        private readonly IAssetStore assetStore;
+        private Task? scheduledGcDumpTask;
+        private Task? scheduledDumpTask;
+        private Timer? timer;
+
+        public int Order => int.MaxValue;
+
+        public Dumper(IOptions<DumperOptions> options, IAssetStore assetStore)
+        {
+            this.options = options.Value;
+            this.assetStore = assetStore;
+        }
+
+        public Task InitializeAsync(
+            CancellationToken ct)
+        {
+            if (options.DumpLimit > 0 || options.GCDumpLimit > 0)
+            {
+                timer = new Timer(CollectDump);
+                timer.Change(TimeSpan.Zero, TimeSpan.FromSeconds(5));
+            }
+
+            return Task.CompletedTask;
+        }
+
+        public Task ReleaseAsync(
+            CancellationToken ct)
+        {
+            var tasks = new List<Task>();
+
+            if (timer != null)
+            {
+                tasks.Add(timer.DisposeAsync().AsTask());
+            }
+
+            if (scheduledDumpTask != null)
+            {
+                tasks.Add(scheduledDumpTask);
+            }
+
+            if (scheduledGcDumpTask != null)
+            {
+                tasks.Add(scheduledGcDumpTask);
+            }
+
+            return Task.WhenAll(tasks);
+        }
+
+        private void CollectDump(object? state)
+        {
+            try
+            {
+                var usage = GC.GetTotalMemory(false) / (1024 * 1024);
+
+                if (options.DumpLimit > 0 && usage > options.DumpLimit && scheduledDumpTask == null)
+                {
+                    scheduledDumpTask = CreateDumpAsync();
+                }
+
+                if (options.GCDumpLimit > 0 && usage > options.GCDumpLimit && scheduledGcDumpTask == null)
+                {
+                    scheduledGcDumpTask = CreateGCDumpAsync();
+                }
+            }
+#pragma warning disable RECS0022 // A catch clause that catches System.Exception and has an empty body
+            catch
+#pragma warning restore RECS0022 // A catch clause that catches System.Exception and has an empty body
+            {
+
+            }
+        }
+
+        public Task<bool> CreateDumpAsync(
+            CancellationToken ct = default)
+        {
+            return CreateDumpAsync(options.DumpTool, "dump", ct);
+        }
+
+        public Task<bool> CreateGCDumpAsync(
+            CancellationToken ct = default)
+        {
+            return CreateDumpAsync(options.GcDumpTool, "gcdump", ct);
+        }
+
+        private async Task<bool> CreateDumpAsync(string? tool, string extension,
+            CancellationToken ct = default)
+        {
+            if (string.IsNullOrWhiteSpace(tool))
+            {
+                return false;
+            }
+
+            using var cts = new CancellationTokenSource(DefaultTimeout);
+            using var ctl = CancellationTokenSource.CreateLinkedTokenSource(cts.Token, ct);
+
+            var tempPath = Path.Combine(Path.GetTempPath(), Path.GetTempFileName());
+
+            var writtenFile = $"{tempPath}.{extension}";
+            try
+            {
+                using var process = new Process();
+                process.StartInfo.Arguments = $"collect -p {Environment.ProcessId} -o {tempPath}";
+                process.StartInfo.FileName = tool;
+                process.StartInfo.UseShellExecute = false;
+                process.Start();
+
+                await process.WaitForExitAsync(ctl.Token);
+
+                var isSucceess = process.ExitCode == 0;
+
+                if (!isSucceess)
+                {
+                    return false;
+                }
+
+                await using (var fs = new FileStream(writtenFile, FileMode.Open))
+                {
+                    var name = $"diagnostics/{extension}/{DateTime.UtcNow:yyyy-MM-dd-hh-mm-ss}.{extension}";
+
+                    await assetStore.UploadAsync(name, fs, true, ctl.Token);
+                }
+            }
+            finally
+            {
+                try
+                {
+                    File.Delete(tempPath);
+                }
+#pragma warning disable RECS0022 // A catch clause that catches System.Exception and has an empty body
+                catch
+#pragma warning restore RECS0022 // A catch clause that catches System.Exception and has an empty body
+                {
+                }
+            }
+
+            return true;
+        }
+    }
+}

backend/src/Squidex.Infrastructure/Dump/DumperOptions.cs

@@ -0,0 +1,20 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+namespace Squidex.Infrastructure.Dump
+{
+    public sealed class DumperOptions
+    {
+        public string? GcDumpTool { get; set; }
+
+        public string? DumpTool { get; set; }
+
+        public int GCDumpLimit { get; set; }
+
+        public int DumpLimit { get; set; }
+    }
+}

backend/src/Squidex/Areas/Api/Controllers/Diagnostics/DiagnosticsController.cs

@@ -0,0 +1,62 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using Microsoft.AspNetCore.Mvc;
+using Squidex.Infrastructure.Commands;
+using Squidex.Infrastructure.Dump;
+using Squidex.Shared;
+using Squidex.Web;
+
+namespace Squidex.Areas.Api.Controllers.Diagnostics
+{
+    /// <summary>
+    /// Makes a diagnostics request.
+    /// </summary>
+    [ApiExplorerSettings(GroupName = nameof(Diagnostics))]
+    public sealed class DiagnosticsController : ApiController
+    {
+        private readonly Dumper dumper;
+
+        public DiagnosticsController(ICommandBus commandBus, Dumper dumper)
+            : base(commandBus)
+        {
+            this.dumper = dumper;
+        }
+
+        /// <summary>
+        /// Creates a dump and writes it into storage..
+        /// </summary>
+        /// <returns>
+        /// 204 => Dump created successful.
+        /// </returns>
+        [HttpGet]
+        [Route("diagnostics/dump")]
+        [ApiPermissionOrAnonymous(Permissions.Admin)]
+        public async Task<IActionResult> GetDump()
+        {
+            await dumper.CreateDumpAsync(HttpContext.RequestAborted);
+
+            return NoContent();
+        }
+
+        /// <summary>
+        /// Creates a gc dump and writes it into storage.
+        /// </summary>
+        /// <returns>
+        /// 204 => Dump created successful.
+        /// </returns>
+        [HttpGet]
+        [Route("diagnostics/gc-dump")]
+        [ApiPermissionOrAnonymous(Permissions.Admin)]
+        public async Task<IActionResult> GetGCDump()
+        {
+            await dumper.CreateGCDumpAsync(HttpContext.RequestAborted);
+
+            return NoContent();
+        }
+    }
+}

backend/src/Squidex/Config/Domain/InfrastructureServices.cs

@@ -23,6 +23,7 @@ using Squidex.Domain.Apps.Entities.Rules.UsageTracking;
 using Squidex.Domain.Apps.Entities.Tags;
 using Squidex.Infrastructure;
 using Squidex.Infrastructure.Commands;
+using Squidex.Infrastructure.Dump;
 using Squidex.Infrastructure.EventSourcing.Grains;
 using Squidex.Infrastructure.Log;
 using Squidex.Infrastructure.Orleans;
@@ -50,6 +51,9 @@ namespace Squidex.Config.Domain
             services.Configure<JintScriptOptions>(config,
                 "scripting");
 
+            services.Configure<DumperOptions>(config,
+                "diagnostics");
+
             services.AddReplicatedCache();
             services.AddAsyncLocalCache();
             services.AddBackgroundCache();
@@ -63,6 +67,9 @@ namespace Squidex.Config.Domain
             services.AddSingletonAs<BackgroundRequestLogStore>()
                 .AsOptional<IRequestLogStore>();
 
+            services.AddSingletonAs<Dumper>()
+                .AsSelf();
+
             services.AddSingletonAs<ScriptingCompleter>()
                 .AsSelf();