Robots Txt settings.

src/Squidex/AppServices.cs

@@ -19,6 +19,7 @@ using Squidex.Domain.Apps.Entities.Contents;
 using Squidex.Extensions.Actions.Twitter;
 using Squidex.Infrastructure.Commands;
 using Squidex.Infrastructure.Diagnostics;
+using Squidex.Pipeline.Robots;
 
 namespace Squidex
 {
@@ -55,6 +56,8 @@ namespace Squidex
                 config.GetSection("mode"));
             services.Configure<TwitterOptions>(
                 config.GetSection("twitter"));
+            services.Configure<RobotsTxtOptions>(
+                config.GetSection("robots"));
             services.Configure<GCHealthCheckOptions>(
                 config.GetSection("healthz:gc"));
 

src/Squidex/Config/Web/WebExtensions.cs

@@ -9,6 +9,7 @@ using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.HttpOverrides;
 using Squidex.Pipeline;
 using Squidex.Pipeline.Diagnostics;
+using Squidex.Pipeline.Robots;
 
 namespace Squidex.Config.Web
 {
@@ -35,6 +36,13 @@ namespace Squidex.Config.Web
             return app;
         }
 
+        public static IApplicationBuilder UseMyRobotsTxt(this IApplicationBuilder app)
+        {
+            app.Map("/robots.txt", builder => builder.UseMiddleware<RobotsTxtMiddleware>());
+
+            return app;
+        }
+
         public static void UseMyCors(this IApplicationBuilder app)
         {
             app.UseCors(builder => builder

src/Squidex/Config/Web/WebServices.cs

@@ -10,6 +10,7 @@ using Microsoft.Extensions.DependencyInjection;
 using Squidex.Config.Domain;
 using Squidex.Pipeline;
 using Squidex.Pipeline.Diagnostics;
+using Squidex.Pipeline.Robots;
 
 namespace Squidex.Config.Web
 {
@@ -29,6 +30,9 @@ namespace Squidex.Config.Web
             services.AddSingletonAs<HealthCheckMiddleware>()
                 .AsSelf();
 
+            services.AddSingletonAs<RobotsTxtMiddleware>()
+                .AsSelf();
+
             services.AddSingletonAs<EnforceHttpsMiddleware>()
                 .AsSelf();
 

src/Squidex/Pipeline/Diagnostics/HealthCheckMiddleware.cs

@@ -36,22 +36,34 @@ namespace Squidex.Pipeline.Diagnostics
 
         public async Task InvokeAsync(HttpContext context, RequestDelegate next)
         {
-            using (var cts = new CancellationTokenSource(Timeout))
+            if (CanServeRequest(context.Request))
             {
-                var checks = await Task.WhenAll(healthChecks.Select(x => MakeHealthCheckAsync(x.Key, x.Value, cts.Token)));
+                using (var cts = new CancellationTokenSource(Timeout))
+                {
+                    var checks = await Task.WhenAll(healthChecks.Select(x => MakeHealthCheckAsync(x.Key, x.Value, cts.Token)));
 
-                context.Response.StatusCode = 200;
+                    context.Response.StatusCode = 200;
-                context.Response.Headers.Add("content-type", "application/json");
+                    context.Response.Headers.Add("content-type", "application/json");
 
-                if (checks.Any(x => !x.Result.IsHealthy))
+                    if (checks.Any(x => !x.Result.IsHealthy))
-                {
+                    {
-                    context.Response.StatusCode = 503;
+                        context.Response.StatusCode = 503;
-                }
+                    }
 
-                var response = checks.ToDictionary(x => x.Name, x => x.Result);
+                    var response = checks.ToDictionary(x => x.Name, x => x.Result);
 
-                await context.Response.WriteAsync(JsonConvert.SerializeObject(new { status = response }, Formatting.Indented, serializerSettings));
+                    await context.Response.WriteAsync(JsonConvert.SerializeObject(new { status = response }, Formatting.Indented, serializerSettings));
+                }
             }
+            else
+            {
+                await next(context);
+            }
+        }
+
+        private static bool CanServeRequest(HttpRequest request)
+        {
+            return request.Method == "GET" && (request.Path == "/" || string.IsNullOrEmpty(request.Path));
         }
 
         private static string GetName(IHealthCheck check)

src/Squidex/Pipeline/Robots/RobotsTxtMiddleware.cs

@@ -0,0 +1,46 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Options;
+using Squidex.Infrastructure;
+
+namespace Squidex.Pipeline.Robots
+{
+    public class RobotsTxtMiddleware : IMiddleware
+    {
+        private readonly RobotsTxtOptions robotsTxtOptions;
+
+        public RobotsTxtMiddleware(IOptions<RobotsTxtOptions> robotsTxtOptions)
+        {
+            Guard.NotNull(robotsTxtOptions, nameof(robotsTxtOptions));
+
+            this.robotsTxtOptions = robotsTxtOptions.Value;
+        }
+
+        public async Task InvokeAsync(HttpContext context, RequestDelegate next)
+        {
+            if (CanServeRequest(context.Request) && !string.IsNullOrWhiteSpace(robotsTxtOptions.Text))
+            {
+                context.Response.ContentType = "text/plain";
+                context.Response.StatusCode = 200;
+
+                await context.Response.WriteAsync(robotsTxtOptions.Text);
+            }
+            else
+            {
+                await next(context);
+            }
+        }
+
+        private static bool CanServeRequest(HttpRequest request)
+        {
+            return request.Method == "GET" && string.IsNullOrEmpty(request.Path);
+        }
+    }
+}

src/Squidex/Pipeline/Robots/RobotsTxtOptions.cs

@@ -0,0 +1,14 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+namespace Squidex.Pipeline.Robots
+{
+    public sealed class RobotsTxtOptions
+    {
+        public string Text { get; set; }
+    }
+}

src/Squidex/WebStartup.cs

@@ -42,6 +42,7 @@ namespace Squidex
             app.ApplicationServices.LogConfiguration();
 
             app.UseMyHealthCheck();
+            app.UseMyRobotsTxt();
             app.UseMyTracking();
             app.UseMyLocalCache();
             app.UseMyCors();

src/Squidex/appsettings.json

@@ -49,13 +49,20 @@
     }
   },
 
+  "robots": {
+    /*
+     * The text for the robots.txt file
+     */
+    "text": "User-agent: *\nDisallow: /"
+  },
+
   "healthz": {
     "gc": {
       /*
        * The maximum number of megabyte that the process can consume until it is marked as not healthy.
        */
-      "threshold": 4096
+    "threshold": 4096
-    }
+  }
   },
 
   "contentsController": {

src/Squidex/wwwroot/robots.txt

@@ -1,2 +0,0 @@
-User-agent: *
-Disallow: /