From 9e5b52b77d04cf2c9b4c371c415aa6d18437fce2 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Sun, 11 Nov 2018 11:53:35 +0100 Subject: [PATCH] Robots Txt settings. --- src/Squidex/AppServices.cs | 3 ++ src/Squidex/Config/Web/WebExtensions.cs | 8 ++++ src/Squidex/Config/Web/WebServices.cs | 4 ++ .../Diagnostics/HealthCheckMiddleware.cs | 32 +++++++++---- .../Pipeline/Robots/RobotsTxtMiddleware.cs | 46 +++++++++++++++++++ .../Pipeline/Robots/RobotsTxtOptions.cs | 14 ++++++ src/Squidex/WebStartup.cs | 1 + src/Squidex/appsettings.json | 11 ++++- src/Squidex/wwwroot/robots.txt | 2 - 9 files changed, 107 insertions(+), 14 deletions(-) create mode 100644 src/Squidex/Pipeline/Robots/RobotsTxtMiddleware.cs create mode 100644 src/Squidex/Pipeline/Robots/RobotsTxtOptions.cs delete mode 100644 src/Squidex/wwwroot/robots.txt diff --git a/src/Squidex/AppServices.cs b/src/Squidex/AppServices.cs index 5dcfcf664..4b1dc563a 100644 --- a/src/Squidex/AppServices.cs +++ b/src/Squidex/AppServices.cs @@ -19,6 +19,7 @@ using Squidex.Domain.Apps.Entities.Contents; using Squidex.Extensions.Actions.Twitter; using Squidex.Infrastructure.Commands; using Squidex.Infrastructure.Diagnostics; +using Squidex.Pipeline.Robots; namespace Squidex { @@ -55,6 +56,8 @@ namespace Squidex config.GetSection("mode")); services.Configure( config.GetSection("twitter")); + services.Configure( + config.GetSection("robots")); services.Configure( config.GetSection("healthz:gc")); diff --git a/src/Squidex/Config/Web/WebExtensions.cs b/src/Squidex/Config/Web/WebExtensions.cs index dea5651d6..fa649cab3 100644 --- a/src/Squidex/Config/Web/WebExtensions.cs +++ b/src/Squidex/Config/Web/WebExtensions.cs @@ -9,6 +9,7 @@ using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.HttpOverrides; using Squidex.Pipeline; using Squidex.Pipeline.Diagnostics; +using Squidex.Pipeline.Robots; namespace Squidex.Config.Web { @@ -35,6 +36,13 @@ namespace Squidex.Config.Web return app; } + public static IApplicationBuilder UseMyRobotsTxt(this IApplicationBuilder app) + { + app.Map("/robots.txt", builder => builder.UseMiddleware()); + + return app; + } + public static void UseMyCors(this IApplicationBuilder app) { app.UseCors(builder => builder diff --git a/src/Squidex/Config/Web/WebServices.cs b/src/Squidex/Config/Web/WebServices.cs index af04bceed..1fae95e4f 100644 --- a/src/Squidex/Config/Web/WebServices.cs +++ b/src/Squidex/Config/Web/WebServices.cs @@ -10,6 +10,7 @@ using Microsoft.Extensions.DependencyInjection; using Squidex.Config.Domain; using Squidex.Pipeline; using Squidex.Pipeline.Diagnostics; +using Squidex.Pipeline.Robots; namespace Squidex.Config.Web { @@ -29,6 +30,9 @@ namespace Squidex.Config.Web services.AddSingletonAs() .AsSelf(); + services.AddSingletonAs() + .AsSelf(); + services.AddSingletonAs() .AsSelf(); diff --git a/src/Squidex/Pipeline/Diagnostics/HealthCheckMiddleware.cs b/src/Squidex/Pipeline/Diagnostics/HealthCheckMiddleware.cs index 2589612e6..5f13d366b 100644 --- a/src/Squidex/Pipeline/Diagnostics/HealthCheckMiddleware.cs +++ b/src/Squidex/Pipeline/Diagnostics/HealthCheckMiddleware.cs @@ -36,22 +36,34 @@ namespace Squidex.Pipeline.Diagnostics public async Task InvokeAsync(HttpContext context, RequestDelegate next) { - using (var cts = new CancellationTokenSource(Timeout)) + if (CanServeRequest(context.Request)) { - var checks = await Task.WhenAll(healthChecks.Select(x => MakeHealthCheckAsync(x.Key, x.Value, cts.Token))); + using (var cts = new CancellationTokenSource(Timeout)) + { + var checks = await Task.WhenAll(healthChecks.Select(x => MakeHealthCheckAsync(x.Key, x.Value, cts.Token))); - context.Response.StatusCode = 200; - context.Response.Headers.Add("content-type", "application/json"); + context.Response.StatusCode = 200; + context.Response.Headers.Add("content-type", "application/json"); - if (checks.Any(x => !x.Result.IsHealthy)) - { - context.Response.StatusCode = 503; - } + if (checks.Any(x => !x.Result.IsHealthy)) + { + context.Response.StatusCode = 503; + } - var response = checks.ToDictionary(x => x.Name, x => x.Result); + var response = checks.ToDictionary(x => x.Name, x => x.Result); - await context.Response.WriteAsync(JsonConvert.SerializeObject(new { status = response }, Formatting.Indented, serializerSettings)); + await context.Response.WriteAsync(JsonConvert.SerializeObject(new { status = response }, Formatting.Indented, serializerSettings)); + } } + else + { + await next(context); + } + } + + private static bool CanServeRequest(HttpRequest request) + { + return request.Method == "GET" && (request.Path == "/" || string.IsNullOrEmpty(request.Path)); } private static string GetName(IHealthCheck check) diff --git a/src/Squidex/Pipeline/Robots/RobotsTxtMiddleware.cs b/src/Squidex/Pipeline/Robots/RobotsTxtMiddleware.cs new file mode 100644 index 000000000..b00aea0bf --- /dev/null +++ b/src/Squidex/Pipeline/Robots/RobotsTxtMiddleware.cs @@ -0,0 +1,46 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Options; +using Squidex.Infrastructure; + +namespace Squidex.Pipeline.Robots +{ + public class RobotsTxtMiddleware : IMiddleware + { + private readonly RobotsTxtOptions robotsTxtOptions; + + public RobotsTxtMiddleware(IOptions robotsTxtOptions) + { + Guard.NotNull(robotsTxtOptions, nameof(robotsTxtOptions)); + + this.robotsTxtOptions = robotsTxtOptions.Value; + } + + public async Task InvokeAsync(HttpContext context, RequestDelegate next) + { + if (CanServeRequest(context.Request) && !string.IsNullOrWhiteSpace(robotsTxtOptions.Text)) + { + context.Response.ContentType = "text/plain"; + context.Response.StatusCode = 200; + + await context.Response.WriteAsync(robotsTxtOptions.Text); + } + else + { + await next(context); + } + } + + private static bool CanServeRequest(HttpRequest request) + { + return request.Method == "GET" && string.IsNullOrEmpty(request.Path); + } + } +} diff --git a/src/Squidex/Pipeline/Robots/RobotsTxtOptions.cs b/src/Squidex/Pipeline/Robots/RobotsTxtOptions.cs new file mode 100644 index 000000000..3d1972f79 --- /dev/null +++ b/src/Squidex/Pipeline/Robots/RobotsTxtOptions.cs @@ -0,0 +1,14 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +namespace Squidex.Pipeline.Robots +{ + public sealed class RobotsTxtOptions + { + public string Text { get; set; } + } +} diff --git a/src/Squidex/WebStartup.cs b/src/Squidex/WebStartup.cs index f080dac12..ff233f5f7 100644 --- a/src/Squidex/WebStartup.cs +++ b/src/Squidex/WebStartup.cs @@ -42,6 +42,7 @@ namespace Squidex app.ApplicationServices.LogConfiguration(); app.UseMyHealthCheck(); + app.UseMyRobotsTxt(); app.UseMyTracking(); app.UseMyLocalCache(); app.UseMyCors(); diff --git a/src/Squidex/appsettings.json b/src/Squidex/appsettings.json index 4f7fd5555..44c22f912 100644 --- a/src/Squidex/appsettings.json +++ b/src/Squidex/appsettings.json @@ -49,13 +49,20 @@ } }, + "robots": { + /* + * The text for the robots.txt file + */ + "text": "User-agent: *\nDisallow: /" + }, + "healthz": { "gc": { /* * The maximum number of megabyte that the process can consume until it is marked as not healthy. */ - "threshold": 4096 - } + "threshold": 4096 + } }, "contentsController": { diff --git a/src/Squidex/wwwroot/robots.txt b/src/Squidex/wwwroot/robots.txt deleted file mode 100644 index f870a67b1..000000000 --- a/src/Squidex/wwwroot/robots.txt +++ /dev/null @@ -1,2 +0,0 @@ -User-agent: * -Disallow: / \ No newline at end of file