From 93607b03ae015d90a5b0f39bb48c3e98aed4e837 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Mon, 6 Jul 2020 14:26:58 +0200 Subject: [PATCH 1/7] Scripting extensions. --- .../Extensions/StringJintExtension.cs | 105 ++++++++++++++++++ .../Extensions/StringWordsJintExtension.cs | 77 +++++++++++++ ...Squidex.Domain.Apps.Core.Operations.csproj | 2 + .../Config/Domain/InfrastructureServices.cs | 3 + .../RuleEventFormatterCompareTests.cs | 3 +- .../HandleRules/RuleEventFormatterTests.cs | 3 +- .../Scripting/JintScriptEngineHelperTests.cs | 71 +++++++++++- .../Scripting/JintScriptEngineTests.cs | 3 +- 8 files changed, 263 insertions(+), 4 deletions(-) create mode 100644 backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/StringWordsJintExtension.cs diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/StringJintExtension.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/StringJintExtension.cs index 4a7569bbc..79e1a067d 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/StringJintExtension.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/StringJintExtension.cs @@ -6,8 +6,12 @@ // ========================================================================== using System; +using System.IO; +using System.Text; +using HtmlAgilityPack; using Jint; using Jint.Native; +using Markdig; using Squidex.Infrastructure; namespace Squidex.Domain.Apps.Core.Scripting.Extensions @@ -52,12 +56,113 @@ namespace Squidex.Domain.Apps.Core.Scripting.Extensions } }; + private readonly Func html2Text = text => + { + try + { + var document = LoadHtml(text); + + var sb = new StringBuilder(); + + WriteTextTo(document.DocumentNode, sb); + + return sb.ToString().Trim(' ', '\n', '\r'); + } + catch + { + return JsValue.Undefined; + } + }; + + private static HtmlDocument LoadHtml(string text) + { + var document = new HtmlDocument(); + + document.LoadHtml(text); + + return document; + } + + private static void WriteTextTo(HtmlNode node, StringBuilder sb) + { + switch (node.NodeType) + { + case HtmlNodeType.Comment: + break; + case HtmlNodeType.Document: + WriteChildrenTextTo(node, sb); + break; + case HtmlNodeType.Text: + var html = ((HtmlTextNode)node).Text; + + if (HtmlNode.IsOverlappedClosingElement(html)) + { + break; + } + + if (!string.IsNullOrWhiteSpace(html)) + { + sb.Append(HtmlEntity.DeEntitize(html)); + } + + break; + + case HtmlNodeType.Element: + switch (node.Name) + { + case "p": + sb.AppendLine(); + break; + case "br": + sb.AppendLine(); + break; + case "style": + return; + case "script": + return; + } + + if (node.HasChildNodes) + { + WriteChildrenTextTo(node, sb); + } + + break; + } + } + + private static void WriteChildrenTextTo(HtmlNode node, StringBuilder sb) + { + foreach (var child in node.ChildNodes) + { + WriteTextTo(child, sb); + } + } + + private readonly Func markdown2Text = text => + { + try + { + return Markdown.ToPlainText(text).Trim(' ', '\n', '\r'); + } + catch + { + return JsValue.Undefined; + } + }; + + public Func Html2Text => html2Text; + public void Extend(Engine engine) { engine.SetValue("slugify", slugify); engine.SetValue("toCamelCase", toCamelCase); engine.SetValue("toPascalCase", toPascalCase); + + engine.SetValue("html2Text", Html2Text); + + engine.SetValue("markdown2Text", markdown2Text); } } } diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/StringWordsJintExtension.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/StringWordsJintExtension.cs new file mode 100644 index 000000000..2de45b6eb --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/StringWordsJintExtension.cs @@ -0,0 +1,77 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using Jint; +using Jint.Native; + +namespace Squidex.Domain.Apps.Core.Scripting.Extensions +{ + public sealed class StringWordsJintExtension : IJintExtension + { + private readonly Func wordCount = text => + { + try + { + var numWords = 0; + + for (int i = 1; i < text.Length; i++) + { + if (char.IsWhiteSpace(text[i - 1])) + { + var character = text[i]; + + if (char.IsLetterOrDigit(character) || char.IsPunctuation(character)) + { + numWords++; + } + } + } + + if (text.Length > 2) + { + numWords++; + } + + return numWords; + } + catch + { + return JsValue.Undefined; + } + }; + + private readonly Func characterCount = text => + { + try + { + var characterCount = 0; + + for (int i = 0; i < text.Length; i++) + { + if (char.IsLetter(text[i])) + { + characterCount++; + } + } + + return characterCount; + } + catch + { + return JsValue.Undefined; + } + }; + + public void Extend(Engine engine) + { + engine.SetValue("wordCount", wordCount); + + engine.SetValue("characterCount", characterCount); + } + } +} diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Squidex.Domain.Apps.Core.Operations.csproj b/backend/src/Squidex.Domain.Apps.Core.Operations/Squidex.Domain.Apps.Core.Operations.csproj index f51c23b51..256f8929f 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Squidex.Domain.Apps.Core.Operations.csproj +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Squidex.Domain.Apps.Core.Operations.csproj @@ -17,7 +17,9 @@ + + diff --git a/backend/src/Squidex/Config/Domain/InfrastructureServices.cs b/backend/src/Squidex/Config/Domain/InfrastructureServices.cs index 4d2c40d75..62702c299 100644 --- a/backend/src/Squidex/Config/Domain/InfrastructureServices.cs +++ b/backend/src/Squidex/Config/Domain/InfrastructureServices.cs @@ -71,6 +71,9 @@ namespace Squidex.Config.Domain services.AddSingletonAs() .As(); + services.AddSingletonAs() + .As(); + services.AddSingletonAs() .As(); diff --git a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterCompareTests.cs b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterCompareTests.cs index e6e24824e..a1dfae6b5 100644 --- a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterCompareTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterCompareTests.cs @@ -109,7 +109,8 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules { new DateTimeJintExtension(), new EventJintExtension(urlGenerator), - new StringJintExtension() + new StringJintExtension(), + new StringWordsJintExtension() }; var cache = new MemoryCache(Options.Create(new MemoryCacheOptions())); diff --git a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs index f57f7d6fb..7ceb15c23 100644 --- a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs @@ -103,7 +103,8 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules { new DateTimeJintExtension(), new EventJintExtension(urlGenerator), - new StringJintExtension() + new StringJintExtension(), + new StringWordsJintExtension() }; var cache = new MemoryCache(Options.Create(new MemoryCacheOptions())); diff --git a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/Scripting/JintScriptEngineHelperTests.cs b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/Scripting/JintScriptEngineHelperTests.cs index b31090f98..7678bcbf6 100644 --- a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/Scripting/JintScriptEngineHelperTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/Scripting/JintScriptEngineHelperTests.cs @@ -33,7 +33,8 @@ namespace Squidex.Domain.Apps.Core.Operations.Scripting { new DateTimeJintExtension(), new HttpJintExtension(httpClientFactory), - new StringJintExtension() + new StringJintExtension(), + new StringWordsJintExtension() }; var cache = new MemoryCache(Options.Create(new MemoryCacheOptions())); @@ -44,6 +45,74 @@ namespace Squidex.Domain.Apps.Core.Operations.Scripting }; } + [Fact] + public void Should_convert_html_to_text() + { + const string script = @" + return html2Text(value); + "; + + var vars = new ScriptVars + { + ["value"] = "

Hello World

" + }; + + var result = sut.Execute(vars, script).ToString(); + + Assert.Equal("Hello World", result); + } + + [Fact] + public void Should_convert_markdown_to_text() + { + const string script = @" + return markdown2Text(value); + "; + + var vars = new ScriptVars + { + ["value"] = "## Hello World" + }; + + var result = sut.Execute(vars, script).ToString(); + + Assert.Equal("Hello World", result); + } + + [Fact] + public void Should_count_words() + { + const string script = @" + return wordCount(value); + "; + + var vars = new ScriptVars + { + ["value"] = "Hello, World" + }; + + var result = ((JsonNumber)sut.Execute(vars, script)).Value; + + Assert.Equal(2, result); + } + + [Fact] + public void Should_count_characters() + { + const string script = @" + return characterCount(value); + "; + + var vars = new ScriptVars + { + ["value"] = "Hello, World" + }; + + var result = ((JsonNumber)sut.Execute(vars, script)).Value; + + Assert.Equal(10, result); + } + [Fact] public void Should_camel_case_value() { diff --git a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/Scripting/JintScriptEngineTests.cs b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/Scripting/JintScriptEngineTests.cs index 86ca2925f..25267a819 100644 --- a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/Scripting/JintScriptEngineTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/Scripting/JintScriptEngineTests.cs @@ -40,7 +40,8 @@ namespace Squidex.Domain.Apps.Core.Operations.Scripting { new DateTimeJintExtension(), new HttpJintExtension(httpClientFactory), - new StringJintExtension() + new StringJintExtension(), + new StringWordsJintExtension() }; var httpResponse = new HttpResponseMessage(HttpStatusCode.OK) From 69a49bcb1d2935949d7d1ebb3377895636462998 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 7 Jul 2020 12:31:43 +0200 Subject: [PATCH 2/7] Log bytes --- .../Apps/DefaultAppLogStore.cs | 37 ++++++---- .../Apps/IAppLogStore.cs | 3 +- .../Apps/RequestLog.cs | 30 +++++++++ .../Squidex.Web/Pipeline/UsageMiddleware.cs | 54 +++++++-------- .../src/Squidex.Web/Pipeline/UsageStream.cs | 4 +- .../Apps/DefaultAppLogStoreTests.cs | 31 +++++---- .../Pipeline/UsageMiddlewareTests.cs | 67 ++++++++++++++++++- 7 files changed, 167 insertions(+), 59 deletions(-) create mode 100644 backend/src/Squidex.Domain.Apps.Entities/Apps/RequestLog.cs diff --git a/backend/src/Squidex.Domain.Apps.Entities/Apps/DefaultAppLogStore.cs b/backend/src/Squidex.Domain.Apps.Entities/Apps/DefaultAppLogStore.cs index c70a3d40b..6819a0426 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Apps/DefaultAppLogStore.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Apps/DefaultAppLogStore.cs @@ -14,7 +14,6 @@ using System.Threading; using System.Threading.Tasks; using CsvHelper; using CsvHelper.Configuration; -using NodaTime; using Squidex.Infrastructure; using Squidex.Infrastructure.Log.Store; @@ -24,6 +23,7 @@ namespace Squidex.Domain.Apps.Entities.Apps { private const string FieldAuthClientId = "AuthClientId"; private const string FieldAuthUserId = "AuthUserId"; + private const string FieldBytes = "Bytes"; private const string FieldCosts = "Costs"; private const string FieldRequestElapsedMs = "RequestElapsedMs"; private const string FieldRequestMethod = "RequestMethod"; @@ -39,26 +39,27 @@ namespace Squidex.Domain.Apps.Entities.Apps this.requestLogStore = requestLogStore; } - public Task LogAsync(Guid appId, Instant timestamp, string? requestMethod, string? requestPath, string? userId, string? clientId, long elapsedMs, double costs) + public Task LogAsync(Guid appId, RequestLog request) { - var request = new Request + var storedRequest = new Request { Key = appId.ToString(), Properties = new Dictionary { - [FieldCosts] = costs.ToString(CultureInfo.InvariantCulture) + [FieldCosts] = request.Costs.ToString(CultureInfo.InvariantCulture) }, - Timestamp = timestamp + Timestamp = request.Timestamp }; - Append(request, FieldAuthClientId, clientId); - Append(request, FieldAuthUserId, userId); - Append(request, FieldCosts, costs.ToString(CultureInfo.InvariantCulture)); - Append(request, FieldRequestElapsedMs, elapsedMs.ToString(CultureInfo.InvariantCulture)); - Append(request, FieldRequestMethod, requestMethod); - Append(request, FieldRequestPath, requestPath); + Append(storedRequest, FieldAuthClientId, request.UserClientId); + Append(storedRequest, FieldAuthUserId, request.UserId); + Append(storedRequest, FieldBytes, request.Bytes); + Append(storedRequest, FieldCosts, request.Costs); + Append(storedRequest, FieldRequestElapsedMs, request.ElapsedMs); + Append(storedRequest, FieldRequestMethod, request.RequestMethod); + Append(storedRequest, FieldRequestPath, request.RequestPath); - return requestLogStore.LogAsync(request); + return requestLogStore.LogAsync(storedRequest); } public async Task ReadLogAsync(Guid appId, DateTime fromDate, DateTime toDate, Stream stream, CancellationToken ct = default) @@ -77,6 +78,7 @@ namespace Squidex.Domain.Apps.Entities.Apps csv.WriteField(FieldCosts); csv.WriteField(FieldAuthClientId); csv.WriteField(FieldAuthUserId); + csv.WriteField(FieldBytes); await csv.NextRecordAsync(); @@ -89,6 +91,7 @@ namespace Squidex.Domain.Apps.Entities.Apps csv.WriteField(GetDouble(request, FieldCosts)); csv.WriteField(GetString(request, FieldAuthClientId)); csv.WriteField(GetString(request, FieldAuthUserId)); + csv.WriteField(GetString(request, FieldBytes)); await csv.NextRecordAsync(); }, appId.ToString(), fromDate, toDate, ct); @@ -108,6 +111,16 @@ namespace Squidex.Domain.Apps.Entities.Apps } } + private static void Append(Request request, string key, double value) + { + request.Properties[key] = value.ToString(CultureInfo.InvariantCulture); + } + + private static void Append(Request request, string key, long value) + { + request.Properties[key] = value.ToString(CultureInfo.InvariantCulture); + } + private static string GetString(Request request, string key) { return request.Properties.GetValueOrDefault(key, string.Empty)!; diff --git a/backend/src/Squidex.Domain.Apps.Entities/Apps/IAppLogStore.cs b/backend/src/Squidex.Domain.Apps.Entities/Apps/IAppLogStore.cs index 1088ff0de..44ac03f47 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Apps/IAppLogStore.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Apps/IAppLogStore.cs @@ -9,13 +9,12 @@ using System; using System.IO; using System.Threading; using System.Threading.Tasks; -using NodaTime; namespace Squidex.Domain.Apps.Entities.Apps { public interface IAppLogStore { - Task LogAsync(Guid appId, Instant timestamp, string? requestMethod, string? requestPath, string? userId, string? clientId, long elapsedMs, double costs); + Task LogAsync(Guid appId, RequestLog request); Task ReadLogAsync(Guid appId, DateTime fromDate, DateTime toDate, Stream stream, CancellationToken ct = default); } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Apps/RequestLog.cs b/backend/src/Squidex.Domain.Apps.Entities/Apps/RequestLog.cs new file mode 100644 index 000000000..f737d90c2 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities/Apps/RequestLog.cs @@ -0,0 +1,30 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using NodaTime; + +namespace Squidex.Domain.Apps.Entities.Apps +{ + public struct RequestLog + { + public Instant Timestamp; + + public string? RequestMethod; + + public string? RequestPath; + + public string? UserId; + + public string? UserClientId; + + public long ElapsedMs; + + public long Bytes; + + public double Costs; + } +} diff --git a/backend/src/Squidex.Web/Pipeline/UsageMiddleware.cs b/backend/src/Squidex.Web/Pipeline/UsageMiddleware.cs index 24f993436..4202e8d17 100644 --- a/backend/src/Squidex.Web/Pipeline/UsageMiddleware.cs +++ b/backend/src/Squidex.Web/Pipeline/UsageMiddleware.cs @@ -18,17 +18,17 @@ namespace Squidex.Web.Pipeline { public sealed class UsageMiddleware : IMiddleware { - private readonly IAppLogStore log; + private readonly IAppLogStore logStore; private readonly IApiUsageTracker usageTracker; private readonly IClock clock; - public UsageMiddleware(IAppLogStore log, IApiUsageTracker usageTracker, IClock clock) + public UsageMiddleware(IAppLogStore logStore, IApiUsageTracker usageTracker, IClock clock) { - Guard.NotNull(log, nameof(log)); + Guard.NotNull(logStore, nameof(logStore)); Guard.NotNull(usageTracker, nameof(usageTracker)); Guard.NotNull(clock, nameof(clock)); - this.log = log; + this.logStore = logStore; this.usageTracker = usageTracker; @@ -51,37 +51,37 @@ namespace Squidex.Web.Pipeline { var appId = context.Features.Get()?.AppId; - var costs = context.Features.Get()?.Costs ?? 0; - if (appId != null) { - var elapsedMs = watch.Stop(); - - var now = clock.GetCurrentInstant(); + var bytes = usageBody.BytesWritten; - var userId = context.User.OpenIdSubject(); - var userClient = context.User.OpenIdClientId(); + if (context.Request.ContentLength != null) + { + bytes += context.Request.ContentLength.Value; + } - await log.LogAsync(appId.Id, now, - context.Request.Method, - context.Request.Path, - userId, - userClient, - elapsedMs, - costs); + var request = default(RequestLog); - if (costs > 0) - { - var bytes = usageBody.BytesWritten; + request.Bytes = bytes; + request.Costs = context.Features.Get()?.Costs ?? 0; + request.ElapsedMs = watch.Stop(); + request.RequestMethod = context.Request.Method; + request.RequestPath = context.Request.Path; + request.Timestamp = clock.GetCurrentInstant(); + request.UserClientId = context.User.OpenIdClientId(); + request.UserId = context.User.OpenIdSubject(); - if (context.Request.ContentLength != null) - { - bytes += context.Request.ContentLength.Value; - } + await logStore.LogAsync(appId.Id, request); - var date = now.ToDateTimeUtc().Date; + if (request.Costs > 0) + { + var date = request.Timestamp.ToDateTimeUtc().Date; - await usageTracker.TrackAsync(date, appId.Id.ToString(), userClient, costs, elapsedMs, bytes); + await usageTracker.TrackAsync(date, appId.Id.ToString(), + request.UserClientId, + request.Costs, + request.ElapsedMs, + request.Bytes); } } } diff --git a/backend/src/Squidex.Web/Pipeline/UsageStream.cs b/backend/src/Squidex.Web/Pipeline/UsageStream.cs index 4268008ee..2e878945a 100644 --- a/backend/src/Squidex.Web/Pipeline/UsageStream.cs +++ b/backend/src/Squidex.Web/Pipeline/UsageStream.cs @@ -64,7 +64,7 @@ namespace Squidex.Web.Pipeline public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { - await base.WriteAsync(buffer, offset, count, cancellationToken); + await inner.WriteAsync(buffer, offset, count, cancellationToken); bytesWritten += count; } @@ -78,7 +78,7 @@ namespace Squidex.Web.Pipeline public override async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) { - await base.WriteAsync(buffer, cancellationToken); + await inner.WriteAsync(buffer, cancellationToken); bytesWritten += buffer.Length; } diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/DefaultAppLogStoreTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/DefaultAppLogStoreTests.cs index a293a48f0..12c003cf7 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/DefaultAppLogStoreTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/DefaultAppLogStoreTests.cs @@ -33,22 +33,27 @@ namespace Squidex.Domain.Apps.Entities.Apps A.CallTo(() => requestLogStore.LogAsync(A._)) .Invokes((Request request) => recordedRequest = request); - var clientId = "frontend"; - var costs = 2; - var elapsedMs = 120; - var requestMethod = "GET"; - var requestPath = "/my-path"; - var userId = "user1"; - - await sut.LogAsync(Guid.NewGuid(), default, requestMethod, requestPath, userId, clientId, elapsedMs, costs); + var request = default(RequestLog); + request.Bytes = 1024; + request.Costs = 1.5; + request.ElapsedMs = 120; + request.RequestMethod = "GET"; + request.RequestPath = "/my-path"; + request.Timestamp = default; + request.UserClientId = "frontend"; + request.UserId = "user1"; + + await sut.LogAsync(Guid.NewGuid(), request); Assert.NotNull(recordedRequest); - Assert.Contains(clientId, recordedRequest!.Properties.Values); - Assert.Contains(costs.ToString(), recordedRequest!.Properties.Values); - Assert.Contains(elapsedMs.ToString(), recordedRequest!.Properties.Values); - Assert.Contains(requestMethod, recordedRequest!.Properties.Values); - Assert.Contains(requestPath, recordedRequest!.Properties.Values); + Assert.Contains(request.Bytes.ToString(), recordedRequest!.Properties.Values); + Assert.Contains(request.Costs.ToString(), recordedRequest!.Properties.Values); + Assert.Contains(request.ElapsedMs.ToString(), recordedRequest!.Properties.Values); + Assert.Contains(request.RequestMethod, recordedRequest!.Properties.Values); + Assert.Contains(request.RequestPath, recordedRequest!.Properties.Values); + Assert.Contains(request.UserClientId, recordedRequest!.Properties.Values); + Assert.Contains(request.UserId, recordedRequest!.Properties.Values); } [Fact] diff --git a/backend/tests/Squidex.Web.Tests/Pipeline/UsageMiddlewareTests.cs b/backend/tests/Squidex.Web.Tests/Pipeline/UsageMiddlewareTests.cs index 2cbdad475..6eb71bea5 100644 --- a/backend/tests/Squidex.Web.Tests/Pipeline/UsageMiddlewareTests.cs +++ b/backend/tests/Squidex.Web.Tests/Pipeline/UsageMiddlewareTests.cs @@ -6,6 +6,8 @@ // ========================================================================== using System; +using System.IO; +using System.Text; using System.Threading.Tasks; using FakeItEasy; using Microsoft.AspNetCore.Http; @@ -15,6 +17,8 @@ using Squidex.Infrastructure; using Squidex.Infrastructure.UsageTracking; using Xunit; +#pragma warning disable RECS0018 // Comparison of floating point numbers with equality operator + namespace Squidex.Web.Pipeline { public class UsageMiddlewareTests @@ -109,14 +113,35 @@ namespace Squidex.Web.Pipeline } [Fact] - public async Task Should_track_response_bytes() + public async Task Should_track_response_bytes_with_writer() + { + httpContext.Features.Set(new AppFeature(appId)); + httpContext.Features.Set(new ApiCostsAttribute(13)); + + await sut.InvokeAsync(httpContext, async x => + { + await x.Response.BodyWriter.WriteAsync(Encoding.Default.GetBytes("Hello World")); + + await next(x); + }); + + Assert.True(isNextCalled); + + var date = instant.ToDateTimeUtc().Date; + + A.CallTo(() => usageTracker.TrackAsync(date, A._, A._, 13, A._, 11)) + .MustHaveHappened(); + } + + [Fact] + public async Task Should_track_response_bytes_with_stream() { httpContext.Features.Set(new AppFeature(appId)); httpContext.Features.Set(new ApiCostsAttribute(13)); await sut.InvokeAsync(httpContext, async x => { - await x.Response.WriteAsync("Hello World"); + await x.Response.Body.WriteAsync(Encoding.Default.GetBytes("Hello World")); await next(x); }); @@ -129,6 +154,37 @@ namespace Squidex.Web.Pipeline .MustHaveHappened(); } + [Fact] + public async Task Should_track_response_bytes_with_file() + { + httpContext.Features.Set(new AppFeature(appId)); + httpContext.Features.Set(new ApiCostsAttribute(13)); + + var tempFileName = Path.GetTempFileName(); + try + { + File.WriteAllText(tempFileName, "Hello World"); + + await sut.InvokeAsync(httpContext, async x => + { + await x.Response.SendFileAsync(tempFileName, 0, new FileInfo(tempFileName).Length); + + await next(x); + }); + } + finally + { + File.Delete(tempFileName); + } + + Assert.True(isNextCalled); + + var date = instant.ToDateTimeUtc().Date; + + A.CallTo(() => usageTracker.TrackAsync(date, A._, A._, 13, A._, 11)) + .MustHaveHappened(); + } + [Fact] public async Task Should_not_track_if_costs_are_zero() { @@ -156,7 +212,12 @@ namespace Squidex.Web.Pipeline await sut.InvokeAsync(httpContext, next); - A.CallTo(() => appLogStore.LogAsync(appId.Id, instant, "GET", "/my-path", null, null, A._, 0)) + A.CallTo(() => appLogStore.LogAsync(appId.Id, + A.That.Matches(x => + x.Timestamp == instant && + x.RequestMethod == "GET" && + x.RequestPath == "/my-path" && + x.Costs == 0))) .MustHaveHappened(); } } From f77ed4de1c0e9c5a88ca1720c58066c7192458b1 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 7 Jul 2020 14:16:57 +0200 Subject: [PATCH 3/7] Liquid support for new text helpers. --- .../Extensions/StringJintExtension.cs | 79 +--------- .../Extensions/StringWordsJintExtension.cs | 34 +---- .../Extensions/StringFluidExtension.cs | 36 +++-- .../Extensions/StringWordsFluidExtension.cs | 31 ++++ .../TextHelpers.cs | 137 ++++++++++++++++++ .../Config/Domain/InfrastructureServices.cs | 7 + .../RuleEventFormatterCompareTests.cs | 1 + .../Templates/FluidTemplateEngineTests.cs | 64 +++++++- 8 files changed, 267 insertions(+), 122 deletions(-) create mode 100644 backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/StringWordsFluidExtension.cs create mode 100644 backend/src/Squidex.Domain.Apps.Core.Operations/TextHelpers.cs diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/StringJintExtension.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/StringJintExtension.cs index 79e1a067d..b5ddeaa57 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/StringJintExtension.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/StringJintExtension.cs @@ -6,12 +6,8 @@ // ========================================================================== using System; -using System.IO; -using System.Text; -using HtmlAgilityPack; using Jint; using Jint.Native; -using Markdig; using Squidex.Infrastructure; namespace Squidex.Domain.Apps.Core.Scripting.Extensions @@ -60,13 +56,7 @@ namespace Squidex.Domain.Apps.Core.Scripting.Extensions { try { - var document = LoadHtml(text); - - var sb = new StringBuilder(); - - WriteTextTo(document.DocumentNode, sb); - - return sb.ToString().Trim(' ', '\n', '\r'); + return TextHelpers.Html2Text(text); } catch { @@ -74,76 +64,11 @@ namespace Squidex.Domain.Apps.Core.Scripting.Extensions } }; - private static HtmlDocument LoadHtml(string text) - { - var document = new HtmlDocument(); - - document.LoadHtml(text); - - return document; - } - - private static void WriteTextTo(HtmlNode node, StringBuilder sb) - { - switch (node.NodeType) - { - case HtmlNodeType.Comment: - break; - case HtmlNodeType.Document: - WriteChildrenTextTo(node, sb); - break; - case HtmlNodeType.Text: - var html = ((HtmlTextNode)node).Text; - - if (HtmlNode.IsOverlappedClosingElement(html)) - { - break; - } - - if (!string.IsNullOrWhiteSpace(html)) - { - sb.Append(HtmlEntity.DeEntitize(html)); - } - - break; - - case HtmlNodeType.Element: - switch (node.Name) - { - case "p": - sb.AppendLine(); - break; - case "br": - sb.AppendLine(); - break; - case "style": - return; - case "script": - return; - } - - if (node.HasChildNodes) - { - WriteChildrenTextTo(node, sb); - } - - break; - } - } - - private static void WriteChildrenTextTo(HtmlNode node, StringBuilder sb) - { - foreach (var child in node.ChildNodes) - { - WriteTextTo(child, sb); - } - } - private readonly Func markdown2Text = text => { try { - return Markdown.ToPlainText(text).Trim(' ', '\n', '\r'); + return TextHelpers.Markdown2Text(text); } catch { diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/StringWordsJintExtension.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/StringWordsJintExtension.cs index 2de45b6eb..c355d88c2 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/StringWordsJintExtension.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/StringWordsJintExtension.cs @@ -17,27 +17,7 @@ namespace Squidex.Domain.Apps.Core.Scripting.Extensions { try { - var numWords = 0; - - for (int i = 1; i < text.Length; i++) - { - if (char.IsWhiteSpace(text[i - 1])) - { - var character = text[i]; - - if (char.IsLetterOrDigit(character) || char.IsPunctuation(character)) - { - numWords++; - } - } - } - - if (text.Length > 2) - { - numWords++; - } - - return numWords; + return TextHelpers.WordCount(text); } catch { @@ -49,17 +29,7 @@ namespace Squidex.Domain.Apps.Core.Scripting.Extensions { try { - var characterCount = 0; - - for (int i = 0; i < text.Length; i++) - { - if (char.IsLetter(text[i])) - { - characterCount++; - } - } - - return characterCount; + return TextHelpers.CharacterCount(text); } catch { diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/StringFluidExtension.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/StringFluidExtension.cs index ffed40632..585f8fe34 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/StringFluidExtension.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/StringFluidExtension.cs @@ -14,14 +14,7 @@ namespace Squidex.Domain.Apps.Core.Templates.Extensions { public sealed class StringFluidExtension : IFluidExtension { - public void RegisterGlobalTypes(IMemberAccessStrategy memberAccessStrategy) - { - TemplateContext.GlobalFilters.AddFilter("escape", Escape); - TemplateContext.GlobalFilters.AddFilter("slugify", Slugify); - TemplateContext.GlobalFilters.AddFilter("trim", Trim); - } - - public static FluidValue Slugify(FluidValue input, FilterArguments arguments, TemplateContext context) + private static readonly FilterDelegate Slugify = (input, arguments, context) => { if (input is StringValue value) { @@ -31,9 +24,9 @@ namespace Squidex.Domain.Apps.Core.Templates.Extensions } return input; - } + }; - public static FluidValue Escape(FluidValue input, FilterArguments arguments, TemplateContext context) + private static readonly FilterDelegate Escape = (input, arguments, context) => { var result = input.ToStringValue(); @@ -41,11 +34,30 @@ namespace Squidex.Domain.Apps.Core.Templates.Extensions result = result[1..^1]; return FluidValue.Create(result); - } + }; - public static FluidValue Trim(FluidValue input, FilterArguments arguments, TemplateContext context) + private static readonly FilterDelegate Markdown2Text = (input, arguments, context) => + { + return FluidValue.Create(TextHelpers.Markdown2Text(input.ToStringValue())); + }; + + private static readonly FilterDelegate Html2Text = (input, arguments, context) => + { + return FluidValue.Create(TextHelpers.Html2Text(input.ToStringValue())); + }; + + private static readonly FilterDelegate Trim = (input, arguments, context) => { return FluidValue.Create(input.ToStringValue().Trim()); + }; + + public void RegisterGlobalTypes(IMemberAccessStrategy memberAccessStrategy) + { + TemplateContext.GlobalFilters.AddFilter("html2text", Html2Text); + TemplateContext.GlobalFilters.AddFilter("markdown2text", Markdown2Text); + TemplateContext.GlobalFilters.AddFilter("escape", Escape); + TemplateContext.GlobalFilters.AddFilter("slugify", Slugify); + TemplateContext.GlobalFilters.AddFilter("trim", Trim); } } } diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/StringWordsFluidExtension.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/StringWordsFluidExtension.cs new file mode 100644 index 000000000..589008b56 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/StringWordsFluidExtension.cs @@ -0,0 +1,31 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using Fluid; +using Fluid.Values; + +namespace Squidex.Domain.Apps.Core.Templates.Extensions +{ + public class StringWordsFluidExtension : IFluidExtension + { + private static readonly FilterDelegate WordCount = (input, arguments, context) => + { + return FluidValue.Create(TextHelpers.WordCount(input.ToStringValue())); + }; + + private static readonly FilterDelegate CharacterCount = (input, arguments, context) => + { + return FluidValue.Create(TextHelpers.CharacterCount(input.ToStringValue())); + }; + + public void RegisterGlobalTypes(IMemberAccessStrategy memberAccessStrategy) + { + TemplateContext.GlobalFilters.AddFilter("word_count", WordCount); + TemplateContext.GlobalFilters.AddFilter("character_count", CharacterCount); + } + } +} diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/TextHelpers.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/TextHelpers.cs new file mode 100644 index 000000000..0c8171b46 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/TextHelpers.cs @@ -0,0 +1,137 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Text; +using HtmlAgilityPack; +using Markdig; + +namespace Squidex.Domain.Apps.Core +{ + public static class TextHelpers + { + public static string Markdown2Text(string markdown) + { + return Markdown.ToPlainText(markdown).Trim(' ', '\n', '\r'); + } + + public static string Html2Text(string html) + { + var document = LoadHtml(html); + + var sb = new StringBuilder(); + + WriteTextTo(document.DocumentNode, sb); + + return sb.ToString().Trim(' ', '\n', '\r'); + } + + private static HtmlDocument LoadHtml(string text) + { + var document = new HtmlDocument(); + + document.LoadHtml(text); + + return document; + } + + private static void WriteTextTo(HtmlNode node, StringBuilder sb) + { + switch (node.NodeType) + { + case HtmlNodeType.Comment: + break; + case HtmlNodeType.Document: + WriteChildrenTextTo(node, sb); + break; + case HtmlNodeType.Text: + var html = ((HtmlTextNode)node).Text; + + if (HtmlNode.IsOverlappedClosingElement(html)) + { + break; + } + + if (!string.IsNullOrWhiteSpace(html)) + { + sb.Append(HtmlEntity.DeEntitize(html)); + } + + break; + + case HtmlNodeType.Element: + switch (node.Name) + { + case "p": + sb.AppendLine(); + break; + case "br": + sb.AppendLine(); + break; + case "style": + return; + case "script": + return; + } + + if (node.HasChildNodes) + { + WriteChildrenTextTo(node, sb); + } + + break; + } + } + + private static void WriteChildrenTextTo(HtmlNode node, StringBuilder sb) + { + foreach (var child in node.ChildNodes) + { + WriteTextTo(child, sb); + } + } + + public static int CharacterCount(string text) + { + var characterCount = 0; + + for (int i = 0; i < text.Length; i++) + { + if (char.IsLetter(text[i])) + { + characterCount++; + } + } + + return characterCount; + } + + public static int WordCount(string text) + { + var numWords = 0; + + for (int i = 1; i < text.Length; i++) + { + if (char.IsWhiteSpace(text[i - 1])) + { + var character = text[i]; + + if (char.IsLetterOrDigit(character) || char.IsPunctuation(character)) + { + numWords++; + } + } + } + + if (text.Length > 2) + { + numWords++; + } + + return numWords; + } + } +} diff --git a/backend/src/Squidex/Config/Domain/InfrastructureServices.cs b/backend/src/Squidex/Config/Domain/InfrastructureServices.cs index 62702c299..e2c90bfb4 100644 --- a/backend/src/Squidex/Config/Domain/InfrastructureServices.cs +++ b/backend/src/Squidex/Config/Domain/InfrastructureServices.cs @@ -86,9 +86,16 @@ namespace Squidex.Config.Domain services.AddSingletonAs() .As(); + services.AddSingletonAs() + .As(); + + services.AddSingletonAs() + .As(); + services.AddSingletonAs() .As(); + services.AddSingleton>(DomainObjectGrainFormatter.Format); } diff --git a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterCompareTests.cs b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterCompareTests.cs index a1dfae6b5..9348c140c 100644 --- a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterCompareTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterCompareTests.cs @@ -97,6 +97,7 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules new DateTimeFluidExtension(), new EventFluidExtensions(urlGenerator), new StringFluidExtension(), + new StringWordsFluidExtension(), new UserFluidExtension() }; diff --git a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/Templates/FluidTemplateEngineTests.cs b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/Templates/FluidTemplateEngineTests.cs index 72c817403..cbd5961a1 100644 --- a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/Templates/FluidTemplateEngineTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/Templates/FluidTemplateEngineTests.cs @@ -24,7 +24,9 @@ namespace Squidex.Domain.Apps.Core.Operations.Templates { var extensions = new IFluidExtension[] { - new DateTimeFluidExtension() + new DateTimeFluidExtension(), + new StringFluidExtension(), + new StringWordsFluidExtension() }; sut = new FluidTemplateEngine(extensions); @@ -113,6 +115,66 @@ namespace Squidex.Domain.Apps.Core.Operations.Templates Assert.Equal("Hello", result); } + [Fact] + public async Task Should_format_html_to_text() + { + var template = "{{ e.text | html2text }}"; + + var value = new + { + Text = "

Hello World

" + }; + + var result = await RenderAync(template, value); + + Assert.Equal("Hello World", result); + } + + [Fact] + public async Task Should_convert_markdown_to_text() + { + var template = "{{ e.text | markdown2text }}"; + + var value = new + { + Text = "## Hello World" + }; + + var result = await RenderAync(template, value); + + Assert.Equal("Hello World", result); + } + + [Fact] + public async Task Should_format_word_count() + { + var template = "{{ e.text | word_count }}"; + + var value = new + { + Text = "Hello World" + }; + + var result = await RenderAync(template, value); + + Assert.Equal("2", result); + } + + [Fact] + public async Task Should_format_character_count() + { + var template = "{{ e.text | character_count }}"; + + var value = new + { + text = "Hello World" + }; + + var result = await RenderAync(template, value); + + Assert.Equal("10", result); + } + [Fact] public async Task Should_throw_exception_when_template_invalid() { From 2c6519b26fa2c85780237e20bd3f02d9f7947b41 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 7 Jul 2020 15:10:05 +0200 Subject: [PATCH 4/7] Styling fix. --- backend/src/Squidex/Config/Domain/InfrastructureServices.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/Squidex/Config/Domain/InfrastructureServices.cs b/backend/src/Squidex/Config/Domain/InfrastructureServices.cs index e2c90bfb4..23c5ca589 100644 --- a/backend/src/Squidex/Config/Domain/InfrastructureServices.cs +++ b/backend/src/Squidex/Config/Domain/InfrastructureServices.cs @@ -95,7 +95,6 @@ namespace Squidex.Config.Domain services.AddSingletonAs() .As(); - services.AddSingleton>(DomainObjectGrainFormatter.Format); } From c0cfabca12a7584532ac149e9bb228cb125b2fa4 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 7 Jul 2020 20:23:46 +0200 Subject: [PATCH 5/7] Optimize permission parsing. --- .../Security/Permission.Part.cs | 136 +++++++++++++++--- .../Security/PermissionTests.cs | 11 ++ .../TestSuite.LoadTests/ReadingBenchmarks.cs | 9 +- .../ReadingContentBenchmarks.cs | 10 +- .../TestSuite.LoadTests/ReadingFixture.cs | 4 +- .../TestSuite/TestSuite.LoadTests/Run.cs | 21 +-- .../Fixtures/ContentQueryFixture.cs | 2 +- 7 files changed, 155 insertions(+), 38 deletions(-) diff --git a/backend/src/Squidex.Infrastructure/Security/Permission.Part.cs b/backend/src/Squidex.Infrastructure/Security/Permission.Part.cs index eb63501b3..8b6934dd0 100644 --- a/backend/src/Squidex.Infrastructure/Security/Permission.Part.cs +++ b/backend/src/Squidex.Infrastructure/Security/Permission.Part.cs @@ -6,22 +6,23 @@ // ========================================================================== using System; -using System.Linq; namespace Squidex.Infrastructure.Security { public sealed partial class Permission { - internal struct Part + internal readonly struct Part { - private static readonly char[] AlternativeSeparators = { '|' }; - private static readonly char[] MainSeparators = { '.' }; + private const char SeparatorAlternative = '|'; + private const char SeparatorMain = '.'; + private const char CharAny = '*'; + private const char CharExclude = '^'; - public readonly string[]? Alternatives; + public readonly ReadOnlyMemory[]? Alternatives; public readonly bool Exclusion; - public Part(string[]? alternatives, bool exclusion) + public Part(ReadOnlyMemory[]? alternatives, bool exclusion) { Alternatives = alternatives; @@ -30,37 +31,118 @@ namespace Squidex.Infrastructure.Security public static Part[] ParsePath(string path) { - var parts = path.Split(MainSeparators, StringSplitOptions.RemoveEmptyEntries); + if (string.IsNullOrWhiteSpace(path)) + { + return Array.Empty(); + } + + var current = path.AsMemory(); + var currentSpan = current.Span; - var result = new Part[parts.Length]; + var result = new Part[CountOf(currentSpan, SeparatorMain) + 1]; - for (var i = 0; i < result.Length; i++) + if (result.Length == 1) { - result[i] = Parse(parts[i]); + result[0] = Parse(current); + } + else + { + for (int i = 0, j = 0; i < currentSpan.Length; i++) + { + if (currentSpan[i] == SeparatorMain) + { + result[j] = Parse(current.Slice(0, i)); + + current = current.Slice(i + 1); + currentSpan = current.Span; + + i = 0; + j++; + } + else if (i == currentSpan.Length - 1 || currentSpan[i] == SeparatorMain) + { + result[j] = Parse(current); + } + } } return result; } - public static Part Parse(string part) + public static Part Parse(ReadOnlyMemory current) { + var currentSpan = current.Span; + + if (currentSpan.Length == 0) + { + return new Part(Array.Empty>(), false); + } + var isExclusion = false; - if (part.StartsWith(Exclude, StringComparison.OrdinalIgnoreCase)) + if (currentSpan[0] == CharExclude) { isExclusion = true; - part = part.Substring(1); + current = current.Slice(1); + currentSpan = current.Span; + } + + if (currentSpan.Length == 0) + { + return new Part(Array.Empty>(), isExclusion); + } + + if (current.Length > 1 || currentSpan[0] != CharAny) + { + var alternatives = new ReadOnlyMemory[CountOf(currentSpan, SeparatorAlternative) + 1]; + + if (alternatives.Length == 1) + { + alternatives[0] = current; + } + else + { + for (int i = 0, j = 0; i < current.Length; i++) + { + if (currentSpan[i] == SeparatorAlternative) + { + alternatives[j] = current.Slice(0, i); + + current = current.Slice(i + 1); + currentSpan = current.Span; + + i = 0; + j++; + } + else if (i == current.Length - 1) + { + alternatives[j] = current; + } + } + } + + return new Part(alternatives, isExclusion); } + else + { + return new Part(null, isExclusion); + } + } - string[]? alternatives = null; + private static int CountOf(ReadOnlySpan text, char character) + { + var count = 0; - if (part != Any) + for (var i = 0; i < text.Length; i++) { - alternatives = part.Split(AlternativeSeparators, StringSplitOptions.RemoveEmptyEntries); + if (text[i] == character) + { + count++; + } } - return new Part(alternatives, isExclusion); + return count; } public static bool Intersects(ref Part lhs, ref Part rhs, bool allowNull) @@ -70,14 +152,28 @@ namespace Squidex.Infrastructure.Security return true; } - if (allowNull && rhs.Alternatives == null) + if (rhs.Alternatives == null) { - return true; + return allowNull; } var shouldIntersect = !(lhs.Exclusion ^ rhs.Exclusion); - return rhs.Alternatives != null && lhs.Alternatives.Intersect(rhs.Alternatives).Any() == shouldIntersect; + var isIntersected = false; + + for (var i = 0; i < lhs.Alternatives.Length; i++) + { + for (var j = 0; j < rhs.Alternatives.Length; j++) + { + if (lhs.Alternatives[i].Span.Equals(rhs.Alternatives[j].Span, StringComparison.OrdinalIgnoreCase)) + { + isIntersected = true; + break; + } + } + } + + return isIntersected == shouldIntersect; } } } diff --git a/backend/tests/Squidex.Infrastructure.Tests/Security/PermissionTests.cs b/backend/tests/Squidex.Infrastructure.Tests/Security/PermissionTests.cs index c3802ca81..6286f0223 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/Security/PermissionTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/Security/PermissionTests.cs @@ -166,5 +166,16 @@ namespace Squidex.Infrastructure.Security Assert.Equal(new List { source[2], source[1], source[0] }, sorted); } + + [Theory] + [InlineData("permission")] + [InlineData("permission...")] + [InlineData("permission.||..")] + public void Should_parse_invalid_permissions(string source) + { + var permission = new Permission(source); + + permission.Allows(new Permission(Permission.Any)); + } } } diff --git a/backend/tools/TestSuite/TestSuite.LoadTests/ReadingBenchmarks.cs b/backend/tools/TestSuite/TestSuite.LoadTests/ReadingBenchmarks.cs index ecfa0ee6a..f15cdbc24 100644 --- a/backend/tools/TestSuite/TestSuite.LoadTests/ReadingBenchmarks.cs +++ b/backend/tools/TestSuite/TestSuite.LoadTests/ReadingBenchmarks.cs @@ -9,6 +9,7 @@ using System.Collections.Generic; using System.Threading.Tasks; using TestSuite.Fixtures; using Xunit; +using Xunit.Abstractions; #pragma warning disable SA1300 // Element should begin with upper-case letter #pragma warning disable SA1507 // Code should not contain multiple blank lines in a row @@ -17,10 +18,14 @@ namespace TestSuite.LoadTests { public class ReadingBenchmarks : IClassFixture { + private readonly ITestOutputHelper testOutput; + public CreatedAppFixture _ { get; } - public ReadingBenchmarks(CreatedAppFixture fixture) + public ReadingBenchmarks(CreatedAppFixture fixture, ITestOutputHelper testOutput) { + this.testOutput = testOutput; + _ = fixture; } @@ -65,7 +70,7 @@ namespace TestSuite.LoadTests await Run.Parallel(numUsers, numIterationsPerUser, async () => { await _.Apps.GetClientsAsync(_.AppName); - }); + }, 100, testOutput); } } } diff --git a/backend/tools/TestSuite/TestSuite.LoadTests/ReadingContentBenchmarks.cs b/backend/tools/TestSuite/TestSuite.LoadTests/ReadingContentBenchmarks.cs index 8fe47b336..3a6156a47 100644 --- a/backend/tools/TestSuite/TestSuite.LoadTests/ReadingContentBenchmarks.cs +++ b/backend/tools/TestSuite/TestSuite.LoadTests/ReadingContentBenchmarks.cs @@ -64,7 +64,7 @@ namespace TestSuite.LoadTests { await Run.Parallel(numUsers, numIterationsPerUser, async () => { - await _.Contents.GetAsync(new ContentQuery { OrderBy = "data/value/iv asc" }); + await _.Contents.GetAsync(new ContentQuery { OrderBy = "data/number/iv asc" }); }); } @@ -74,7 +74,7 @@ namespace TestSuite.LoadTests { await Run.Parallel(numUsers, numIterationsPerUser, async () => { - await _.Contents.GetAsync(new ContentQuery { Skip = 5, OrderBy = "data/value/iv asc" }); + await _.Contents.GetAsync(new ContentQuery { Skip = 5, OrderBy = "data/number/iv asc" }); }); } @@ -84,7 +84,7 @@ namespace TestSuite.LoadTests { await Run.Parallel(numUsers, numIterationsPerUser, async () => { - await _.Contents.GetAsync(new ContentQuery { Skip = 2, Top = 5, OrderBy = "data/value/iv asc" }); + await _.Contents.GetAsync(new ContentQuery { Skip = 2, Top = 5, OrderBy = "data/number/iv asc" }); }); } @@ -94,7 +94,7 @@ namespace TestSuite.LoadTests { await Run.Parallel(numUsers, numIterationsPerUser, async () => { - await _.Contents.GetAsync(new ContentQuery { Skip = 2, Top = 5, OrderBy = "data/value/iv desc" }); + await _.Contents.GetAsync(new ContentQuery { Skip = 2, Top = 5, OrderBy = "data/number/iv desc" }); }); } @@ -104,7 +104,7 @@ namespace TestSuite.LoadTests { await Run.Parallel(numUsers, numIterationsPerUser, async () => { - await _.Contents.GetAsync(new ContentQuery { Filter = "data/value/iv gt 3 and data/value/iv lt 7", OrderBy = "data/value/iv asc" }); + await _.Contents.GetAsync(new ContentQuery { Filter = "data/number/iv gt 3 and data/number/iv lt 7", OrderBy = "data/number/iv asc" }); }); } } diff --git a/backend/tools/TestSuite/TestSuite.LoadTests/ReadingFixture.cs b/backend/tools/TestSuite/TestSuite.LoadTests/ReadingFixture.cs index d1765c6a3..2e97d0b52 100644 --- a/backend/tools/TestSuite/TestSuite.LoadTests/ReadingFixture.cs +++ b/backend/tools/TestSuite/TestSuite.LoadTests/ReadingFixture.cs @@ -9,10 +9,10 @@ using TestSuite.Fixtures; namespace TestSuite.LoadTests { - public sealed class ReadingFixture : ContentFixture + public sealed class ReadingFixture : ContentQueryFixture { public ReadingFixture() - : base("benchmark_reading") + : base("benchmark-reading") { } } diff --git a/backend/tools/TestSuite/TestSuite.LoadTests/Run.cs b/backend/tools/TestSuite/TestSuite.LoadTests/Run.cs index f3d421d6a..95f150b29 100644 --- a/backend/tools/TestSuite/TestSuite.LoadTests/Run.cs +++ b/backend/tools/TestSuite/TestSuite.LoadTests/Run.cs @@ -13,13 +13,16 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; using Xunit; +using Xunit.Abstractions; namespace TestSuite.LoadTests { public static class Run { - public static async Task Parallel(int numUsers, int numIterationsPerUser, Func action, int expectedAvg = 100) + public static async Task Parallel(int numUsers, int numIterationsPerUser, Func action, int expectedAvg = 100, ITestOutputHelper testOutput = null) { + await action(); + var elapsedMs = new ConcurrentBag(); var errors = 0; @@ -56,16 +59,18 @@ namespace TestSuite.LoadTests var count = elapsedMs.Count; - var max = elapsedMs.Max(); - var min = elapsedMs.Min(); - var avg = elapsedMs.Average(); - Assert.Equal(0, errors); - Assert.Equal(count, numUsers * numIterationsPerUser); + if (testOutput != null) + { + testOutput.WriteLine("Total Errors: {0}/{1}", errors, numUsers * numIterationsPerUser); + testOutput.WriteLine("Total Count: {0}/{1}", count, numUsers * numIterationsPerUser); - Assert.InRange(max, 0, expectedAvg * 10); - Assert.InRange(min, 0, expectedAvg); + testOutput.WriteLine(string.Empty); + testOutput.WriteLine("Performance Average: {0}", avg); + testOutput.WriteLine("Performance Max: {0}", elapsedMs.Max()); + testOutput.WriteLine("Performance Min: {0}", elapsedMs.Min()); + } Assert.InRange(avg, 0, expectedAvg); } diff --git a/backend/tools/TestSuite/TestSuite.Shared/Fixtures/ContentQueryFixture.cs b/backend/tools/TestSuite/TestSuite.Shared/Fixtures/ContentQueryFixture.cs index 13e834f58..3d88e3463 100644 --- a/backend/tools/TestSuite/TestSuite.Shared/Fixtures/ContentQueryFixture.cs +++ b/backend/tools/TestSuite/TestSuite.Shared/Fixtures/ContentQueryFixture.cs @@ -13,7 +13,7 @@ namespace TestSuite.Fixtures public class ContentQueryFixture : ContentFixture { public ContentQueryFixture() - : this("my-reads") + : this("my-reads") { } From c8803aacfdf61930b35a308a8304936b00dff688 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 8 Jul 2020 11:17:39 +0200 Subject: [PATCH 6/7] Started with k6 --- backend/src/Squidex/Program.cs | 5 +++++ backend/tools/k6/docker-compose.yml | 25 +++++++++++++++++++++ backend/tools/k6/get-clients.js | 35 +++++++++++++++++++++++++++++ backend/tools/k6/shared.js | 35 +++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+) create mode 100644 backend/tools/k6/docker-compose.yml create mode 100644 backend/tools/k6/get-clients.js create mode 100644 backend/tools/k6/shared.js diff --git a/backend/src/Squidex/Program.cs b/backend/src/Squidex/Program.cs index ebb6922a1..99886b670 100644 --- a/backend/src/Squidex/Program.cs +++ b/backend/src/Squidex/Program.cs @@ -71,6 +71,11 @@ namespace Squidex IPAddress.Any, 5001, listenOptions => listenOptions.UseHttps("../../../dev/squidex-dev.pfx", "password")); + + serverOptions.Listen( + IPAddress.IPv6Any, + 5001, + listenOptions => listenOptions.UseHttps("../../../dev/squidex-dev.pfx", "password")); } }); diff --git a/backend/tools/k6/docker-compose.yml b/backend/tools/k6/docker-compose.yml new file mode 100644 index 000000000..3e7d28660 --- /dev/null +++ b/backend/tools/k6/docker-compose.yml @@ -0,0 +1,25 @@ +version: '3.4' + +networks: + grafana: + +services: + influxdb: + image: influxdb:latest + networks: + - grafana + ports: + - "8086:8086" + environment: + - INFLUXDB_DB=k6 + + grafana: + image: grafana/grafana:latest + networks: + - grafana + ports: + - "4000:3000" + environment: + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_BASIC_ENABLED=false diff --git a/backend/tools/k6/get-clients.js b/backend/tools/k6/get-clients.js new file mode 100644 index 000000000..ad67f6e2a --- /dev/null +++ b/backend/tools/k6/get-clients.js @@ -0,0 +1,35 @@ +import { check } from 'k6'; +import http from 'k6/http'; +import { variables, getBearerToken } from './shared.js'; + +export let options = { + stages: [ + { duration: "2m", target: 200 }, + { duration: "2m", target: 200 }, + { duration: "2m", target: 0 }, + ], + thresholds: { + 'http_req_duration': ['p(99)<1500'], // 99% of requests must complete below 1.5s + } +}; + + +export function setup() { + const token = getBearerToken(); + + return { token }; +} + +export default function (data) { + const url = `${variables.serverUrl}/api/apps/${variables.appName}/clients`; + + const response = http.get(url, { + headers: { + Authorization: `Bearer ${data.token}` + } + }); + + check(response, { + 'is status 200': (r) => r.status === 200, + }); +} \ No newline at end of file diff --git a/backend/tools/k6/shared.js b/backend/tools/k6/shared.js new file mode 100644 index 000000000..a8aef7935 --- /dev/null +++ b/backend/tools/k6/shared.js @@ -0,0 +1,35 @@ +import http from 'k6/http'; + +export const variables = { + appName: getValue('APP__NAME', 'integration-tests'), + clientId: getValue('CLIENT__ID', 'root'), + clientSecret: getValue('CLIENT__SECRET', 'xeLd6jFxqbXJrfmNLlO2j1apagGGGSyZJhFnIuHp4I0='), + serverUrl: getValue('SERVER__URL', 'https://localhost:5001') +}; + +let bearerToken = null; + +export function getBearerToken() { + if (!bearerToken) { + const url = `${variables.serverUrl}/identity-server/connect/token`; + + const response = http.post(url, { + grant_type: 'client_credentials', + client_id: variables.clientId, + client_secret: variables.clientSecret, + scope: 'squidex-api' + }); + + const json = JSON.parse(response.body); + + bearerToken = json.access_token; + } + + return bearerToken; +} + +function getValue(key, fallback) { + const result = __ENV[key] || fallback; + + return result; +} \ No newline at end of file From 0db295af2e350a11146f2b4270542229df213ae5 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 8 Jul 2020 15:45:47 +0200 Subject: [PATCH 7/7] Simplified config. --- backend/src/Squidex/Program.cs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/backend/src/Squidex/Program.cs b/backend/src/Squidex/Program.cs index 99886b670..00b462958 100644 --- a/backend/src/Squidex/Program.cs +++ b/backend/src/Squidex/Program.cs @@ -67,13 +67,7 @@ namespace Squidex { if (context.HostingEnvironment.IsDevelopment() || context.Configuration.GetValue("devMode:enable")) { - serverOptions.Listen( - IPAddress.Any, - 5001, - listenOptions => listenOptions.UseHttps("../../../dev/squidex-dev.pfx", "password")); - - serverOptions.Listen( - IPAddress.IPv6Any, + serverOptions.ListenAnyIP( 5001, listenOptions => listenOptions.UseHttps("../../../dev/squidex-dev.pfx", "password")); }