Mongo request log. (#468)

backend/src/Squidex.Domain.Apps.Entities/Apps/DefaultAppLogStore.cs

@@ -6,29 +6,121 @@
 // ==========================================================================
 
 using System;
+using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
+using System.Text;
+using System.Threading;
 using System.Threading.Tasks;
+using CsvHelper;
+using CsvHelper.Configuration;
+using NodaTime;
 using Squidex.Infrastructure;
-using Squidex.Infrastructure.Log;
+using Squidex.Infrastructure.Log.Store;
 
 namespace Squidex.Domain.Apps.Entities.Apps
 {
     public sealed class DefaultAppLogStore : IAppLogStore
     {
-        private readonly ILogStore logStore;
+        private const string FieldAuthClientId = "AuthClientId";
+        private const string FieldAuthUserId = "AuthUserId";
+        private const string FieldCosts = "Costs";
+        private const string FieldRequestElapsedMs = "RequestElapsedMs";
+        private const string FieldRequestMethod = "RequestMethod";
+        private const string FieldRequestPath = "RequestPath";
+        private const string FieldTimestamp = "Timestamp";
+        private readonly Configuration csvConfiguration = new Configuration { Delimiter = "|" };
+        private readonly IRequestLogStore requestLogStore;
 
-        public DefaultAppLogStore(ILogStore logStore)
+        public DefaultAppLogStore(IRequestLogStore requestLogStore)
         {
-            Guard.NotNull(logStore);
+            Guard.NotNull(requestLogStore);
 
-            this.logStore = logStore;
+            this.requestLogStore = requestLogStore;
         }
 
-        public Task ReadLogAsync(string appId, DateTime from, DateTime to, Stream stream)
+        public Task LogAsync(Guid appId, Instant timestamp, string? requestMethod, string? requestPath, string? userId, string? clientId, long elapsedMs, double costs)
+        {
+            var request = new Request
+            {
+                Key = appId.ToString(),
+                Properties = new Dictionary<string, string>
+                {
+                    [FieldCosts] = costs.ToString()
+                },
+                Timestamp = timestamp
+            };
+
+            Append(request, FieldAuthClientId, clientId);
+            Append(request, FieldAuthUserId, userId);
+            Append(request, FieldCosts, costs.ToString(CultureInfo.InvariantCulture));
+            Append(request, FieldRequestElapsedMs, elapsedMs.ToString(CultureInfo.InvariantCulture));
+            Append(request, FieldRequestMethod, requestMethod);
+            Append(request, FieldRequestPath, requestPath);
+
+            return requestLogStore.LogAsync(request);
+        }
+
+        public async Task ReadLogAsync(Guid appId, DateTime fromDate, DateTime toDate, Stream stream, CancellationToken ct = default)
         {
             Guard.NotNull(appId);
 
-            return logStore.ReadLogAsync(appId, from, to, stream);
+            var writer = new StreamWriter(stream, Encoding.UTF8, 4096, true);
+            try
+            {
+                using (var csv = new CsvWriter(writer, csvConfiguration, true))
+                {
+                    csv.WriteField(FieldTimestamp);
+                    csv.WriteField(FieldRequestPath);
+                    csv.WriteField(FieldRequestMethod);
+                    csv.WriteField(FieldRequestElapsedMs);
+                    csv.WriteField(FieldCosts);
+                    csv.WriteField(FieldAuthClientId);
+                    csv.WriteField(FieldAuthUserId);
+
+                    await csv.NextRecordAsync();
+
+                    await requestLogStore.QueryAllAsync(async request =>
+                    {
+                        csv.WriteField(request.Timestamp.ToString());
+                        csv.WriteField(GetString(request, FieldRequestPath));
+                        csv.WriteField(GetString(request, FieldRequestMethod));
+                        csv.WriteField(GetDouble(request, FieldRequestElapsedMs));
+                        csv.WriteField(GetDouble(request, FieldCosts));
+                        csv.WriteField(GetString(request, FieldAuthClientId));
+                        csv.WriteField(GetString(request, FieldAuthUserId));
+
+                        await csv.NextRecordAsync();
+                    }, appId.ToString(), fromDate, toDate, ct);
+                }
+            }
+            finally
+            {
+                await writer.FlushAsync();
+            }
+        }
+
+        private static void Append(Request request, string key, string? value)
+        {
+            if (!string.IsNullOrWhiteSpace(value))
+            {
+                request.Properties[key] = value;
+            }
+        }
+
+        private static string GetString(Request request, string key)
+        {
+            return request.Properties.GetValueOrDefault(key, string.Empty)!;
+        }
+
+        private static double GetDouble(Request request, string key)
+        {
+            if (request.Properties.TryGetValue(key, out var value) && double.TryParse(value, NumberStyles.Any, CultureInfo.InvariantCulture, out var result))
+            {
+                return result;
+            }
+
+            return 0;
         }
     }
 }

backend/src/Squidex.Domain.Apps.Entities/Apps/IAppLogStore.cs

@@ -7,12 +7,16 @@
 
 using System;
 using System.IO;
+using System.Threading;
 using System.Threading.Tasks;
+using NodaTime;
 
 namespace Squidex.Domain.Apps.Entities.Apps
 {
     public interface IAppLogStore
     {
-        Task ReadLogAsync(string appId, DateTime from, DateTime to, Stream stream);
+        Task LogAsync(Guid appId, Instant timestamp, string? requestMethod, string? requestPath, string? userId, string? clientId, long elapsedMs, double costs);
+
+        Task ReadLogAsync(Guid appId, DateTime fromDate, DateTime toDate, Stream stream, CancellationToken ct = default);
     }
 }

backend/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj

@@ -16,6 +16,7 @@
     <ProjectReference Include="..\Squidex.Shared\Squidex.Shared.csproj" />
   </ItemGroup>
   <ItemGroup>
+    <PackageReference Include="CsvHelper" Version="12.2.3" />
     <PackageReference Include="GraphQL" Version="2.4.0" />
     <PackageReference Include="Lucene.Net" Version="4.8.0-beta00005" />
     <PackageReference Include="Lucene.Net.Analysis.Common" Version="4.8.0-beta00005" />

backend/src/Squidex.Infrastructure.MongoDb/Log/MongoRequest.cs

@@ -0,0 +1,33 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System.Collections.Generic;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+using NodaTime;
+
+namespace Squidex.Infrastructure.Log
+{
+    public sealed class MongoRequest
+    {
+        [BsonId]
+        [BsonElement]
+        public ObjectId Id { get; set; }
+
+        [BsonElement]
+        [BsonRequired]
+        public string Key { get; set; }
+
+        [BsonElement]
+        [BsonRequired]
+        public Instant Timestamp { get; set; }
+
+        [BsonElement]
+        [BsonRequired]
+        public Dictionary<string, string> Properties { get; set; }
+    }
+}

backend/src/Squidex.Infrastructure.MongoDb/Log/MongoRequestLogRepository.cs

@@ -0,0 +1,77 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using MongoDB.Driver;
+using NodaTime;
+using Squidex.Infrastructure.Log.Store;
+using Squidex.Infrastructure.MongoDb;
+
+namespace Squidex.Infrastructure.Log
+{
+    public sealed class MongoRequestLogRepository : MongoRepositoryBase<MongoRequest>, IRequestLogRepository
+    {
+        private static readonly InsertManyOptions Unordered = new InsertManyOptions { IsOrdered = false };
+
+        public MongoRequestLogRepository(IMongoDatabase database)
+            : base(database)
+        {
+        }
+
+        protected override string CollectionName()
+        {
+            return "RequestLog";
+        }
+
+        protected override Task SetupCollectionAsync(IMongoCollection<MongoRequest> collection, CancellationToken ct = default)
+        {
+            return collection.Indexes.CreateManyAsync(new[]
+            {
+                new CreateIndexModel<MongoRequest>(
+                    Index
+                        .Ascending(x => x.Key)
+                        .Ascending(x => x.Timestamp)),
+                new CreateIndexModel<MongoRequest>(
+                    Index
+                        .Ascending(x => x.Timestamp),
+                    new CreateIndexOptions
+                    {
+                        ExpireAfter = TimeSpan.FromDays(90)
+                    }),
+            }, ct);
+        }
+
+        public Task InsertManyAsync(IEnumerable<Request> items)
+        {
+            Guard.NotNull(items);
+
+            var documents = items.Select(x => new MongoRequest { Key = x.Key, Timestamp = x.Timestamp, Properties = x.Properties });
+
+            return Collection.InsertManyAsync(documents, Unordered);
+        }
+
+        public Task QueryAllAsync(Func<Request, Task> callback, string key, DateTime fromDate, DateTime toDate, CancellationToken ct = default)
+        {
+            Guard.NotNull(callback);
+            Guard.NotNullOrEmpty(key);
+
+            var timestampStart = Instant.FromDateTimeUtc(fromDate);
+            var timestampEnd = Instant.FromDateTimeUtc(toDate.AddDays(1));
+
+            return Collection.Find(x => x.Key == key && x.Timestamp >= timestampStart && x.Timestamp < timestampEnd).ForEachAsync(x =>
+            {
+                var request = new Request { Key = x.Key, Timestamp = x.Timestamp, Properties = x.Properties };
+
+                return callback(request);
+            }, ct);
+        }
+    }
+}

backend/src/Squidex.Infrastructure/Log/LockingLogStore.cs

@@ -1,83 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using System;
-using System.IO;
-using System.Text;
-using System.Threading;
-using System.Threading.Tasks;
-using Orleans;
-using Squidex.Infrastructure.Orleans;
-
-namespace Squidex.Infrastructure.Log
-{
-    public sealed class LockingLogStore : ILogStore
-    {
-        private static readonly byte[] LockedText = Encoding.UTF8.GetBytes("Another process is currenty running, try it again later.");
-        private static readonly TimeSpan LockWaitingTime = TimeSpan.FromMinutes(1);
-        private readonly ILogStore inner;
-        private readonly ILockGrain lockGrain;
-
-        public LockingLogStore(ILogStore inner, IGrainFactory grainFactory)
-        {
-            Guard.NotNull(inner);
-            Guard.NotNull(grainFactory);
-
-            this.inner = inner;
-
-            lockGrain = grainFactory.GetGrain<ILockGrain>(SingleGrain.Id);
-        }
-
-        public Task ReadLogAsync(string key, DateTime from, DateTime to, Stream stream)
-        {
-            return ReadLogAsync(key, from, to, stream, LockWaitingTime);
-        }
-
-        public async Task ReadLogAsync(string key, DateTime from, DateTime to, Stream stream, TimeSpan lockTimeout)
-        {
-            using (var cts = new CancellationTokenSource(lockTimeout))
-            {
-                string? releaseToken = null;
-
-                while (!cts.IsCancellationRequested)
-                {
-                    releaseToken = await lockGrain.AcquireLockAsync(key);
-
-                    if (releaseToken != null)
-                    {
-                        break;
-                    }
-
-                    try
-                    {
-                        await Task.Delay(2000, cts.Token);
-                    }
-                    catch (OperationCanceledException)
-                    {
-                        break;
-                    }
-                }
-
-                if (releaseToken != null)
-                {
-                    try
-                    {
-                        await inner.ReadLogAsync(key, from, to, stream);
-                    }
-                    finally
-                    {
-                        await lockGrain.ReleaseLockAsync(releaseToken);
-                    }
-                }
-                else
-                {
-                    await stream.WriteAsync(LockedText, 0, LockedText.Length);
-                }
-            }
-        }
-    }
-}

backend/src/Squidex.Infrastructure/Log/Store/BackgroundRequestLogStore.cs

@@ -0,0 +1,91 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System;
+using System.Collections.Concurrent;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Squidex.Infrastructure.Tasks;
+using Squidex.Infrastructure.Timers;
+
+namespace Squidex.Infrastructure.Log.Store
+{
+    public sealed class BackgroundRequestLogStore : DisposableObjectBase, IRequestLogStore
+    {
+        private const int Intervall = 10 * 1000;
+        private const int BatchSize = 1000;
+        private readonly IRequestLogRepository logRepository;
+        private readonly ISemanticLog log;
+        private readonly CompletionTimer timer;
+        private ConcurrentQueue<Request> jobs = new ConcurrentQueue<Request>();
+
+        public BackgroundRequestLogStore(IRequestLogRepository logRepository, ISemanticLog log)
+        {
+            Guard.NotNull(logRepository);
+            Guard.NotNull(log);
+
+            this.logRepository = logRepository;
+            this.log = log;
+
+            timer = new CompletionTimer(Intervall, ct => TrackAsync(), Intervall);
+        }
+
+        protected override void DisposeObject(bool disposing)
+        {
+            if (disposing)
+            {
+                timer.StopAsync().Wait();
+            }
+        }
+
+        public void Next()
+        {
+            ThrowIfDisposed();
+
+            timer.SkipCurrentDelay();
+        }
+
+        private async Task TrackAsync()
+        {
+            try
+            {
+                var localJobs = Interlocked.Exchange(ref jobs, new ConcurrentQueue<Request>());
+
+                if (localJobs.Count > 0)
+                {
+                    var pages = (int)Math.Ceiling((double)localJobs.Count / BatchSize);
+
+                    for (var i = 0; i < pages; i++)
+                    {
+                        await logRepository.InsertManyAsync(localJobs.Skip(i * BatchSize).Take(BatchSize));
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                log.LogError(ex, w => w
+                    .WriteProperty("action", "TrackUsage")
+                    .WriteProperty("status", "Failed"));
+            }
+        }
+
+        public Task QueryAllAsync(Func<Request, Task> callback, string key, DateTime fromDate, DateTime toDate, CancellationToken ct = default)
+        {
+            return logRepository.QueryAllAsync(callback, key, fromDate, toDate, ct);
+        }
+
+        public Task LogAsync(Request request)
+        {
+            Guard.NotNull(request);
+
+            jobs.Enqueue(request);
+
+            return TaskHelper.Done;
+        }
+    }
+}

backend/src/Squidex.Infrastructure/Log/NoopLogStore.cs → backend/src/Squidex.Infrastructure/Log/Store/IRequestLogRepository.cs

@@ -6,19 +6,16 @@
 // ==========================================================================
 
 using System;
-using System.IO;
-using System.Text;
+using System.Collections.Generic;
+using System.Threading;
 using System.Threading.Tasks;
 
-namespace Squidex.Infrastructure.Log
+namespace Squidex.Infrastructure.Log.Store
 {
-    public sealed class NoopLogStore : ILogStore
+    public interface IRequestLogRepository
     {
-        private static readonly byte[] NoopText = Encoding.UTF8.GetBytes("Not Supported");
+        Task InsertManyAsync(IEnumerable<Request> items);
 
-        public Task ReadLogAsync(string key, DateTime from, DateTime to, Stream stream)
-        {
-            return stream.WriteAsync(NoopText, 0, NoopText.Length);
-        }
+        Task QueryAllAsync(Func<Request, Task> callback, string key, DateTime fromDate, DateTime toDate, CancellationToken ct = default);
     }
 }

backend/src/Squidex.Infrastructure/Log/ILogStore.cs → backend/src/Squidex.Infrastructure/Log/Store/IRequestLogStore.cs

@@ -6,13 +6,15 @@
 // ==========================================================================
 
 using System;
-using System.IO;
+using System.Threading;
 using System.Threading.Tasks;
 
-namespace Squidex.Infrastructure.Log
+namespace Squidex.Infrastructure.Log.Store
 {
-    public interface ILogStore
+    public interface IRequestLogStore
     {
-        Task ReadLogAsync(string key, DateTime from, DateTime to, Stream stream);
+        Task LogAsync(Request request);
+
+        Task QueryAllAsync(Func<Request, Task> callback, string key, DateTime fromDate, DateTime toDate, CancellationToken ct = default);
     }
 }

backend/src/Squidex.Infrastructure/Log/Store/Request.cs

@@ -0,0 +1,23 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System.Collections.Generic;
+using NodaTime;
+
+#pragma warning disable SA1401 // Fields should be private
+
+namespace Squidex.Infrastructure.Log.Store
+{
+    public sealed class Request
+    {
+        public Instant Timestamp;
+
+        public string Key;
+
+        public Dictionary<string, string> Properties;
+    }
+}

backend/src/Squidex.Web/Pipeline/ApiCostsFilter.cs

@@ -9,6 +9,8 @@ using System;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.Mvc.Filters;
+using NodaTime;
+using Squidex.Domain.Apps.Entities.Apps;
 using Squidex.Domain.Apps.Entities.Apps.Services;
 using Squidex.Infrastructure;
 using Squidex.Infrastructure.Log;
@@ -20,12 +22,15 @@ namespace Squidex.Web.Pipeline
     public sealed class ApiCostsFilter : IAsyncActionFilter, IFilterContainer
     {
         private readonly IAppPlansProvider appPlansProvider;
+        private readonly IAppLogStore appLogStore;
         private readonly IUsageTracker usageTracker;
+        private readonly IClock clock;
 
-        public ApiCostsFilter(IAppPlansProvider appPlansProvider, IUsageTracker usageTracker)
+        public ApiCostsFilter(IAppLogStore appLogStore, IAppPlansProvider appPlansProvider, IClock clock, IUsageTracker usageTracker)
         {
+            this.appLogStore = appLogStore;
             this.appPlansProvider = appPlansProvider;
-
+            this.clock = clock;
             this.usageTracker = usageTracker;
         }
 
@@ -49,10 +54,12 @@ namespace Squidex.Web.Pipeline
 
             var app = context.HttpContext.Context().App;
 
-            if (app != null && FilterDefinition.Weight > 0)
+            if (app != null)
             {
                 var appId = app.Id.ToString();
 
+                if (FilterDefinition.Weight > 0)
+                {
                     using (Profiler.Trace("CheckUsage"))
                     {
                         var plan = appPlansProvider.GetPlanForApp(app);
@@ -65,6 +72,7 @@ namespace Squidex.Web.Pipeline
                             return;
                         }
                     }
+                }
 
                 var watch = ValueStopwatch.StartNew();
 
@@ -76,9 +84,20 @@ namespace Squidex.Web.Pipeline
                 {
                     var elapsedMs = watch.Stop();
 
+                    await appLogStore.LogAsync(app.Id, clock.GetCurrentInstant(),
+                        context.HttpContext.Request.Method,
+                        context.HttpContext.Request.Path,
+                        context.HttpContext.User.OpenIdSubject(),
+                        context.HttpContext.User.OpenIdClientId(),
+                        elapsedMs,
+                        FilterDefinition.Weight);
+
+                    if (FilterDefinition.Weight > 0)
+                    {
                         await usageTracker.TrackAsync(appId, context.HttpContext.User.OpenIdClientId(), FilterDefinition.Weight, elapsedMs);
                     }
                 }
+            }
             else
             {
                 await next();

backend/src/Squidex/Areas/Api/Controllers/Statistics/UsagesController.cs

@@ -195,11 +195,11 @@ namespace Squidex.Areas.Api.Controllers.Statistics
         {
             var appId = dataProtector.Unprotect(token);
 
-            var today = DateTime.Today;
+            var today = DateTime.UtcNow.Date;
 
             return new FileCallbackResult("text/csv", $"Usage-{today:yyy-MM-dd}.csv", false, stream =>
             {
-                return appLogStore.ReadLogAsync(appId, today.AddDays(-30), today, stream);
+                return appLogStore.ReadLogAsync(Guid.Parse(appId), today.AddDays(-30), today, stream);
             });
         }
     }

backend/src/Squidex/Config/Domain/LoggingServices.cs

@@ -12,6 +12,7 @@ using Microsoft.Extensions.Logging;
 using Squidex.Domain.Apps.Entities.Apps;
 using Squidex.Infrastructure.Log;
 using Squidex.Infrastructure.Log.Adapter;
+using Squidex.Infrastructure.Log.Store;
 using Squidex.Web.Pipeline;
 
 namespace Squidex.Config.Domain
@@ -80,8 +81,8 @@ namespace Squidex.Config.Domain
             services.AddSingletonAs<DefaultAppLogStore>()
                 .As<IAppLogStore>();
 
-            services.AddSingletonAs<NoopLogStore>()
-                .AsOptional<ILogStore>();
+            services.AddSingletonAs<BackgroundRequestLogStore>()
+                .AsOptional<IRequestLogStore>();
         }
 
         private static void AddFilters(this ILoggingBuilder builder)

backend/src/Squidex/Config/Domain/StoreServices.cs

@@ -34,6 +34,8 @@ using Squidex.Infrastructure;
 using Squidex.Infrastructure.Diagnostics;
 using Squidex.Infrastructure.EventSourcing;
 using Squidex.Infrastructure.Json;
+using Squidex.Infrastructure.Log;
+using Squidex.Infrastructure.Log.Store;
 using Squidex.Infrastructure.Migrations;
 using Squidex.Infrastructure.Reflection;
 using Squidex.Infrastructure.States;
@@ -82,6 +84,9 @@ namespace Squidex.Config.Domain
                     services.AddHealthChecks()
                         .AddCheck<MongoDBHealthCheck>("MongoDB", tags: new[] { "node" });
 
+                    services.AddSingletonAs<MongoRequestLogRepository>()
+                        .As<IRequestLogRepository>();
+
                     services.AddSingletonAs<MongoUsageRepository>()
                         .As<IUsageRepository>();
 

backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/DefaultAppLogStoreTests.cs

@@ -0,0 +1,98 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Threading.Tasks;
+using FakeItEasy;
+using Squidex.Infrastructure.Log.Store;
+using Xunit;
+
+namespace Squidex.Domain.Apps.Entities.Apps
+{
+    public class DefaultAppLogStoreTests
+    {
+        private readonly IRequestLogStore requestLogStore = A.Fake<IRequestLogStore>();
+        private readonly DefaultAppLogStore sut;
+
+        public DefaultAppLogStoreTests()
+        {
+            sut = new DefaultAppLogStore(requestLogStore);
+        }
+
+        [Fact]
+        public async Task Should_forward_request_log_to_store()
+        {
+            Request? recordedRequest = null;
+
+            A.CallTo(() => requestLogStore.LogAsync(A<Request>.Ignored))
+                .Invokes((Request request) => recordedRequest = request);
+
+            var clientId = "frontend";
+            var costs = 2;
+            var elapsedMs = 120;
+            var requestMethod = "GET";
+            var requestPath = "/my-path";
+            var userId = "user1";
+
+            await sut.LogAsync(Guid.NewGuid(), default, requestMethod, requestPath, userId, clientId, elapsedMs, costs);
+
+            Assert.NotNull(recordedRequest);
+
+            Assert.Contains(clientId, recordedRequest!.Properties.Values);
+            Assert.Contains(costs.ToString(), recordedRequest!.Properties.Values);
+            Assert.Contains(elapsedMs.ToString(), recordedRequest!.Properties.Values);
+            Assert.Contains(requestMethod, recordedRequest!.Properties.Values);
+            Assert.Contains(requestPath, recordedRequest!.Properties.Values);
+        }
+
+        [Fact]
+        public async Task Should_create_some_stream()
+        {
+            var dateFrom = DateTime.UtcNow.Date.AddDays(-30);
+            var dateTo = DateTime.UtcNow.Date;
+
+            var appId = Guid.NewGuid();
+
+            A.CallTo(() => requestLogStore.QueryAllAsync(A<Func<Request, Task>>.Ignored, appId.ToString(), dateFrom, dateTo, default))
+                .Invokes(x =>
+                {
+                    var callback = x.GetArgument<Func<Request, Task>>(0);
+
+                    callback(CreateRecord());
+                    callback(CreateRecord());
+                    callback(CreateRecord());
+                    callback(CreateRecord());
+                });
+
+            var stream = new MemoryStream();
+
+            await sut.ReadLogAsync(appId, dateFrom, dateTo, stream);
+
+            stream.Position = 0;
+
+            var lines = 0;
+
+            using (var reader = new StreamReader(stream))
+            {
+                string? line = null;
+                while ((line = reader.ReadLine()) != null)
+                {
+                    lines++;
+                }
+            }
+
+            Assert.Equal(5, lines);
+        }
+
+        private static Request CreateRecord()
+        {
+            return new Request { Properties = new Dictionary<string, string>() };
+        }
+    }
+}

backend/tests/Squidex.Infrastructure.Tests/Log/LockingLogStoreTests.cs

@@ -1,87 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using System;
-using System.IO;
-using System.Threading.Tasks;
-using FakeItEasy;
-using Orleans;
-using Squidex.Infrastructure.Orleans;
-using Xunit;
-
-namespace Squidex.Infrastructure.Log
-{
-    public class LockingLogStoreTests
-    {
-        private readonly IGrainFactory grainFactory = A.Fake<IGrainFactory>();
-        private readonly ILockGrain lockGrain = A.Fake<ILockGrain>();
-        private readonly ILogStore inner = A.Fake<ILogStore>();
-        private readonly LockingLogStore sut;
-
-        public LockingLogStoreTests()
-        {
-            A.CallTo(() => grainFactory.GetGrain<ILockGrain>(SingleGrain.Id, null))
-                .Returns(lockGrain);
-
-            sut = new LockingLogStore(inner, grainFactory);
-        }
-
-        [Fact]
-        public async Task Should_lock_and_call_inner()
-        {
-            var stream = new MemoryStream();
-
-            var dateFrom = DateTime.Today;
-            var dateTo = dateFrom.AddDays(2);
-
-            var key = "MyKey";
-
-            var releaseToken = Guid.NewGuid().ToString();
-
-            A.CallTo(() => lockGrain.AcquireLockAsync(key))
-                .Returns(releaseToken);
-
-            await sut.ReadLogAsync(key, dateFrom, dateTo, stream);
-
-            A.CallTo(() => lockGrain.AcquireLockAsync(key))
-                .MustHaveHappened();
-
-            A.CallTo(() => lockGrain.ReleaseLockAsync(releaseToken))
-                .MustHaveHappened();
-
-            A.CallTo(() => inner.ReadLogAsync(key, dateFrom, dateTo, stream))
-                .MustHaveHappened();
-        }
-
-        [Fact]
-        public async Task Should_write_default_message_if_lock_could_not_be_acquired()
-        {
-            var stream = new MemoryStream();
-
-            var dateFrom = DateTime.Today;
-            var dateTo = dateFrom.AddDays(2);
-
-            var key = "MyKey";
-
-            A.CallTo(() => lockGrain.AcquireLockAsync(key))
-                .Returns(Task.FromResult<string?>(null));
-
-            await sut.ReadLogAsync(key, dateFrom, dateTo, stream, TimeSpan.FromSeconds(1));
-
-            A.CallTo(() => lockGrain.AcquireLockAsync(key))
-                .MustHaveHappened();
-
-            A.CallTo(() => lockGrain.ReleaseLockAsync(A<string>.Ignored))
-                .MustNotHaveHappened();
-
-            A.CallTo(() => inner.ReadLogAsync(A<string>.Ignored, A<DateTime>.Ignored, A<DateTime>.Ignored, A<Stream>.Ignored))
-                .MustNotHaveHappened();
-
-            Assert.True(stream.Length > 0);
-        }
-    }
-}

backend/tests/Squidex.Infrastructure.Tests/Log/Store/BackgroundRequestLogStoreTests.cs

@@ -0,0 +1,52 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using FakeItEasy;
+using Xunit;
+
+namespace Squidex.Infrastructure.Log.Store
+{
+    public class BackgroundRequestLogStoreTests
+    {
+        private readonly IRequestLogRepository requestLogRepository = A.Fake<IRequestLogRepository>();
+        private readonly BackgroundRequestLogStore sut;
+
+        public BackgroundRequestLogStoreTests()
+        {
+            sut = new BackgroundRequestLogStore(requestLogRepository, A.Fake<ISemanticLog>());
+        }
+
+        [Fact]
+        public async Task Should_log_in_batches()
+        {
+            for (var i = 0; i < 2500; i++)
+            {
+                await sut.LogAsync(new Request { Key = i.ToString() });
+            }
+
+            sut.Next();
+            sut.Dispose();
+
+            A.CallTo(() => requestLogRepository.InsertManyAsync(Batch("0", "999")))
+                .MustHaveHappened();
+
+            A.CallTo(() => requestLogRepository.InsertManyAsync(Batch("1000", "1999")))
+                .MustHaveHappened();
+
+            A.CallTo(() => requestLogRepository.InsertManyAsync(Batch("2000", "2499")))
+                .MustHaveHappened();
+        }
+
+        private static IEnumerable<Request> Batch(string from, string to)
+        {
+            return A<IEnumerable<Request>>.That.Matches(x => x.First().Key == from && x.Last().Key == to);
+        }
+    }
+}

backend/tests/Squidex.Web.Tests/Pipeline/ApiCostsFilterTests.cs

@@ -15,6 +15,7 @@ using Microsoft.AspNetCore.Mvc.Abstractions;
 using Microsoft.AspNetCore.Mvc.Filters;
 using Microsoft.AspNetCore.Mvc.Infrastructure;
 using Microsoft.AspNetCore.Routing;
+using NodaTime;
 using Squidex.Domain.Apps.Entities.Apps;
 using Squidex.Domain.Apps.Entities.Apps.Services;
 using Squidex.Infrastructure.UsageTracking;
@@ -26,9 +27,11 @@ namespace Squidex.Web.Pipeline
     {
         private readonly IActionContextAccessor actionContextAccessor = A.Fake<IActionContextAccessor>();
         private readonly IAppEntity appEntity = A.Fake<IAppEntity>();
+        private readonly IAppLimitsPlan appPlan = A.Fake<IAppLimitsPlan>();
+        private readonly IAppLogStore appLogStore = A.Fake<IAppLogStore>();
         private readonly IAppPlansProvider appPlansProvider = A.Fake<IAppPlansProvider>();
+        private readonly IClock clock = A.Fake<IClock>();
         private readonly IUsageTracker usageTracker = A.Fake<IUsageTracker>();
-        private readonly IAppLimitsPlan appPlan = A.Fake<IAppLimitsPlan>();
         private readonly ActionExecutingContext actionContext;
         private readonly HttpContext httpContext = new DefaultHttpContext();
         private readonly ActionExecutionDelegate next;
@@ -67,7 +70,7 @@ namespace Squidex.Web.Pipeline
                 return Task.FromResult<ActionExecutedContext?>(null);
             };
 
-            sut = new ApiCostsFilter(appPlansProvider, usageTracker);
+            sut = new ApiCostsFilter(appLogStore, appPlansProvider, clock, usageTracker);
         }
 
         [Fact]
@@ -159,6 +162,27 @@ namespace Squidex.Web.Pipeline
                 .MustNotHaveHappened();
         }
 
+        [Fact]
+        public async Task Should_log_request_event_if_weight_is_zero()
+        {
+            sut.FilterDefinition = new ApiCostsAttribute(0);
+
+            SetupApp();
+
+            httpContext.Request.Method = "GET";
+            httpContext.Request.Path = "/my-path";
+
+            var instant = SystemClock.Instance.GetCurrentInstant();
+
+            A.CallTo(() => clock.GetCurrentInstant())
+                .Returns(instant);
+
+            await sut.OnActionExecutionAsync(actionContext, next);
+
+            A.CallTo(() => appLogStore.LogAsync(appEntity.Id, instant, "GET", "/my-path", null, null, A<long>.Ignored, 0))
+                .MustHaveHappened();
+        }
+
         private void SetupApp()
         {
             httpContext.Context().App = appEntity;