mirror of https://github.com/Squidex/squidex.git
Browse Source
* Run tests on all servers. * Revert schema tests. * T * t * Text * Fix. * Add migration. * Update tests. * Test degrees * Update test * Update duration. * Improvements. * Add searchable text field. * Fix whitespaces.pull/1192/head
Sebastian Stehle
1 year ago
committed by
GitHub
GitHub
98 changed files with 6533 additions and 570 deletions
@ -0,0 +1,45 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Squidex.Domain.Apps.Entities.Contents.Text; |
|||
using Squidex.Domain.Apps.Entities.Contents.Text.State; |
|||
using Squidex.Infrastructure; |
|||
|
|||
namespace Microsoft.EntityFrameworkCore; |
|||
|
|||
public static class EFTextBuilder |
|||
{ |
|||
public static void UseTextIndex(this ModelBuilder builder) |
|||
{ |
|||
builder.Entity<TextContentState>(b => |
|||
{ |
|||
b.ToTable("TextState"); |
|||
b.HasKey(x => x.UniqueContentId); |
|||
b.Property(x => x.UniqueContentId).AsString(); |
|||
b.Property(x => x.State).AsString(); |
|||
}); |
|||
|
|||
builder.Entity<EFTextIndexTextEntity>(b => |
|||
{ |
|||
b.ToTable("Texts"); |
|||
b.Property(x => x.Id).HasMaxLength(400); |
|||
b.Property(x => x.AppId).AsString(); |
|||
b.Property(x => x.SchemaId).AsString(); |
|||
b.Property(x => x.ContentId).AsString(); |
|||
}); |
|||
|
|||
builder.Entity<EFTextIndexGeoEntity>(b => |
|||
{ |
|||
b.ToTable("Geos"); |
|||
b.Property(x => x.Id).HasMaxLength(400); |
|||
b.Property(x => x.AppId).AsString(); |
|||
b.Property(x => x.SchemaId).AsString(); |
|||
b.Property(x => x.ContentId).AsString(); |
|||
b.Property(x => x.GeoField).HasMaxLength(255); |
|||
}); |
|||
} |
|||
} |
|||
@ -0,0 +1,301 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using EFCore.BulkExtensions; |
|||
using Microsoft.Data.SqlClient; |
|||
using Microsoft.EntityFrameworkCore; |
|||
using NetTopologySuite.Geometries; |
|||
using Squidex.Domain.Apps.Core.Apps; |
|||
using Squidex.Domain.Apps.Core.Schemas; |
|||
using Squidex.Hosting; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.Queries; |
|||
using Squidex.Providers.SqlServer; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text; |
|||
|
|||
public sealed class EFTextIndex<TContext>(IDbContextFactory<TContext> dbContextFactory, SqlDialect dialect) |
|||
: ITextIndex, IInitializable, IDeleter where TContext : DbContext |
|||
{ |
|||
private record struct SearchOperation |
|||
{ |
|||
required public App App { get; init; } |
|||
|
|||
required public List<(DomainId Id, double Score)> Results { get; init; } |
|||
|
|||
required public string SearchTerms { get; init; } |
|||
|
|||
required public int Take { get; set; } |
|||
|
|||
required public SearchScope SearchScope { get; init; } |
|||
} |
|||
|
|||
public async Task InitializeAsync( |
|||
CancellationToken ct) |
|||
{ |
|||
await using var dbContext = await CreateDbContextAsync(ct); |
|||
|
|||
await dbContext.Database.CreateTextIndexAsync(dialect, "IDX_Text", "Texts", "Texts", ct); |
|||
await dbContext.Database.CreateGeoIndexAsync(dialect, "IDX_Geo", "Geos", "GeoObject", ct); |
|||
} |
|||
|
|||
async Task IDeleter.DeleteAppAsync(App app, |
|||
CancellationToken ct) |
|||
{ |
|||
await using var dbContext = await CreateDbContextAsync(ct); |
|||
|
|||
await dbContext.Set<EFTextIndexTextEntity>().Where(x => x.AppId == app.Id) |
|||
.ExecuteDeleteAsync(ct); |
|||
|
|||
await dbContext.Set<EFTextIndexGeoEntity>().Where(x => x.AppId == app.Id) |
|||
.ExecuteDeleteAsync(ct); |
|||
} |
|||
|
|||
async Task IDeleter.DeleteSchemaAsync(App app, Schema schema, |
|||
CancellationToken ct) |
|||
{ |
|||
await using var dbContext = await CreateDbContextAsync(ct); |
|||
|
|||
await dbContext.Set<EFTextIndexTextEntity>().Where(x => x.AppId == app.Id && x.SchemaId == schema.Id) |
|||
.ExecuteDeleteAsync(ct); |
|||
|
|||
await dbContext.Set<EFTextIndexGeoEntity>().Where(x => x.AppId == app.Id && x.SchemaId == schema.Id) |
|||
.ExecuteDeleteAsync(ct); |
|||
} |
|||
|
|||
public async Task ClearAsync(CancellationToken ct = default) |
|||
{ |
|||
await using var dbContext = await CreateDbContextAsync(ct); |
|||
|
|||
await dbContext.Set<EFTextIndexTextEntity>() |
|||
.ExecuteDeleteAsync(ct); |
|||
|
|||
await dbContext.Set<EFTextIndexGeoEntity>() |
|||
.ExecuteDeleteAsync(ct); |
|||
} |
|||
|
|||
public async Task<List<DomainId>?> SearchAsync(App app, GeoQuery query, SearchScope scope, |
|||
CancellationToken ct = default) |
|||
{ |
|||
Guard.NotNull(app); |
|||
Guard.NotNull(query); |
|||
|
|||
await using var dbContext = await CreateDbContextAsync(ct); |
|||
|
|||
var point = new Point(query.Longitude, query.Latitude) { SRID = 4326 }; |
|||
|
|||
// The distance must be converted to decrees (in contrast to MongoDB, which uses radian).
|
|||
var degrees = query.Radius / 111320; |
|||
|
|||
var ids = |
|||
await dbContext.Set<EFTextIndexGeoEntity>() |
|||
.Where(x => x.AppId == app.Id) |
|||
.Where(x => x.SchemaId == query.SchemaId) |
|||
.Where(x => x.GeoField == query.Field) |
|||
.Where(x => x.GeoObject.Distance(point) < degrees) |
|||
.WhereScope(scope) |
|||
.Select(x => x.ContentId) |
|||
.ToListAsync(ct); |
|||
|
|||
return ids; |
|||
} |
|||
|
|||
public async Task<List<DomainId>?> SearchAsync(App app, TextQuery query, SearchScope scope, |
|||
CancellationToken ct = default) |
|||
{ |
|||
Guard.NotNull(app); |
|||
Guard.NotNull(query); |
|||
|
|||
if (string.IsNullOrWhiteSpace(query.Text)) |
|||
{ |
|||
return null; |
|||
} |
|||
|
|||
// Use a custom tokenizer to leverage stop words from multiple languages.
|
|||
var search = new SearchOperation |
|||
{ |
|||
App = app, |
|||
SearchTerms = Tokenizer.Query(query.Text), |
|||
SearchScope = scope, |
|||
Results = [], |
|||
Take = query.Take, |
|||
}; |
|||
|
|||
await using var dbContext = await CreateDbContextAsync(ct); |
|||
|
|||
if (query.RequiredSchemaIds?.Count > 0) |
|||
{ |
|||
await SearchBySchemaAsync(dbContext, search, query.RequiredSchemaIds, 1, ct); |
|||
} |
|||
else if (query.PreferredSchemaId == null) |
|||
{ |
|||
await SearchByAppAsync(dbContext, search, 1, ct); |
|||
} |
|||
else |
|||
{ |
|||
// We cannot write queries that prefer results from the same schema, therefore make two queries.
|
|||
search.Take /= 2; |
|||
|
|||
// Increasing the scoring of the results from the schema by 10 percent.
|
|||
await SearchBySchemaAsync(dbContext, search, Enumerable.Repeat(query.PreferredSchemaId.Value, 1), 1.1, ct); |
|||
await SearchByAppAsync(dbContext, search, 1, ct); |
|||
} |
|||
|
|||
return search.Results.OrderByDescending(x => x.Score).Select(x => x.Id).Distinct().ToList(); |
|||
} |
|||
|
|||
private Task SearchBySchemaAsync(TContext context, SearchOperation search, IEnumerable<DomainId> schemaIds, double factor, |
|||
CancellationToken ct = default) |
|||
{ |
|||
var queryBuilder = |
|||
new SqlQueryBuilder(dialect, "Texts") |
|||
.Where(ClrFilter.Eq("AppId", search.App.Id)) |
|||
.Where(ClrFilter.In("SchemaId", schemaIds.ToList())) |
|||
.WhereMatch("Texts", search.SearchTerms) |
|||
.WhereScope(search.SearchScope); |
|||
|
|||
return SearchAsync(context, search, queryBuilder, factor, ct); |
|||
} |
|||
|
|||
private Task SearchByAppAsync(TContext context, SearchOperation search, double factor, |
|||
CancellationToken ct = default) |
|||
{ |
|||
var queryBuilder = |
|||
new SqlQueryBuilder(dialect, "Texts") |
|||
.Where(ClrFilter.Eq("AppId", search.App.Id)) |
|||
.WhereMatch("Texts", search.SearchTerms) |
|||
.WhereScope(search.SearchScope); |
|||
|
|||
return SearchAsync(context, search, queryBuilder, factor, ct); |
|||
} |
|||
|
|||
private static async Task SearchAsync(TContext context, SearchOperation search, SqlQueryBuilder queryBuilder, double factor, |
|||
CancellationToken ct) |
|||
{ |
|||
var (sql, parameters) = queryBuilder.Compile(); |
|||
|
|||
var ids = |
|||
await context.Set<EFTextIndexTextEntity>().FromSqlRaw(sql, parameters) |
|||
.Select(x => x.ContentId) |
|||
.ToListAsync(ct); |
|||
|
|||
search.Results.AddRange(ids.Select(x => (x, 1 * factor))); |
|||
} |
|||
|
|||
public async Task ExecuteAsync(IndexCommand[] commands, |
|||
CancellationToken ct = default) |
|||
{ |
|||
await using var dbContext = await CreateDbContextAsync(ct); |
|||
|
|||
var insertsText = new List<EFTextIndexTextEntity>(); |
|||
var insertsGeo = new List<EFTextIndexGeoEntity>(); |
|||
|
|||
foreach (var batch in commands.Batch(1000)) |
|||
{ |
|||
foreach (var command in batch) |
|||
{ |
|||
var (appId, contentId) = command.UniqueContentId; |
|||
var id = $"{appId}_{contentId}_{command.Stage}"; |
|||
|
|||
switch (command) |
|||
{ |
|||
case UpsertIndexEntry upsert: |
|||
if (upsert.Texts != null) |
|||
{ |
|||
insertsText.Add(new EFTextIndexTextEntity |
|||
{ |
|||
Id = id, |
|||
AppId = appId, |
|||
ContentId = contentId, |
|||
SchemaId = upsert.SchemaId.Id, |
|||
ServeAll = upsert.ServeAll, |
|||
ServePublished = upsert.ServePublished, |
|||
Stage = upsert.Stage, |
|||
Texts = Tokenizer.Terms(upsert.Texts), |
|||
}); |
|||
} |
|||
|
|||
foreach (var (field, obj) in upsert.GeoObjects.OrEmpty()) |
|||
{ |
|||
obj.SRID = 4326; |
|||
|
|||
if (!obj.IsValid) |
|||
{ |
|||
continue; |
|||
} |
|||
|
|||
var entity = new EFTextIndexGeoEntity |
|||
{ |
|||
Id = id, |
|||
AppId = appId, |
|||
ContentId = contentId, |
|||
GeoField = field, |
|||
GeoObject = obj, |
|||
SchemaId = upsert.SchemaId.Id, |
|||
ServeAll = upsert.ServeAll, |
|||
ServePublished = upsert.ServePublished, |
|||
Stage = upsert.Stage, |
|||
}; |
|||
|
|||
// We can only check the validatity by inserting them one by one.
|
|||
if (dialect is SqlServerDialect) |
|||
{ |
|||
try |
|||
{ |
|||
await dbContext.Set<EFTextIndexGeoEntity>().AddAsync(entity, ct); |
|||
await dbContext.SaveChangesAsync(ct); |
|||
} |
|||
catch (DbUpdateException ex) when (ex.InnerException is SqlException { Number: 8023 }) |
|||
{ |
|||
// Geo object is not valid.
|
|||
dbContext.Entry(entity).State = EntityState.Detached; |
|||
} |
|||
} |
|||
else |
|||
{ |
|||
insertsGeo.Add(entity); |
|||
} |
|||
} |
|||
|
|||
break; |
|||
case DeleteIndexEntry: |
|||
await dbContext.Set<EFTextIndexTextEntity>() |
|||
.Where(x => x.Id == id) |
|||
.ExecuteDeleteAsync(ct); |
|||
|
|||
await dbContext.Set<EFTextIndexGeoEntity>() |
|||
.Where(x => x.Id == id) |
|||
.ExecuteDeleteAsync(ct); |
|||
break; |
|||
case UpdateIndexEntry update: |
|||
await dbContext.Set<EFTextIndexTextEntity>() |
|||
.Where(x => x.Id == id) |
|||
.ExecuteUpdateAsync(u => u |
|||
.SetProperty(x => x.ServeAll, update.ServeAll) |
|||
.SetProperty(x => x.ServePublished, update.ServePublished), |
|||
ct); |
|||
|
|||
await dbContext.Set<EFTextIndexGeoEntity>() |
|||
.Where(x => x.Id == id) |
|||
.ExecuteUpdateAsync(u => u |
|||
.SetProperty(x => x.ServeAll, update.ServeAll) |
|||
.SetProperty(x => x.ServePublished, update.ServePublished), |
|||
ct); |
|||
break; |
|||
} |
|||
} |
|||
} |
|||
|
|||
await dbContext.BulkInsertOrUpdateAsync(insertsText, cancellationToken: ct); |
|||
await dbContext.BulkInsertOrUpdateAsync(insertsGeo, cancellationToken: ct); |
|||
} |
|||
|
|||
private Task<TContext> CreateDbContextAsync(CancellationToken ct) |
|||
{ |
|||
return dbContextFactory.CreateDbContextAsync(ct); |
|||
} |
|||
} |
|||
@ -0,0 +1,34 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.ComponentModel.DataAnnotations; |
|||
using NetTopologySuite.Geometries; |
|||
using Squidex.Infrastructure; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text; |
|||
|
|||
public sealed class EFTextIndexGeoEntity |
|||
{ |
|||
[Key] |
|||
required public string Id { get; set; } |
|||
|
|||
public DomainId AppId { get; set; } |
|||
|
|||
public DomainId SchemaId { get; set; } |
|||
|
|||
public DomainId ContentId { get; set; } |
|||
|
|||
public byte Stage { get; set; } |
|||
|
|||
public bool ServeAll { get; set; } |
|||
|
|||
public bool ServePublished { get; set; } |
|||
|
|||
public string GeoField { get; set; } |
|||
|
|||
public Geometry GeoObject { get; set; } |
|||
} |
|||
@ -0,0 +1,31 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.ComponentModel.DataAnnotations; |
|||
using Squidex.Infrastructure; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text; |
|||
|
|||
public sealed class EFTextIndexTextEntity |
|||
{ |
|||
[Key] |
|||
required public string Id { get; set; } |
|||
|
|||
public DomainId AppId { get; set; } |
|||
|
|||
public DomainId SchemaId { get; set; } |
|||
|
|||
public DomainId ContentId { get; set; } |
|||
|
|||
public byte Stage { get; set; } |
|||
|
|||
public bool ServeAll { get; set; } |
|||
|
|||
public bool ServePublished { get; set; } |
|||
|
|||
public string Texts { get; set; } |
|||
} |
|||
@ -0,0 +1,49 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Squidex.Infrastructure.Queries; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text; |
|||
|
|||
internal static class Extensions |
|||
{ |
|||
public static SqlQueryBuilder WhereScope(this SqlQueryBuilder queryBuilder, SearchScope scope) |
|||
{ |
|||
if (scope == SearchScope.All) |
|||
{ |
|||
return queryBuilder.Where(ClrFilter.Eq("ServeAll", true)); |
|||
} |
|||
else |
|||
{ |
|||
return queryBuilder.Where(ClrFilter.Eq("ServePublished", true)); |
|||
} |
|||
} |
|||
|
|||
public static IQueryable<EFTextIndexTextEntity> WhereScope(this IQueryable<EFTextIndexTextEntity> query, SearchScope scope) |
|||
{ |
|||
if (scope == SearchScope.All) |
|||
{ |
|||
return query.Where(x => x.ServeAll); |
|||
} |
|||
else |
|||
{ |
|||
return query.Where(x => x.ServePublished); |
|||
} |
|||
} |
|||
|
|||
public static IQueryable<EFTextIndexGeoEntity> WhereScope(this IQueryable<EFTextIndexGeoEntity> query, SearchScope scope) |
|||
{ |
|||
if (scope == SearchScope.All) |
|||
{ |
|||
return query.Where(x => x.ServeAll); |
|||
} |
|||
else |
|||
{ |
|||
return query.Where(x => x.ServePublished); |
|||
} |
|||
} |
|||
} |
|||
@ -1,38 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Squidex.Domain.Apps.Core.Apps; |
|||
using Squidex.Infrastructure; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text; |
|||
|
|||
public sealed class NullTextIndex : ITextIndex |
|||
{ |
|||
public Task ClearAsync( |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Task.CompletedTask; |
|||
} |
|||
|
|||
public Task ExecuteAsync(IndexCommand[] commands, |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Task.CompletedTask; |
|||
} |
|||
|
|||
public Task<List<DomainId>?> SearchAsync(App app, TextQuery query, SearchScope scope, |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Task.FromResult<List<DomainId>?>(null); |
|||
} |
|||
|
|||
public Task<List<DomainId>?> SearchAsync(App app, GeoQuery query, SearchScope scope, |
|||
CancellationToken ct = default) |
|||
{ |
|||
return Task.FromResult<List<DomainId>?>(null); |
|||
} |
|||
} |
|||
@ -0,0 +1,56 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Microsoft.EntityFrameworkCore; |
|||
using Microsoft.EntityFrameworkCore.Infrastructure; |
|||
|
|||
namespace Squidex.Infrastructure.Queries; |
|||
|
|||
public static class QueryExtensions |
|||
{ |
|||
public static async Task CreateGeoIndexAsync(this DatabaseFacade database, SqlDialect dialect, string name, string table, string column, |
|||
CancellationToken ct = default) |
|||
{ |
|||
var sql = dialect.GeoIndex(name, table, column); |
|||
try |
|||
{ |
|||
await database.ExecuteSqlRawAsync(sql, ct); |
|||
} |
|||
catch (Exception ex) when (dialect.IsDuplicateIndexException(ex, name)) |
|||
{ |
|||
// NOOP
|
|||
} |
|||
} |
|||
|
|||
public static async Task CreateTextIndexAsync(this DatabaseFacade database, SqlDialect dialect, string name, string table, string column, |
|||
CancellationToken ct = default) |
|||
{ |
|||
var prepareSql = dialect.TextIndexPrepare(name); |
|||
|
|||
if (!string.IsNullOrWhiteSpace(prepareSql)) |
|||
{ |
|||
try |
|||
{ |
|||
await database.ExecuteSqlRawAsync(prepareSql, ct); |
|||
} |
|||
catch (Exception ex) when (dialect.IsDuplicateIndexException(ex, name)) |
|||
{ |
|||
// NOOP
|
|||
} |
|||
} |
|||
|
|||
var sql = dialect.TextIndex(name, table, column); |
|||
try |
|||
{ |
|||
await database.ExecuteSqlRawAsync(sql, ct); |
|||
} |
|||
catch (Exception ex) when (dialect.IsDuplicateIndexException(ex, name)) |
|||
{ |
|||
// NOOP
|
|||
} |
|||
} |
|||
} |
|||
File diff suppressed because it is too large
@ -0,0 +1,74 @@ |
|||
using Microsoft.EntityFrameworkCore.Migrations; |
|||
using NetTopologySuite.Geometries; |
|||
|
|||
#nullable disable |
|||
|
|||
namespace Squidex.Providers.MySql.Migrations |
|||
{ |
|||
/// <inheritdoc />
|
|||
public partial class AddFullText : Migration |
|||
{ |
|||
/// <inheritdoc />
|
|||
protected override void Up(MigrationBuilder migrationBuilder) |
|||
{ |
|||
migrationBuilder.CreateTable( |
|||
name: "Geos", |
|||
columns: table => new |
|||
{ |
|||
Id = table.Column<string>(type: "varchar(400)", maxLength: 400, nullable: false) |
|||
.Annotation("MySql:CharSet", "utf8mb4"), |
|||
AppId = table.Column<string>(type: "varchar(255)", maxLength: 255, nullable: false) |
|||
.Annotation("MySql:CharSet", "utf8mb4"), |
|||
SchemaId = table.Column<string>(type: "varchar(255)", maxLength: 255, nullable: false) |
|||
.Annotation("MySql:CharSet", "utf8mb4"), |
|||
ContentId = table.Column<string>(type: "varchar(255)", maxLength: 255, nullable: false) |
|||
.Annotation("MySql:CharSet", "utf8mb4"), |
|||
Stage = table.Column<byte>(type: "tinyint unsigned", nullable: false), |
|||
ServeAll = table.Column<bool>(type: "tinyint(1)", nullable: false), |
|||
ServePublished = table.Column<bool>(type: "tinyint(1)", nullable: false), |
|||
GeoField = table.Column<string>(type: "varchar(255)", maxLength: 255, nullable: false) |
|||
.Annotation("MySql:CharSet", "utf8mb4"), |
|||
GeoObject = table.Column<Geometry>(type: "geometry", nullable: false) |
|||
}, |
|||
constraints: table => |
|||
{ |
|||
table.PrimaryKey("PK_Geos", x => x.Id); |
|||
}) |
|||
.Annotation("MySql:CharSet", "utf8mb4"); |
|||
|
|||
migrationBuilder.CreateTable( |
|||
name: "Texts", |
|||
columns: table => new |
|||
{ |
|||
Id = table.Column<string>(type: "varchar(400)", maxLength: 400, nullable: false) |
|||
.Annotation("MySql:CharSet", "utf8mb4"), |
|||
AppId = table.Column<string>(type: "varchar(255)", maxLength: 255, nullable: false) |
|||
.Annotation("MySql:CharSet", "utf8mb4"), |
|||
SchemaId = table.Column<string>(type: "varchar(255)", maxLength: 255, nullable: false) |
|||
.Annotation("MySql:CharSet", "utf8mb4"), |
|||
ContentId = table.Column<string>(type: "varchar(255)", maxLength: 255, nullable: false) |
|||
.Annotation("MySql:CharSet", "utf8mb4"), |
|||
Stage = table.Column<byte>(type: "tinyint unsigned", nullable: false), |
|||
ServeAll = table.Column<bool>(type: "tinyint(1)", nullable: false), |
|||
ServePublished = table.Column<bool>(type: "tinyint(1)", nullable: false), |
|||
Texts = table.Column<string>(type: "longtext", nullable: false) |
|||
.Annotation("MySql:CharSet", "utf8mb4") |
|||
}, |
|||
constraints: table => |
|||
{ |
|||
table.PrimaryKey("PK_Texts", x => x.Id); |
|||
}) |
|||
.Annotation("MySql:CharSet", "utf8mb4"); |
|||
} |
|||
|
|||
/// <inheritdoc />
|
|||
protected override void Down(MigrationBuilder migrationBuilder) |
|||
{ |
|||
migrationBuilder.DropTable( |
|||
name: "Geos"); |
|||
|
|||
migrationBuilder.DropTable( |
|||
name: "Texts"); |
|||
} |
|||
} |
|||
} |
|||
File diff suppressed because it is too large
@ -0,0 +1,68 @@ |
|||
using Microsoft.EntityFrameworkCore.Migrations; |
|||
using NetTopologySuite.Geometries; |
|||
|
|||
#nullable disable |
|||
|
|||
namespace Squidex.Providers.Postgres.Migrations |
|||
{ |
|||
/// <inheritdoc />
|
|||
public partial class AddFullText : Migration |
|||
{ |
|||
/// <inheritdoc />
|
|||
protected override void Up(MigrationBuilder migrationBuilder) |
|||
{ |
|||
migrationBuilder.AlterDatabase() |
|||
.Annotation("Npgsql:PostgresExtension:postgis", ",,"); |
|||
|
|||
migrationBuilder.CreateTable( |
|||
name: "Geos", |
|||
columns: table => new |
|||
{ |
|||
Id = table.Column<string>(type: "character varying(400)", maxLength: 400, nullable: false), |
|||
AppId = table.Column<string>(type: "character varying(255)", maxLength: 255, nullable: false), |
|||
SchemaId = table.Column<string>(type: "character varying(255)", maxLength: 255, nullable: false), |
|||
ContentId = table.Column<string>(type: "character varying(255)", maxLength: 255, nullable: false), |
|||
Stage = table.Column<byte>(type: "smallint", nullable: false), |
|||
ServeAll = table.Column<bool>(type: "boolean", nullable: false), |
|||
ServePublished = table.Column<bool>(type: "boolean", nullable: false), |
|||
GeoField = table.Column<string>(type: "character varying(255)", maxLength: 255, nullable: false), |
|||
GeoObject = table.Column<Geometry>(type: "geometry", nullable: false) |
|||
}, |
|||
constraints: table => |
|||
{ |
|||
table.PrimaryKey("PK_Geos", x => x.Id); |
|||
}); |
|||
|
|||
migrationBuilder.CreateTable( |
|||
name: "Texts", |
|||
columns: table => new |
|||
{ |
|||
Id = table.Column<string>(type: "character varying(400)", maxLength: 400, nullable: false), |
|||
AppId = table.Column<string>(type: "character varying(255)", maxLength: 255, nullable: false), |
|||
SchemaId = table.Column<string>(type: "character varying(255)", maxLength: 255, nullable: false), |
|||
ContentId = table.Column<string>(type: "character varying(255)", maxLength: 255, nullable: false), |
|||
Stage = table.Column<byte>(type: "smallint", nullable: false), |
|||
ServeAll = table.Column<bool>(type: "boolean", nullable: false), |
|||
ServePublished = table.Column<bool>(type: "boolean", nullable: false), |
|||
Texts = table.Column<string>(type: "text", nullable: false) |
|||
}, |
|||
constraints: table => |
|||
{ |
|||
table.PrimaryKey("PK_Texts", x => x.Id); |
|||
}); |
|||
} |
|||
|
|||
/// <inheritdoc />
|
|||
protected override void Down(MigrationBuilder migrationBuilder) |
|||
{ |
|||
migrationBuilder.DropTable( |
|||
name: "Geos"); |
|||
|
|||
migrationBuilder.DropTable( |
|||
name: "Texts"); |
|||
|
|||
migrationBuilder.AlterDatabase() |
|||
.OldAnnotation("Npgsql:PostgresExtension:postgis", ",,"); |
|||
} |
|||
} |
|||
} |
|||
File diff suppressed because it is too large
@ -0,0 +1,62 @@ |
|||
using Microsoft.EntityFrameworkCore.Migrations; |
|||
using NetTopologySuite.Geometries; |
|||
|
|||
#nullable disable |
|||
|
|||
namespace Squidex.Providers.SqlServer.Migrations |
|||
{ |
|||
/// <inheritdoc />
|
|||
public partial class AddFullText : Migration |
|||
{ |
|||
/// <inheritdoc />
|
|||
protected override void Up(MigrationBuilder migrationBuilder) |
|||
{ |
|||
migrationBuilder.CreateTable( |
|||
name: "Geos", |
|||
columns: table => new |
|||
{ |
|||
Id = table.Column<string>(type: "nvarchar(400)", maxLength: 400, nullable: false), |
|||
AppId = table.Column<string>(type: "nvarchar(255)", maxLength: 255, nullable: false), |
|||
SchemaId = table.Column<string>(type: "nvarchar(255)", maxLength: 255, nullable: false), |
|||
ContentId = table.Column<string>(type: "nvarchar(255)", maxLength: 255, nullable: false), |
|||
Stage = table.Column<byte>(type: "tinyint", nullable: false), |
|||
ServeAll = table.Column<bool>(type: "bit", nullable: false), |
|||
ServePublished = table.Column<bool>(type: "bit", nullable: false), |
|||
GeoField = table.Column<string>(type: "nvarchar(255)", maxLength: 255, nullable: false), |
|||
GeoObject = table.Column<Geometry>(type: "geography", nullable: false) |
|||
}, |
|||
constraints: table => |
|||
{ |
|||
table.PrimaryKey("PK_Geos", x => x.Id); |
|||
}); |
|||
|
|||
migrationBuilder.CreateTable( |
|||
name: "Texts", |
|||
columns: table => new |
|||
{ |
|||
Id = table.Column<string>(type: "nvarchar(400)", maxLength: 400, nullable: false), |
|||
AppId = table.Column<string>(type: "nvarchar(255)", maxLength: 255, nullable: false), |
|||
SchemaId = table.Column<string>(type: "nvarchar(255)", maxLength: 255, nullable: false), |
|||
ContentId = table.Column<string>(type: "nvarchar(255)", maxLength: 255, nullable: false), |
|||
Stage = table.Column<byte>(type: "tinyint", nullable: false), |
|||
ServeAll = table.Column<bool>(type: "bit", nullable: false), |
|||
ServePublished = table.Column<bool>(type: "bit", nullable: false), |
|||
Texts = table.Column<string>(type: "nvarchar(max)", nullable: false) |
|||
}, |
|||
constraints: table => |
|||
{ |
|||
table.PrimaryKey("PK_Texts", x => x.Id); |
|||
}); |
|||
} |
|||
|
|||
/// <inheritdoc />
|
|||
protected override void Down(MigrationBuilder migrationBuilder) |
|||
{ |
|||
migrationBuilder.DropTable( |
|||
name: "Geos"); |
|||
|
|||
migrationBuilder.DropTable( |
|||
name: "Texts"); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,42 @@ |
|||
<Project Sdk="Microsoft.NET.Sdk"> |
|||
<PropertyGroup> |
|||
<TargetFramework>netstandard2.0</TargetFramework> |
|||
<LangVersion>latest</LangVersion> |
|||
<ImplicitUsings>enable</ImplicitUsings> |
|||
<IsRoslynComponent>true</IsRoslynComponent> |
|||
<Nullable>enable</Nullable> |
|||
<LangVersion>latest</LangVersion> |
|||
<RootNamespace>Squidex</RootNamespace> |
|||
<EnforceExtendedAnalyzerRules>true</EnforceExtendedAnalyzerRules> |
|||
</PropertyGroup> |
|||
<ItemGroup> |
|||
<None Remove="Template.handlebar" /> |
|||
</ItemGroup> |
|||
<ItemGroup> |
|||
<PackageReference Include="Handlebars.Net" Version="2.1.6" PrivateAssets="all" GeneratePathProperty="true" /> |
|||
<PackageReference Include="Meziantou.Analyzer" Version="2.0.179"> |
|||
<PrivateAssets>all</PrivateAssets> |
|||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> |
|||
</PackageReference> |
|||
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.12.0" PrivateAssets="all" /> |
|||
<PackageReference Include="Microsoft.CodeAnalysis.Analyzers" Version="3.11.0"> |
|||
<PrivateAssets>all</PrivateAssets> |
|||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> |
|||
</PackageReference> |
|||
<PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" /> |
|||
</ItemGroup> |
|||
<PropertyGroup> |
|||
<GetTargetPathDependsOn>$(GetTargetPathDependsOn);GetDependencyTargetPaths</GetTargetPathDependsOn> |
|||
</PropertyGroup> |
|||
<Target Name="GetDependencyTargetPaths"> |
|||
<ItemGroup> |
|||
<TargetPathWithTargetPlatformMoniker Include="$(PKGHandlebars_Net)\lib\netstandard2.0\Handlebars.dll" IncludeRuntimeDependency="false" /> |
|||
</ItemGroup> |
|||
</Target> |
|||
<ItemGroup> |
|||
<AdditionalFiles Include="..\..\stylecop.json" Link="stylecop.json" /> |
|||
</ItemGroup> |
|||
<ItemGroup> |
|||
<EmbeddedResource Include="Template.handlebar" /> |
|||
</ItemGroup> |
|||
</Project> |
|||
@ -0,0 +1,23 @@ |
|||
#pragma warning disable |
|||
// Auto-generated code |
|||
using Squidex.EntityFramework.TestHelpers; |
|||
|
|||
namespace {{classNamespace}}; |
|||
|
|||
[Trait("Category", "TestContainer")] |
|||
[Collection("Postgres")] |
|||
public class Postgres{{className}}(PostgresFixture fixture) : {{baseName}}<TestDbContextPostgres>(fixture) |
|||
{ |
|||
} |
|||
|
|||
[Trait("Category", "TestContainer")] |
|||
[Collection("MySql")] |
|||
public class MySql{{className}}(MySqlFixture fixture) : {{baseName}}<TestDbContextMySql>(fixture) |
|||
{ |
|||
} |
|||
|
|||
[Trait("Category", "TestContainer")] |
|||
[Collection("SqlServer")] |
|||
public class SqlServer{{className}}(SqlServerFixture fixture) : {{baseName}}<TestDbContextSqlServer>(fixture) |
|||
{ |
|||
} |
|||
@ -0,0 +1,17 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
namespace Squidex; |
|||
|
|||
internal sealed class TemplateModel |
|||
{ |
|||
public string ClassNamespace { get; set; } |
|||
|
|||
public string ClassName { get; set; } |
|||
|
|||
public string BaseName { get; set; } |
|||
} |
|||
@ -0,0 +1,77 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Diagnostics; |
|||
using System.Text; |
|||
using HandlebarsDotNet; |
|||
using Microsoft.CodeAnalysis; |
|||
using Microsoft.CodeAnalysis.CSharp; |
|||
using Microsoft.CodeAnalysis.CSharp.Syntax; |
|||
using Microsoft.CodeAnalysis.Text; |
|||
|
|||
namespace Squidex.Data.Tests.CodeGenerator; |
|||
|
|||
[Generator] |
|||
public class TestGenerator : IIncrementalGenerator |
|||
{ |
|||
public void Initialize(IncrementalGeneratorInitializationContext context) |
|||
{ |
|||
var templateStream = typeof(TestGenerator).Assembly.GetManifestResourceStream("Squidex.Template.handlebar")!; |
|||
var templateText = new StreamReader(templateStream).ReadToEnd(); |
|||
|
|||
var template = Handlebars.Compile(templateText); |
|||
|
|||
static TemplateModel? Transform(GeneratorSyntaxContext ctx) |
|||
{ |
|||
var classSyntax = (ClassDeclarationSyntax)ctx.Node; |
|||
|
|||
var className = classSyntax.Identifier.Text; |
|||
if (!className.StartsWith("EF", StringComparison.OrdinalIgnoreCase)) |
|||
{ |
|||
return null; |
|||
} |
|||
|
|||
if (!classSyntax.Modifiers.Any(x => x.IsKind(SyntaxKind.AbstractKeyword))) |
|||
{ |
|||
return null; |
|||
} |
|||
|
|||
if (classSyntax.TypeParameterList == null || |
|||
classSyntax.TypeParameterList.Parameters.Count != 1 || |
|||
classSyntax.TypeParameterList.Parameters[0].Identifier.Text != "TContext") |
|||
{ |
|||
return null; |
|||
} |
|||
|
|||
var namespaceDeclaration = |
|||
classSyntax.Ancestors() |
|||
.OfType<BaseNamespaceDeclarationSyntax>().First(); |
|||
|
|||
return new TemplateModel |
|||
{ |
|||
BaseName = classSyntax.Identifier.Text, |
|||
ClassName = classSyntax.Identifier.Text.Substring(2), |
|||
ClassNamespace = namespaceDeclaration.Name.ToString(), |
|||
}; |
|||
} |
|||
|
|||
var fieldDeclarations = context.SyntaxProvider.CreateSyntaxProvider( |
|||
static (node, _) => |
|||
{ |
|||
return node is ClassDeclarationSyntax; |
|||
}, |
|||
static (ctx, _) => Transform(ctx)) |
|||
.Where(x => x != null); |
|||
|
|||
context.RegisterSourceOutput(fieldDeclarations, (context, model) => |
|||
{ |
|||
var source = template(model); |
|||
|
|||
context.AddSource($"{model!.BaseName}_Tests.cs", SourceText.From(source, Encoding.UTF8)); |
|||
}); |
|||
} |
|||
} |
|||
@ -0,0 +1,25 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Microsoft.EntityFrameworkCore; |
|||
using Squidex.Domain.Apps.Entities.Contents.Text; |
|||
using Squidex.EntityFramework.TestHelpers; |
|||
|
|||
namespace Squidex.EntityFramework.Domain.Contents.Text; |
|||
|
|||
public abstract class EFTextIndexTests<TContext>(ISqlFixture<TContext> fixture) : TextIndexerTests where TContext : DbContext |
|||
{ |
|||
public override bool SupportsQuerySyntax => false; |
|||
|
|||
public override async Task<ITextIndex> CreateSutAsync() |
|||
{ |
|||
var sut = new EFTextIndex<TContext>(fixture.DbContextFactory, fixture.Dialect); |
|||
|
|||
await sut.InitializeAsync(default); |
|||
return sut; |
|||
} |
|||
} |
|||
@ -1,29 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Squidex.EntityFramework.TestHelpers; |
|||
using Squidex.Infrastructure.Queries; |
|||
using Squidex.Providers.MySql; |
|||
|
|||
namespace Squidex.EntityFramework.Infrastructure.Queries; |
|||
|
|||
[Trait("Category", "TestContainer")] |
|||
[Collection("MySql")] |
|||
public class MySqlQueryTests(MySqlFixture fixture) : SqlQueryTests<TestDbContext> |
|||
{ |
|||
protected override async Task<TestDbContext> CreateDbContextAsync() |
|||
{ |
|||
var context = await fixture.DbContextFactory.CreateDbContextAsync(); |
|||
|
|||
return context; |
|||
} |
|||
|
|||
protected override SqlDialect CreateDialect() |
|||
{ |
|||
return MySqlDialect.Instance; |
|||
} |
|||
} |
|||
@ -1,29 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Squidex.EntityFramework.TestHelpers; |
|||
using Squidex.Infrastructure.Queries; |
|||
using Squidex.Providers.Postgres; |
|||
|
|||
namespace Squidex.EntityFramework.Infrastructure.Queries; |
|||
|
|||
[Trait("Category", "TestContainer")] |
|||
[Collection("Postgres")] |
|||
public class PostgresQueryTests(PostgresFixture fixture) : SqlQueryTests<TestDbContext> |
|||
{ |
|||
protected override async Task<TestDbContext> CreateDbContextAsync() |
|||
{ |
|||
var context = await fixture.DbContextFactory.CreateDbContextAsync(); |
|||
|
|||
return context; |
|||
} |
|||
|
|||
protected override SqlDialect CreateDialect() |
|||
{ |
|||
return PostgresDialect.Instance; |
|||
} |
|||
} |
|||
@ -1,29 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Squidex.EntityFramework.TestHelpers; |
|||
using Squidex.Infrastructure.Queries; |
|||
using Squidex.Providers.SqlServer; |
|||
|
|||
namespace Squidex.EntityFramework.Infrastructure.Queries; |
|||
|
|||
[Trait("Category", "TestContainer")] |
|||
[Collection("SqlServer")] |
|||
public class SqlServerQueryTests(SqlServerFixture fixture) : SqlQueryTests<TestDbContext> |
|||
{ |
|||
protected override async Task<TestDbContext> CreateDbContextAsync() |
|||
{ |
|||
var context = await fixture.DbContextFactory.CreateDbContextAsync(); |
|||
|
|||
return context; |
|||
} |
|||
|
|||
protected override SqlDialect CreateDialect() |
|||
{ |
|||
return SqlServerDialect.Instance; |
|||
} |
|||
} |
|||
@ -0,0 +1,18 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Microsoft.EntityFrameworkCore; |
|||
using Squidex.Infrastructure.Queries; |
|||
|
|||
namespace Squidex.EntityFramework.TestHelpers; |
|||
|
|||
public interface ISqlFixture<TContext> where TContext : DbContext |
|||
{ |
|||
SqlDialect Dialect { get; } |
|||
|
|||
IDbContextFactory<TContext> DbContextFactory { get; } |
|||
} |
|||
@ -0,0 +1,23 @@ |
|||
# Base OS layer: Latest Ubuntu LTS |
|||
FROM --platform=linux/amd64 ubuntu:focal |
|||
|
|||
# Install prerequistes since it is needed to get repo config for SQL server |
|||
RUN export DEBIAN_FRONTEND=noninteractive && \ |
|||
apt-get update && \ |
|||
apt-get install -yq curl apt-transport-https gnupg && \ |
|||
# Get official Microsoft repository configuration |
|||
curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \ |
|||
curl https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb --output packages-microsoft-prod.deb && dpkg -i packages-microsoft-prod.deb && \ |
|||
curl https://packages.microsoft.com/config/ubuntu/20.04/mssql-server-2022.list | tee /etc/apt/sources.list.d/mssql-server.list && \ |
|||
apt-get update && \ |
|||
# Install SQL Server from apt |
|||
apt-get install -y mssql-server && \ |
|||
# Install optional packages |
|||
apt-get install -y mssql-server-fts && \ |
|||
ACCEPT_EULA=Y apt-get install -y mssql-tools && \ |
|||
# Cleanup the Dockerfile |
|||
apt-get clean && \ |
|||
rm -rf /var/lib/apt/lists |
|||
|
|||
# Run SQL Server process |
|||
CMD /opt/mssql/bin/sqlservr |
|||
Loading…
Reference in new issue