mirror of https://github.com/Squidex/squidex.git
Sebastian Stehle
4 years ago
committed by
GitHub
GitHub
34 changed files with 612 additions and 124 deletions
@ -0,0 +1,130 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Linq; |
|||
using System.Reflection; |
|||
using Azure.Search.Documents.Indexes.Models; |
|||
|
|||
namespace Squidex.Extensions.Text.Azure |
|||
{ |
|||
public static class AzureIndexDefinition |
|||
{ |
|||
private static readonly Dictionary<string, (string Field, string Analyzer)> AllowedLanguages = new Dictionary<string, (string Field, string Analyzer)>(StringComparer.OrdinalIgnoreCase) |
|||
{ |
|||
["iv"] = ("text_iv", LexicalAnalyzerName.StandardLucene.ToString()), |
|||
["zh"] = ("text_zh", LexicalAnalyzerName.ZhHansLucene.ToString()) |
|||
}; |
|||
|
|||
static AzureIndexDefinition() |
|||
{ |
|||
var analyzers = |
|||
typeof(LexicalAnalyzerName) |
|||
.GetProperties(BindingFlags.Public | BindingFlags.Static) |
|||
.Select(x => x.GetValue(null)) |
|||
.Select(x => x.ToString()) |
|||
.OrderBy(x => x) |
|||
.ToList(); |
|||
|
|||
var addedLanguage = new HashSet<string>(); |
|||
|
|||
foreach (var analyzer in analyzers) |
|||
{ |
|||
var indexOfDot = analyzer.IndexOf('.', StringComparison.Ordinal); |
|||
|
|||
if (indexOfDot > 0) |
|||
{ |
|||
var language = analyzer[0..indexOfDot]; |
|||
|
|||
var isValidLanguage = |
|||
language.Length == 2 || |
|||
language.StartsWith("zh-", StringComparison.Ordinal); |
|||
|
|||
if (isValidLanguage && addedLanguage.Add(language)) |
|||
{ |
|||
var fieldName = $"text_{language.Replace('-', '_')}"; |
|||
|
|||
AllowedLanguages[language] = (fieldName, analyzer); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
public static string GetTextField(string key) |
|||
{ |
|||
if (AllowedLanguages.TryGetValue(key, out var field)) |
|||
{ |
|||
return field.Field; |
|||
} |
|||
|
|||
if (key.Length > 2 && AllowedLanguages.TryGetValue(key[2..], out field)) |
|||
{ |
|||
return field.Field; |
|||
} |
|||
|
|||
return AllowedLanguages["iv"].Field; |
|||
} |
|||
|
|||
public static SearchIndex Create(string indexName) |
|||
{ |
|||
var fields = new List<SearchField> |
|||
{ |
|||
new SimpleField("docId", SearchFieldDataType.String) |
|||
{ |
|||
IsKey = true, |
|||
}, |
|||
new SimpleField("appId", SearchFieldDataType.String) |
|||
{ |
|||
IsFilterable = true, |
|||
}, |
|||
new SimpleField("appName", SearchFieldDataType.String) |
|||
{ |
|||
IsFilterable = false |
|||
}, |
|||
new SimpleField("contentId", SearchFieldDataType.String) |
|||
{ |
|||
IsFilterable = false |
|||
}, |
|||
new SearchableField("schemaId") |
|||
{ |
|||
IsFilterable = true |
|||
}, |
|||
new SimpleField("schemaName", SearchFieldDataType.String) |
|||
{ |
|||
IsFilterable = false |
|||
}, |
|||
new SimpleField("serveAll", SearchFieldDataType.Boolean) |
|||
{ |
|||
IsFilterable = true |
|||
}, |
|||
new SimpleField("servePublished", SearchFieldDataType.Boolean) |
|||
{ |
|||
IsFilterable = true |
|||
} |
|||
}; |
|||
|
|||
foreach (var (field, analyzer) in AllowedLanguages.Values) |
|||
{ |
|||
fields.Add( |
|||
new SearchableField(field) |
|||
{ |
|||
IsFilterable = false, |
|||
IsFacetable = false, |
|||
AnalyzerName = analyzer |
|||
}); |
|||
} |
|||
|
|||
var index = new SearchIndex(indexName) |
|||
{ |
|||
Fields = fields |
|||
}; |
|||
|
|||
return index; |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,147 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Linq; |
|||
using System.Text; |
|||
using System.Threading; |
|||
using System.Threading.Tasks; |
|||
using Azure; |
|||
using Azure.Search.Documents; |
|||
using Azure.Search.Documents.Indexes; |
|||
using Azure.Search.Documents.Models; |
|||
using Squidex.Domain.Apps.Entities.Apps; |
|||
using Squidex.Domain.Apps.Entities.Contents; |
|||
using Squidex.Domain.Apps.Entities.Contents.Text; |
|||
using Squidex.Hosting; |
|||
using Squidex.Infrastructure; |
|||
|
|||
namespace Squidex.Extensions.Text.Azure |
|||
{ |
|||
public sealed class AzureTextIndex : IInitializable, ITextIndex |
|||
{ |
|||
private readonly SearchIndexClient indexClient; |
|||
private readonly SearchClient searchClient; |
|||
|
|||
public AzureTextIndex( |
|||
string serviceEndpoint, |
|||
string serviceApiKey, |
|||
string indexName) |
|||
{ |
|||
indexClient = new SearchIndexClient(new Uri(serviceEndpoint), new AzureKeyCredential(serviceApiKey)); |
|||
|
|||
searchClient = indexClient.GetSearchClient(indexName); |
|||
} |
|||
|
|||
public async Task InitializeAsync( |
|||
CancellationToken ct) |
|||
{ |
|||
await CreateIndexAsync(ct); |
|||
} |
|||
|
|||
public async Task ClearAsync( |
|||
CancellationToken ct = default) |
|||
{ |
|||
await indexClient.DeleteIndexAsync(searchClient.IndexName, ct); |
|||
|
|||
await CreateIndexAsync(ct); |
|||
} |
|||
|
|||
private async Task CreateIndexAsync( |
|||
CancellationToken ct) |
|||
{ |
|||
var index = AzureIndexDefinition.Create(searchClient.IndexName); |
|||
|
|||
await indexClient.CreateOrUpdateIndexAsync(index, true, true, ct); |
|||
} |
|||
|
|||
public async Task ExecuteAsync(IndexCommand[] commands, |
|||
CancellationToken ct = default) |
|||
{ |
|||
var batch = IndexDocumentsBatch.Create<SearchDocument>(); |
|||
|
|||
commands.Foreach(x => CommandFactory.CreateCommands(x, batch.Actions)); |
|||
|
|||
await searchClient.IndexDocumentsAsync(batch, cancellationToken: ct); |
|||
} |
|||
|
|||
public Task<List<DomainId>> SearchAsync(IAppEntity app, GeoQuery query, SearchScope scope, |
|||
CancellationToken ct = default) |
|||
{ |
|||
Guard.NotNull(app, nameof(app)); |
|||
Guard.NotNull(query, nameof(query)); |
|||
|
|||
return Task.FromResult<List<DomainId>>(null); |
|||
} |
|||
|
|||
public async Task<List<DomainId>> SearchAsync(IAppEntity app, TextQuery query, SearchScope scope, |
|||
CancellationToken ct = default) |
|||
{ |
|||
Guard.NotNull(app, nameof(app)); |
|||
Guard.NotNull(query, nameof(query)); |
|||
|
|||
if (string.IsNullOrWhiteSpace(query.Text)) |
|||
{ |
|||
return null; |
|||
} |
|||
|
|||
var searchOptions = new SearchOptions |
|||
{ |
|||
Filter = BuildFilter(app, query, scope) |
|||
}; |
|||
|
|||
searchOptions.Select.Add("contentId"); |
|||
searchOptions.Size = 2000; |
|||
|
|||
var results = await searchClient.SearchAsync<SearchDocument>("*", searchOptions, ct); |
|||
|
|||
var ids = new List<DomainId>(); |
|||
|
|||
await foreach (var item in results.Value.GetResultsAsync().WithCancellation(ct)) |
|||
{ |
|||
if (item != null) |
|||
{ |
|||
ids.Add(DomainId.Create(item.Document["contentId"].ToString())); |
|||
} |
|||
} |
|||
|
|||
return ids; |
|||
} |
|||
|
|||
private static string BuildFilter(IAppEntity app, TextQuery query, SearchScope scope) |
|||
{ |
|||
var sb = new StringBuilder(); |
|||
|
|||
sb.Append($"appId eq '{app.Id}' and {GetServeField(scope)} eq true"); |
|||
|
|||
if (query.RequiredSchemaIds?.Count > 0) |
|||
{ |
|||
var schemaIds = string.Join(" or ", query.RequiredSchemaIds.Select(x => $"schemaId eq '{x}'")); |
|||
|
|||
sb.Append($" and ({schemaIds}) and search.ismatchscoring('{query.Text}')"); |
|||
} |
|||
else if (query.PreferredSchemaId.HasValue) |
|||
{ |
|||
sb.Append($" and ((search.ismatchscoring('{query.Text}') and search.ismatchscoring('{query.PreferredSchemaId}', 'schemaId')) or search.ismatchscoring('{query.Text}'))"); |
|||
} |
|||
else |
|||
{ |
|||
sb.Append($" and search.ismatchscoring('{query.Text}')"); |
|||
} |
|||
|
|||
return sb.ToString(); |
|||
} |
|||
|
|||
private static string GetServeField(SearchScope scope) |
|||
{ |
|||
return scope == SearchScope.Published ? |
|||
"servePublished" : |
|||
"serveAll"; |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,62 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using Microsoft.Extensions.Configuration; |
|||
using Microsoft.Extensions.DependencyInjection; |
|||
using Squidex.Domain.Apps.Entities.Contents.Text; |
|||
using Squidex.Hosting; |
|||
using Squidex.Hosting.Configuration; |
|||
using Squidex.Infrastructure.Plugins; |
|||
|
|||
namespace Squidex.Extensions.Text.Azure |
|||
{ |
|||
public sealed class AzureTextPlugin : IPlugin |
|||
{ |
|||
public void ConfigureServices(IServiceCollection services, IConfiguration config) |
|||
{ |
|||
var fullTextType = config.GetValue<string>("fullText:type"); |
|||
|
|||
if (string.Equals(fullTextType, "Azure", StringComparison.OrdinalIgnoreCase)) |
|||
{ |
|||
var serviceEndpoint = config.GetValue<string>("fullText:azure:serviceEndpoint"); |
|||
|
|||
if (string.IsNullOrWhiteSpace(serviceEndpoint)) |
|||
{ |
|||
var error = new ConfigurationError("Value is required.", "fullText:azure:serviceEndpoint"); |
|||
|
|||
throw new ConfigurationException(error); |
|||
} |
|||
|
|||
var serviceApiKey = config.GetValue<string>("fullText:azure:apiKey"); |
|||
|
|||
if (string.IsNullOrWhiteSpace(serviceApiKey)) |
|||
{ |
|||
var error = new ConfigurationError("Value is required.", "fullText:azure:apiKey"); |
|||
|
|||
throw new ConfigurationException(error); |
|||
} |
|||
|
|||
var indexName = config.GetValue<string>("fullText:azure:indexName"); |
|||
|
|||
if (string.IsNullOrWhiteSpace(indexName)) |
|||
{ |
|||
indexName = "squidex-index"; |
|||
} |
|||
|
|||
services.AddSingleton( |
|||
c => new AzureTextIndex(serviceEndpoint, serviceApiKey, indexName)); |
|||
|
|||
services.AddSingleton<ITextIndex>( |
|||
c => c.GetRequiredService<AzureTextIndex>()); |
|||
|
|||
services.AddSingleton<IInitializable>( |
|||
c => c.GetRequiredService<AzureTextIndex>()); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,81 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Text; |
|||
using Azure.Search.Documents.Models; |
|||
using Squidex.Domain.Apps.Entities.Contents.Text; |
|||
|
|||
namespace Squidex.Extensions.Text.Azure |
|||
{ |
|||
public static class CommandFactory |
|||
{ |
|||
public static void CreateCommands(IndexCommand command, IList<IndexDocumentsAction<SearchDocument>> batch) |
|||
{ |
|||
switch (command) |
|||
{ |
|||
case UpsertIndexEntry upsert: |
|||
batch.Add(UpsertEntry(upsert)); |
|||
break; |
|||
case UpdateIndexEntry update: |
|||
batch.Add(UpdateEntry(update)); |
|||
break; |
|||
case DeleteIndexEntry delete: |
|||
batch.Add(DeleteEntry(delete)); |
|||
break; |
|||
} |
|||
} |
|||
|
|||
private static IndexDocumentsAction<SearchDocument> UpsertEntry(UpsertIndexEntry upsert) |
|||
{ |
|||
var searchDocument = new SearchDocument |
|||
{ |
|||
["docId"] = upsert.DocId.ToBase64(), |
|||
["appId"] = upsert.AppId.Id.ToString(), |
|||
["appName"] = upsert.AppId.Name, |
|||
["contentId"] = upsert.ContentId.ToString(), |
|||
["schemaId"] = upsert.SchemaId.Id.ToString(), |
|||
["schemaName"] = upsert.SchemaId.Name, |
|||
["serveAll"] = upsert.ServeAll, |
|||
["servePublished"] = upsert.ServePublished |
|||
}; |
|||
|
|||
if (upsert.Texts != null) |
|||
{ |
|||
foreach (var (key, value) in upsert.Texts) |
|||
{ |
|||
searchDocument[AzureIndexDefinition.GetTextField(key)] = value; |
|||
} |
|||
} |
|||
|
|||
return IndexDocumentsAction.MergeOrUpload(searchDocument); |
|||
} |
|||
|
|||
private static IndexDocumentsAction<SearchDocument> UpdateEntry(UpdateIndexEntry update) |
|||
{ |
|||
var searchDocument = new SearchDocument |
|||
{ |
|||
["docId"] = update.DocId.ToBase64(), |
|||
["serveAll"] = update.ServeAll, |
|||
["servePublished"] = update.ServePublished, |
|||
}; |
|||
|
|||
return IndexDocumentsAction.MergeOrUpload(searchDocument); |
|||
} |
|||
|
|||
private static IndexDocumentsAction<SearchDocument> DeleteEntry(DeleteIndexEntry delete) |
|||
{ |
|||
return IndexDocumentsAction.Delete("docId", delete.DocId.ToBase64()); |
|||
} |
|||
|
|||
private static string ToBase64(this string value) |
|||
{ |
|||
return Convert.ToBase64String(Encoding.Default.GetBytes(value)); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,53 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using Microsoft.Extensions.Configuration; |
|||
using Microsoft.Extensions.DependencyInjection; |
|||
using Squidex.Domain.Apps.Entities.Contents.Text; |
|||
using Squidex.Hosting; |
|||
using Squidex.Hosting.Configuration; |
|||
using Squidex.Infrastructure.Plugins; |
|||
|
|||
namespace Squidex.Extensions.Text.ElasticSearch |
|||
{ |
|||
public sealed class ElasticSearchTextPlugin : IPlugin |
|||
{ |
|||
public void ConfigureServices(IServiceCollection services, IConfiguration config) |
|||
{ |
|||
var fullTextType = config.GetValue<string>("fullText:type"); |
|||
|
|||
if (string.Equals(fullTextType, "elastic", StringComparison.OrdinalIgnoreCase)) |
|||
{ |
|||
var elasticConfiguration = config.GetValue<string>("fullText:elastic:configuration"); |
|||
|
|||
if (string.IsNullOrWhiteSpace(elasticConfiguration)) |
|||
{ |
|||
var error = new ConfigurationError("Value is required.", "fullText:elastic:configuration"); |
|||
|
|||
throw new ConfigurationException(error); |
|||
} |
|||
|
|||
var indexName = config.GetValue<string>("fullText:elastic:indexName"); |
|||
|
|||
if (string.IsNullOrWhiteSpace(indexName)) |
|||
{ |
|||
indexName = "squidex-index"; |
|||
} |
|||
|
|||
services.AddSingleton( |
|||
c => new ElasticSearchTextIndex(elasticConfiguration, indexName)); |
|||
|
|||
services.AddSingleton<ITextIndex>( |
|||
c => c.GetRequiredService<ElasticSearchTextIndex>()); |
|||
|
|||
services.AddSingleton<IInitializable>( |
|||
c => c.GetRequiredService<ElasticSearchTextIndex>()); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
Loading…
Reference in new issue