Initial commit

This commit is contained in:
Marco Minerva
2024-06-14 11:47:00 +02:00
parent 246bf56955
commit 9284ae5377
23 changed files with 905 additions and 16 deletions
+297
View File
@@ -0,0 +1,297 @@
# Remove the line below if you want to inherit .editorconfig settings from higher directories
root = true
[*]
#### Core EditorConfig Options ####
# Indentation and spacing
indent_size = 4
indent_style = space
tab_width = 4
trim_trailing_whitespace = true
# New line preferences
end_of_line = unset
insert_final_newline = unset
dotnet_style_null_propagation = true:suggestion
dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion
dotnet_style_prefer_auto_properties = true:suggestion
dotnet_style_operator_placement_when_wrapping = beginning_of_line
dotnet_style_object_initializer = true:suggestion
dotnet_style_coalesce_expression = true:suggestion
dotnet_style_collection_initializer = true:suggestion
dotnet_style_prefer_simplified_boolean_expressions = true:suggestion
dotnet_style_prefer_conditional_expression_over_assignment = false:silent
dotnet_style_prefer_conditional_expression_over_return = false:silent
dotnet_style_explicit_tuple_names = true:suggestion
dotnet_style_prefer_inferred_tuple_names = true:suggestion
dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion
dotnet_style_prefer_compound_assignment = true:suggestion
dotnet_style_prefer_simplified_interpolation = true:suggestion
dotnet_style_namespace_match_folder = false:silent
dotnet_style_readonly_field = true:suggestion
dotnet_style_predefined_type_for_locals_parameters_members = true:silent
dotnet_style_predefined_type_for_member_access = true:silent
dotnet_style_require_accessibility_modifiers = for_non_interface_members:silent
dotnet_style_allow_multiple_blank_lines_experimental = false:error
dotnet_style_allow_statement_immediately_after_block_experimental = false:error
dotnet_code_quality_unused_parameters = all:suggestion
dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent
dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent
dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent
dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent
dotnet_style_qualification_for_event = false:silent
dotnet_style_qualification_for_method = false:silent
dotnet_style_qualification_for_property = false:silent
dotnet_style_qualification_for_field = false:silent
# C# files
[*.cs]
#### .NET Coding Conventions ####
# Organize usings
dotnet_separate_import_directive_groups = false
dotnet_sort_system_directives_first = true
file_header_template = unset
# this. and Me. preferences
dotnet_style_qualification_for_event = false:silent
dotnet_style_qualification_for_field = false:silent
dotnet_style_qualification_for_method = false:silent
dotnet_style_qualification_for_property = false:silent
# Language keywords vs BCL types preferences
dotnet_style_predefined_type_for_locals_parameters_members = true:silent
dotnet_style_predefined_type_for_member_access = true:silent
# Parentheses preferences
dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent
dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent
dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent
dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent
# Modifier preferences
dotnet_style_require_accessibility_modifiers = for_non_interface_members:silent
# Expression-level preferences
csharp_style_prefer_local_over_anonymous_function = true:silent
csharp_style_prefer_extended_property_pattern = true:suggestion
csharp_style_implicit_object_creation_when_type_is_apparent = true:silent
csharp_style_prefer_tuple_swap = true:silent
# Field preferences
dotnet_style_readonly_field = true:suggestion
# Parameter preferences
dotnet_code_quality_unused_parameters = all:suggestion
# Suppression preferences
dotnet_remove_unnecessary_suppression_exclusions = none
#### C# Coding Conventions ####
# var preferences
csharp_style_var_elsewhere = true:suggestion
csharp_style_var_for_built_in_types = true:suggestion
csharp_style_var_when_type_is_apparent = true:suggestion
# Expression-bodied members preferences
csharp_style_expression_bodied_accessors = true:silent
csharp_style_expression_bodied_constructors = false:silent
csharp_style_expression_bodied_indexers = true:silent
csharp_style_expression_bodied_lambdas = true:silent
csharp_style_expression_bodied_local_functions = true:silent
csharp_style_expression_bodied_methods = when_on_single_line:silent
csharp_style_expression_bodied_operators = true:silent
csharp_style_expression_bodied_properties = true:silent
# Pattern matching preferences
csharp_style_pattern_matching_over_as_with_null_check = true:suggestion
csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion
csharp_style_prefer_not_pattern = true:suggestion
csharp_style_prefer_pattern_matching = true:suggestion
csharp_style_prefer_switch_expression = true:suggestion
# Null-checking preferences
csharp_style_conditional_delegate_call = true:suggestion
csharp_style_prefer_parameter_null_checking = true:suggestion
csharp_style_prefer_null_check_over_type_check = true:suggestion
# Modifier preferences
csharp_prefer_static_local_function = true:suggestion
csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:silent
# Code-block preferences
csharp_style_prefer_top_level_statements = true:suggestion
csharp_style_prefer_primary_constructors = true:suggestion
csharp_prefer_braces = true:silent
csharp_prefer_simple_using_statement = true:suggestion
csharp_style_namespace_declarations = file_scoped:suggestion
csharp_style_prefer_method_group_conversion = true:silent
# Expression-level preferences
csharp_prefer_simple_default_expression = true:suggestion
csharp_style_deconstructed_variable_declaration = false:suggestion
csharp_style_inlined_variable_declaration = true:suggestion
csharp_style_pattern_local_over_anonymous_function = true:suggestion
csharp_style_prefer_index_operator = true:suggestion
csharp_style_prefer_range_operator = true:suggestion
csharp_style_throw_expression = true:suggestion
csharp_style_unused_value_assignment_preference = discard_variable:none
csharp_style_unused_value_expression_statement_preference = discard_variable:none
# 'using' directive preferences
csharp_using_directive_placement = outside_namespace:suggestion
# Struct preferences
csharp_style_prefer_readonly_struct = true:suggestion
csharp_style_prefer_readonly_struct_member = true:suggestion
#### C# Formatting Rules ####
# New line preferences
csharp_new_line_before_catch = true
csharp_new_line_before_else = true
csharp_new_line_before_finally = true
csharp_new_line_before_members_in_anonymous_types = true
csharp_new_line_before_members_in_object_initializers = true
csharp_new_line_before_open_brace = all
csharp_new_line_between_query_expression_clauses = true
csharp_style_allow_embedded_statements_on_same_line_experimental = false:error
csharp_style_allow_blank_lines_between_consecutive_braces_experimental = false:error
csharp_style_allow_blank_line_after_colon_in_constructor_initializer_experimental = true:silent
csharp_style_allow_blank_line_after_token_in_conditional_expression_experimental = true:silent
csharp_style_allow_blank_line_after_token_in_arrow_expression_clause_experimental = true:silent
# Indentation preferences
csharp_indent_block_contents = true
csharp_indent_braces = false
csharp_indent_case_contents = true
csharp_indent_case_contents_when_block = true
csharp_indent_labels = one_less_than_current
csharp_indent_switch_labels = true
# Space preferences
csharp_space_after_cast = false
csharp_space_after_colon_in_inheritance_clause = true
csharp_space_after_comma = true
csharp_space_after_dot = false
csharp_space_after_keywords_in_control_flow_statements = true
csharp_space_after_semicolon_in_for_statement = true
csharp_space_around_binary_operators = before_and_after
csharp_space_around_declaration_statements = false
csharp_space_before_colon_in_inheritance_clause = true
csharp_space_before_comma = false
csharp_space_before_dot = false
csharp_space_before_open_square_brackets = false
csharp_space_before_semicolon_in_for_statement = false
csharp_space_between_empty_square_brackets = false
csharp_space_between_method_call_empty_parameter_list_parentheses = false
csharp_space_between_method_call_name_and_opening_parenthesis = false
csharp_space_between_method_call_parameter_list_parentheses = false
csharp_space_between_method_declaration_empty_parameter_list_parentheses = false
csharp_space_between_method_declaration_name_and_open_parenthesis = false
csharp_space_between_method_declaration_parameter_list_parentheses = false
csharp_space_between_parentheses = false
csharp_space_between_square_brackets = false
# Wrapping preferences
csharp_preserve_single_line_blocks = true
csharp_preserve_single_line_statements = true
csharp_style_prefer_utf8_string_literals = true:suggestion
#### Naming styles ####
# Naming rules
dotnet_naming_rule.interface_should_be_begins_with_i.severity = suggestion
dotnet_naming_rule.interface_should_be_begins_with_i.symbols = interface
dotnet_naming_rule.interface_should_be_begins_with_i.style = begins_with_i
dotnet_naming_rule.types_should_be_pascal_case.severity = suggestion
dotnet_naming_rule.types_should_be_pascal_case.symbols = types
dotnet_naming_rule.types_should_be_pascal_case.style = pascal_case
dotnet_naming_rule.non_field_members_should_be_pascal_case.severity = suggestion
dotnet_naming_rule.non_field_members_should_be_pascal_case.symbols = non_field_members
dotnet_naming_rule.non_field_members_should_be_pascal_case.style = pascal_case
dotnet_naming_rule.constant_fields_should_be_upper_case.severity = suggestion
dotnet_naming_rule.constant_fields_should_be_upper_case.symbols = constant_fields
dotnet_naming_rule.constant_fields_should_be_upper_case.style = pascal_case
dotnet_naming_symbols.constant_fields.applicable_kinds = field
dotnet_naming_symbols.constant_fields.applicable_accessibilities = *
dotnet_naming_symbols.constant_fields.required_modifiers = const
dotnet_naming_rule.private_or_internal_field_should_be_camel_case.severity = suggestion
dotnet_naming_rule.private_or_internal_field_should_be_camel_case.symbols = private_or_internal_field
dotnet_naming_rule.private_or_internal_field_should_be_camel_case.style = camel_case
dotnet_naming_rule.method_should_be_pascal_case.severity = suggestion
dotnet_naming_rule.method_should_be_pascal_case.symbols = method
dotnet_naming_rule.method_should_be_pascal_case.style = pascal_case
dotnet_naming_rule.async_method_should_be_ends_with_async.severity = suggestion
dotnet_naming_rule.async_method_should_be_ends_with_async.symbols = async_method
dotnet_naming_rule.async_method_should_be_ends_with_async.style = ends_with_async
# Symbol specifications
dotnet_naming_symbols.interface.applicable_kinds = interface
dotnet_naming_symbols.interface.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected
dotnet_naming_symbols.interface.required_modifiers =
dotnet_naming_symbols.method.applicable_kinds = method
dotnet_naming_symbols.method.applicable_accessibilities = public
dotnet_naming_symbols.method.required_modifiers =
dotnet_naming_symbols.private_or_internal_field.applicable_kinds = field
dotnet_naming_symbols.private_or_internal_field.applicable_accessibilities = internal, private, private_protected
dotnet_naming_symbols.private_or_internal_field.required_modifiers =
dotnet_naming_symbols.types.applicable_kinds = class, struct, interface, enum
dotnet_naming_symbols.types.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected
dotnet_naming_symbols.types.required_modifiers =
dotnet_naming_symbols.non_field_members.applicable_kinds = property, event, method
dotnet_naming_symbols.non_field_members.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected
dotnet_naming_symbols.non_field_members.required_modifiers =
dotnet_naming_symbols.async_method.applicable_kinds = method
dotnet_naming_symbols.async_method.applicable_accessibilities = *
dotnet_naming_symbols.async_method.required_modifiers = async
# Naming styles
dotnet_naming_style.pascal_case.required_prefix =
dotnet_naming_style.pascal_case.required_suffix =
dotnet_naming_style.pascal_case.word_separator =
dotnet_naming_style.pascal_case.capitalization = pascal_case
dotnet_naming_style.begins_with_i.required_prefix = I
dotnet_naming_style.begins_with_i.required_suffix =
dotnet_naming_style.begins_with_i.word_separator =
dotnet_naming_style.begins_with_i.capitalization = pascal_case
dotnet_naming_style.camel_case.required_prefix =
dotnet_naming_style.camel_case.required_suffix =
dotnet_naming_style.camel_case.word_separator =
dotnet_naming_style.camel_case.capitalization = camel_case
dotnet_naming_style.ends_with_async.required_prefix =
dotnet_naming_style.ends_with_async.required_suffix = Async
dotnet_naming_style.ends_with_async.word_separator =
dotnet_naming_style.ends_with_async.capitalization = pascal_case
# IDE0058: Expression value is never used
dotnet_diagnostic.IDE0058.severity = none
# IDE0010: Add missing cases
dotnet_diagnostic.IDE0010.severity = none
# IDE0072: Add missing cases
dotnet_diagnostic.IDE0072.severity = none
+63
View File
@@ -0,0 +1,63 @@
###############################################################################
# Set default behavior to automatically normalize line endings.
###############################################################################
* text=auto
###############################################################################
# Set default behavior for command prompt diff.
#
# This is need for earlier builds of msysgit that does not have it on by
# default for csharp files.
# Note: This is only used by command line
###############################################################################
#*.cs diff=csharp
###############################################################################
# Set the merge driver for project and solution files
#
# Merging from the command prompt will add diff markers to the files if there
# are conflicts (Merging from VS is not affected by the settings below, in VS
# the diff markers are never inserted). Diff markers may cause the following
# file extensions to fail to load in VS. An alternative would be to treat
# these files as binary and thus will always conflict and require user
# intervention with every merge. To do so, just uncomment the entries below
###############################################################################
#*.sln merge=binary
#*.csproj merge=binary
#*.vbproj merge=binary
#*.vcxproj merge=binary
#*.vcproj merge=binary
#*.dbproj merge=binary
#*.fsproj merge=binary
#*.lsproj merge=binary
#*.wixproj merge=binary
#*.modelproj merge=binary
#*.sqlproj merge=binary
#*.wwaproj merge=binary
###############################################################################
# behavior for image files
#
# image files are treated as binary by default.
###############################################################################
#*.jpg binary
#*.png binary
#*.gif binary
###############################################################################
# diff behavior for common document formats
#
# Convert binary document formats to text before diffing them. This feature
# is only available from the command line. Turn it on by uncommenting the
# entries below.
###############################################################################
#*.doc diff=astextplain
#*.DOC diff=astextplain
#*.docx diff=astextplain
#*.DOCX diff=astextplain
#*.dot diff=astextplain
#*.DOT diff=astextplain
#*.pdf diff=astextplain
#*.PDF diff=astextplain
#*.rtf diff=astextplain
#*.RTF diff=astextplain
+3 -15
View File
@@ -29,6 +29,7 @@ x86/
bld/
[Bb]in/
[Oo]bj/
[Oo]ut/
[Ll]og/
[Ll]ogs/
@@ -90,7 +91,6 @@ StyleCopReport.xml
*.tmp_proj
*_wpftmp.csproj
*.log
*.tlog
*.vspscc
*.vssscc
.builds
@@ -294,17 +294,6 @@ node_modules/
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
*.vbw
# Visual Studio 6 auto-generated project file (contains which files were open etc.)
*.vbp
# Visual Studio 6 workspace and project file (working project files containing files to include in project)
*.dsw
*.dsp
# Visual Studio 6 technical files
*.ncb
*.aps
# Visual Studio LightSwitch build output
**/*.HTMLClient/GeneratedArtifacts
**/*.DesktopClient/GeneratedArtifacts
@@ -361,9 +350,6 @@ ASALocalRun/
# Local History for Visual Studio
.localhistory/
# Visual Studio History (VSHistory) files
.vshistory/
# BeatPulse healthcheck temp database
healthchecksdb
@@ -396,3 +382,5 @@ FodyWeavers.xsd
# JetBrains Rider
*.sln.iml
/SqlDatabaseVectorSearch/appsettings.local.json
+7
View File
@@ -0,0 +1,7 @@
<Project>
<PropertyGroup>
<ArtifactsPath>$(MSBuildThisFileDirectory)artifacts</ArtifactsPath>
</PropertyGroup>
</Project>
+32
View File
@@ -0,0 +1,32 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.8.34330.188
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SqlDatabaseVectorSearch", "SqlDatabaseVectorSearch\SqlDatabaseVectorSearch.csproj", "{A30F41AA-3FC1-41BE-99B7-7637A6EADDDC}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{0D00EFA8-60BD-47AF-BE33-9D219B8AC7F6}"
ProjectSection(SolutionItems) = preProject
.editorconfig = .editorconfig
Directory.Build.props = Directory.Build.props
README.md = README.md
EndProjectSection
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{A30F41AA-3FC1-41BE-99B7-7637A6EADDDC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A30F41AA-3FC1-41BE-99B7-7637A6EADDDC}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A30F41AA-3FC1-41BE-99B7-7637A6EADDDC}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A30F41AA-3FC1-41BE-99B7-7637A6EADDDC}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {F8D9A242-E395-4B2D-BF14-0C15B70E9D10}
EndGlobalSection
EndGlobal
@@ -0,0 +1,43 @@
using EntityFramework.Exceptions.SqlServer;
using Microsoft.EntityFrameworkCore;
using SqlDatabaseVectorSearch.DataAccessLayer.Entities;
namespace SqlDatabaseVectorSearch.DataAccessLayer;
public class ApplicationDbContext(DbContextOptions<ApplicationDbContext> options) : DbContext(options)
{
public virtual DbSet<Document> Documents { get; set; }
public virtual DbSet<DocumentChunk> DocumentChunks { get; set; }
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
{
base.OnConfiguring(optionsBuilder);
optionsBuilder.UseExceptionProcessor();
}
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
modelBuilder.Entity<Document>(entity =>
{
entity.Property(e => e.Id).HasDefaultValueSql("(newid())");
entity.Property(e => e.Name)
.IsRequired()
.HasMaxLength(255);
});
modelBuilder.Entity<DocumentChunk>(entity =>
{
entity.Property(e => e.Id).HasDefaultValueSql("(newid())");
entity.Property(e => e.Content).IsRequired();
entity.Property(e => e.Embedding)
.IsRequired()
.HasMaxLength(8000).IsVector();
entity.HasOne(d => d.Document).WithMany(p => p.DocumentChunks)
.HasForeignKey(d => d.DocumentId)
.OnDelete(DeleteBehavior.NoAction)
.HasConstraintName("FK_DocumentChunks_Documents");
});
}
}
@@ -0,0 +1,12 @@
namespace SqlDatabaseVectorSearch.DataAccessLayer.Entities;
public class Document
{
public Guid Id { get; set; }
public required string Name { get; set; }
public DateTimeOffset CreationDate { get; set; }
public virtual ICollection<DocumentChunk> DocumentChunks { get; set; } = [];
}
@@ -0,0 +1,14 @@
namespace SqlDatabaseVectorSearch.DataAccessLayer.Entities;
public partial class DocumentChunk
{
public Guid Id { get; set; }
public Guid DocumentId { get; set; }
public required string Content { get; set; }
public required float[] Embedding { get; set; }
public virtual Document Document { get; set; } = null!;
}
@@ -0,0 +1,3 @@
namespace SqlDatabaseVectorSearch.Models;
public record class MemoryResponse(string Question, string Answer);
@@ -0,0 +1,3 @@
namespace SqlDatabaseVectorSearch.Models;
public record Question(Guid ConversationId, string Text) : Search(Text);
+4
View File
@@ -0,0 +1,4 @@
namespace SqlDatabaseVectorSearch.Models;
public record Search(string Text);
@@ -0,0 +1,3 @@
namespace SqlDatabaseVectorSearch.Models;
public record class UploadDocumentResponse(Guid DocumentId);
+134
View File
@@ -0,0 +1,134 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.OpenApi.Models;
using Microsoft.SemanticKernel;
using MinimalHelpers.OpenApi;
using SqlDatabaseVectorSearch.DataAccessLayer;
using SqlDatabaseVectorSearch.Models;
using SqlDatabaseVectorSearch.Services;
using SqlDatabaseVectorSearch.Settings;
using TinyHelpers.AspNetCore.Extensions;
using TinyHelpers.AspNetCore.Swagger;
var builder = WebApplication.CreateBuilder(args);
builder.Configuration.AddJsonFile("appsettings.local.json", optional: true, reloadOnChange: true);
// Add services to the container.
var aiSettings = builder.Configuration.GetSection<AzureOpenAISettings>("AzureOpenAI")!;
var appSettings = builder.Services.ConfigureAndGet<AppSettings>(builder.Configuration, nameof(AppSettings))!;
builder.Services.AddSqlServer<ApplicationDbContext>(builder.Configuration.GetConnectionString("SqlConnection"), options =>
{
options.EnableRetryOnFailure(3, TimeSpan.FromSeconds(1), null);
options.UseVectorSearch();
});
builder.Services.AddMemoryCache();
// Semantical Kernel is used to reformulate questions taking into account all the previous interactions, so that embeddings can be generate more accurately.
builder.Services.AddKernel()
.AddAzureOpenAIChatCompletion(aiSettings.ChatCompletion.Deployment, aiSettings.ChatCompletion.Endpoint, aiSettings.ChatCompletion.ApiKey);
builder.Services.AddScoped<ChatService>();
builder.Services.AddScoped<VectorSearchService>();
builder.Services.AddEndpointsApiExplorer();
builder.Services.AddSwaggerGen(options =>
{
options.SwaggerDoc("v1", new OpenApiInfo { Title = "SQL Server Vector Search API", Version = "v1" });
options.AddDefaultResponse();
options.AddFormFile();
});
builder.Services.AddDefaultProblemDetails();
builder.Services.AddDefaultExceptionHandler();
var app = builder.Build();
// Configure the HTTP request pipeline.
app.UseHttpsRedirection();
app.UseExceptionHandler();
app.UseStatusCodePages();
if (app.Environment.IsDevelopment())
{
app.UseSwagger();
app.UseSwaggerUI(options =>
{
options.RoutePrefix = string.Empty;
options.SwaggerEndpoint("/swagger/v1/swagger.json", "Kernel Memory Service API v1");
options.InjectStylesheet("/css/swagger.css");
});
}
var documentsApiGroup = app.MapGroup("/api/documents");
documentsApiGroup.MapPost(string.Empty, async (IFormFile file, VectorSearchService vectorSearchService, LinkGenerator linkGenerator, Guid? documentId = null) =>
{
documentId = await vectorSearchService.ImportAsync(file.OpenReadStream(), file.FileName, documentId);
return TypedResults.Ok(new UploadDocumentResponse(documentId.Value));
})
.DisableAntiforgery()
.WithOpenApi(operation =>
{
operation.Summary = "Uploads a document. Currently, only PDF files are supported";
operation.Description = "Uploads a document to SQL Server. The document will be indexed and used to answer questions. The documentId is optional, if not provided a new one will be generated. If you specify an existing documentId, the document will be overridden.";
operation.Parameter("documentId").Description = "The unique identifier of the document. If not provided, a new one will be generated. If you specify an existing documentId, the document will be overridden.";
return operation;
})
;
documentsApiGroup.MapDelete("{documentId:guid}", async (Guid documentId, VectorSearchService vectorSearchService) =>
{
await vectorSearchService.DeleteDocumentAsync(documentId);
return TypedResults.NoContent();
})
.WithOpenApi(operation =>
{
operation.Summary = "Delete a document from SQL Server";
return operation;
});
//app.MapPost("/api/search", async (Search search, ApplicationMemoryService memory, double minimumRelevance = 0, string? index = null) =>
//{
// var response = await memory.SearchAsync(search, minimumRelevance, index);
// return TypedResults.Ok(response);
//})
//.WithOpenApi(operation =>
//{
// operation.Summary = "Search into Kernel Memory";
// operation.Description = "Search into Kernel Memory using the provided question and optional tags. If tags are provided, they will be used as filters with OR logic.";
// operation.Parameter("minimumRelevance").Description = "The minimum Cosine Similarity required.";
// operation.Parameter("index").Description = "The index in which to search for documents. If not provided, the default index will be used ('default').";
// return operation;
//});
//app.MapPost("/api/ask", async Task<Results<Ok<MemoryResponse>, NotFound>> (Question question, ApplicationMemoryService memory, bool reformulate = true, double minimumRelevance = 0, string? index = null) =>
//{
// var response = await memory.AskQuestionAsync(question, reformulate, minimumRelevance, index);
// if (response is null)
// {
// return TypedResults.NotFound();
// }
// return TypedResults.Ok(response);
//})
//.WithOpenApi(operation =>
//{
// operation.Summary = "Ask a question to the Kernel Memory Service";
// operation.Description = "Ask a question to the Kernel Memory Service using the provided question and optional tags. The question will be reformulated taking into account the context of the chat identified by the given ConversationId. If tags are provided, they will be used as filters with OR logic.";
// operation.Parameter("reformulate").Description = "If true, the question will be reformulated taking into account the context of the chat identified by the given ConversationId.";
// operation.Parameter("minimumRelevance").Description = "The minimum Cosine Similarity required.";
// operation.Parameter("index").Description = "The index in which to search for documents. If not provided, the default index will be used ('default').";
// return operation;
//});
app.Run();
@@ -0,0 +1,15 @@
{
"$schema": "http://json.schemastore.org/launchsettings.json",
"profiles": {
"https": {
"commandName": "Project",
"dotnetRunMessages": true,
"launchBrowser": true,
"launchUrl": "",
"applicationUrl": "https://localhost:7024;http://localhost:5178",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
}
}
}
@@ -0,0 +1,53 @@
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Options;
using Microsoft.SemanticKernel.ChatCompletion;
using SqlDatabaseVectorSearch.Settings;
namespace SqlDatabaseVectorSearch.Services;
public class ChatService(IMemoryCache cache, IChatCompletionService chatCompletionService, IOptions<AppSettings> appSettingsOptions)
{
public async Task<string> CreateQuestionAsync(Guid conversationId, string question)
{
var chat = new ChatHistory(cache.Get<ChatHistory?>(conversationId) ?? []);
var embeddingQuestion = $"""
Reformulate the following question taking into account the context of the chat to perform embeddings search:
---
{question}
---
You must reformulate the question in the same language of the user's question.
Never add "in this chat", "in the context of this chat", "in the context of our conversation", "search for" or something like that in your answer.
""";
chat.AddUserMessage(embeddingQuestion);
var reformulatedQuestion = await chatCompletionService.GetChatMessageContentAsync(chat)!;
chat.AddAssistantMessage(reformulatedQuestion.Content!);
await UpdateCacheAsync(conversationId, chat);
return reformulatedQuestion.Content!;
}
public async Task AddInteractionAsync(Guid conversationId, string question, string answer)
{
var chat = new ChatHistory(cache.Get<ChatHistory?>(conversationId) ?? []);
chat.AddUserMessage(question);
chat.AddAssistantMessage(answer);
await UpdateCacheAsync(conversationId, chat);
}
private Task UpdateCacheAsync(Guid conversationId, ChatHistory chat)
{
if (chat.Count > appSettingsOptions.Value.MessageLimit)
{
chat = new ChatHistory(chat.TakeLast(appSettingsOptions.Value.MessageLimit));
}
cache.Set(conversationId, chat, appSettingsOptions.Value.MessageExpiration);
return Task.CompletedTask;
}
}
@@ -0,0 +1,112 @@
using System.Text;
using Microsoft.EntityFrameworkCore;
using Microsoft.SemanticKernel.Embeddings;
using Microsoft.SemanticKernel.Text;
using SqlDatabaseVectorSearch.DataAccessLayer;
using SqlDatabaseVectorSearch.DataAccessLayer.Entities;
using UglyToad.PdfPig;
using UglyToad.PdfPig.DocumentLayoutAnalysis.TextExtractor;
namespace SqlDatabaseVectorSearch.Services;
public class VectorSearchService(ApplicationDbContext dbContext, ITextEmbeddingGenerationService textEmbeddingGenerationService, ChatService chatService)
{
public async Task<Guid> ImportAsync(Stream stream, string name, Guid? documentId)
{
// Extract the contents of the file (current, only PDF are supported).
var content = await GetContentAsync(stream);
if (documentId.HasValue)
{
// If the user is importing a document that already exists, delete the previous one.
await DeleteDocumentAsync(documentId.Value);
}
else
{
// Creates a new document.
documentId = Guid.NewGuid();
}
var document = new Document { Id = documentId.Value, Name = name, CreationDate = DateTimeOffset.UtcNow };
dbContext.Documents.Add(document);
// Splits the content into chunks of at most 1024 tokens and generate the embeddings for each one.
var paragraphs = TextChunker.SplitPlainTextParagraphs(TextChunker.SplitPlainTextLines(content, 300), 1024, 100);
var embeddings = await textEmbeddingGenerationService.GenerateEmbeddingsAsync(paragraphs);
foreach (var (paragraph, embedding) in paragraphs.Zip(embeddings, (p, e) => (p, e.ToArray())))
{
var documentChunk = new DocumentChunk { DocumentId = documentId.Value, Content = paragraph, Embedding = embedding };
dbContext.DocumentChunks.Add(documentChunk);
}
await dbContext.SaveChangesAsync();
return documentId.Value;
}
public async Task DeleteDocumentAsync(Guid documentId)
{
var document = await dbContext.Documents.Include(d => d.DocumentChunks).FirstOrDefaultAsync(d => d.Id == documentId);
if (document is null)
{
return;
}
dbContext.DocumentChunks.RemoveRange(document.DocumentChunks);
dbContext.Documents.Remove(document);
await dbContext.SaveChangesAsync();
}
//public async Task<MemoryResponse?> AskQuestionAsync(Question question, bool reformulate = true, double minimumRelevance = 0, string? index = null)
//{
// // Reformulate the following question taking into account the context of the chat to perform keyword search and embeddings:
// var reformulatedQuestion = reformulate ? await chatService.CreateQuestionAsync(question.ConversationId, question.Text) : question.Text;
// // Ask using the embedding search via Kernel Memory and the reformulated question.
// // If tags are provided, use them as filters with OR logic.
// var answer = await memory.AskAsync(reformulatedQuestion.TrimEnd([' ', '?']), index, filters: question.Tags.ToMemoryFilters(), minRelevance: minimumRelevance);
// // If you want to use an AND logic, set the "filter" parameter (instead of "filters").
// //var answer = await memory.AskAsync(reformulatedQuestion.TrimEnd([' ', '?'], index, filter: question.Tags.ToMemoryFilter(), minRelevance: minimumRelevance);
// if (answer.NoResult == false)
// {
// // If the answer has been found, add the interaction to the chat, so that it will be used for the next reformulation.
// await chatService.AddInteractionAsync(question.ConversationId, reformulatedQuestion, answer.Result);
// var response = new MemoryResponse(answer.Question, answer.Result, answer.RelevantSources);
// return response;
// }
// return null;
//}
//public async Task<SearchResult?> SearchAsync(Search search, double minimumRelevance = 0, string? index = null)
//{
// // Search using the embedding search via Kernel Memory .
// // If tags are provided, use them as filters with OR logic.
// var searchResult = await memory.SearchAsync(search.Text.TrimEnd([' ', '?']), index, filters: search.Tags.ToMemoryFilters(), minRelevance: minimumRelevance, limit: 50);
// // If you want to use an AND logic, set the "filter" parameter (instead of "filters").
// //var searchResult = await memory.SearchAsync(search.Text.TrimEnd([' ', '?']), index, filter: search.Tags.ToMemoryFilter(), minRelevance: minimumRelevance);
// return searchResult;
//}
private Task<string> GetContentAsync(Stream stream)
{
var content = new StringBuilder();
// Reads the content of the PDF document using PdfPig.
using var pdfDocument = PdfDocument.Open(stream);
foreach (var page in pdfDocument.GetPages().Where(x => x != null))
{
var pageContent = ContentOrderTextExtractor.GetText(page) ?? string.Empty;
content.AppendLine(pageContent);
}
return Task.FromResult(content.ToString());
}
}
@@ -0,0 +1,14 @@
namespace SqlDatabaseVectorSearch.Settings;
public class AppSettings
{
public int MessageLimit { get; init; }
public TimeSpan MessageExpiration { get; init; }
public required string StoragePath { get; init; }
public required string VectorDbPath { get; init; }
public required string QueuePath { get; init; }
}
@@ -0,0 +1,19 @@
namespace SqlDatabaseVectorSearch.Settings;
public class AzureOpenAISettings
{
public required ServiceSettings ChatCompletion { get; init; }
public required ServiceSettings Embedding { get; init; }
}
public class ServiceSettings
{
public required string Endpoint { get; init; }
public required string Deployment { get; init; }
public required string ApiKey { get; init; }
public required int MaxTokens { get; init; } = 8191;
}
@@ -0,0 +1,23 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<NoWarn>$(NoWarn);SKEXP0001;SKEXP0050;</NoWarn>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="EFCore.SqlServer.VectorSearch" Version="0.1.1" />
<PackageReference Include="EntityFrameworkCore.Exceptions.SqlServer" Version="8.1.2" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.6" />
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="8.0.6" />
<PackageReference Include="Microsoft.SemanticKernel" Version="1.14.1" />
<PackageReference Include="MinimalHelpers.OpenApi" Version="2.0.8" />
<PackageReference Include="PdfPig" Version="0.1.8" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
<PackageReference Include="TinyHelpers" Version="3.1.5" />
<PackageReference Include="TinyHelpers.AspNetCore" Version="3.1.4" />
</ItemGroup>
</Project>
@@ -0,0 +1,12 @@
{
"ConnectionStrings": {
"SqlConnection": ""
},
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.AspNetCore": "Warning",
"Microsoft.KernelMemory": "Debug"
}
}
}
+34
View File
@@ -0,0 +1,34 @@
{
"ConnectionStrings": {
"SqlConnection": ""
},
"AzureOpenAI": {
"ChatCompletion": {
"Endpoint": "",
"Deployment": "",
"ApiKey": "",
"MaxTokens": 32768
},
"Embedding": {
"Endpoint": "",
"Deployment": "",
"ApiKey": "",
"MaxTokens": 8191
}
},
"AppSettings": {
"MessageLimit": 20,
"MessageExpiration": "00:05:00",
"StoragePath": "",
"VectorDbPath": "",
"QueuePath": ""
},
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.AspNetCore": "Warning"
}
},
"https_port": 443,
"AllowedHosts": "*"
}
@@ -0,0 +1,4 @@
.swagger-ui .parameters-col_description input[type=checkbox] {
max-width: 340px;
width: auto;
}