From f5011bc44bbd56fdf17f1534fbbdf7ab420fc168 Mon Sep 17 00:00:00 2001 From: Marco Minerva Date: Wed, 10 Sep 2025 16:05:10 +0200 Subject: [PATCH 01/11] Upgrade to .NET 10.0 and update NuGet dependencies Updated the project to target .NET 10.0, replacing the previous .NET 9.0 target framework. Upgraded several NuGet packages to their latest pre-release versions: - `Microsoft.AspNetCore.OpenApi` to `10.0.0-rc.1.25451.107` - `Microsoft.EntityFrameworkCore.SqlServer` to `10.0.0-rc.1.25451.107` - `Microsoft.EntityFrameworkCore.Tools` to `10.0.0-rc.1.25451.107` Retained `` and `` settings for `Microsoft.EntityFrameworkCore.Tools`. These changes ensure compatibility with the updated framework and leverage new features and improvements. --- SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj b/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj index 5802c66..ecd9ad7 100644 --- a/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj +++ b/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj @@ -1,7 +1,7 @@  - net9.0 + net10.0 enable enable $(NoWarn);SKEXP0010;SKEXP0050 @@ -13,9 +13,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive From 404cd7565a7a1d05ba6d042030e9b8fa2a2fb788 Mon Sep 17 00:00:00 2001 From: Marco Minerva Date: Wed, 10 Sep 2025 16:45:16 +0200 Subject: [PATCH 02/11] Switch to SqlVector for embeddings Updated the application to use SQL Server's native vector data type (`SqlVector`) for embeddings, replacing the previous `float[]` or `string` representations. - Updated `.editorconfig` with new code style preferences and diagnostic rule severities. - Modified `DocumentChunk.cs` to use `SqlVector` for the `Embedding` property. - Updated migrations and `ApplicationDbContextModelSnapshot` to reflect the new `SqlVector` type. - Replaced `AddAzureSql` with `AddSqlServer` in `Program.cs` and removed `UseVectorSearch`. - Adjusted `DocumentService` and `VectorSearchService` to handle `SqlVector` and updated vector search logic. - Removed the `EFCore.SqlServer.VectorSearch` package and upgraded EF Core to `10.0.0-rc.1`. - Made minor adjustments to OpenAPI configuration and dependency management. --- .editorconfig | 6 +++++- SqlDatabaseVectorSearch/Data/Entities/DocumentChunk.cs | 6 ++++-- .../Data/Migrations/00000000000000_Initial.Designer.cs | 8 ++++---- .../Data/Migrations/00000000000000_Initial.cs | 3 ++- .../Migrations/ApplicationDbContextModelSnapshot.cs | 6 +++--- SqlDatabaseVectorSearch/Program.cs | 10 +++------- SqlDatabaseVectorSearch/Services/DocumentService.cs | 2 +- .../Services/VectorSearchService.cs | 6 ++++-- SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj | 1 - 9 files changed, 26 insertions(+), 22 deletions(-) diff --git a/.editorconfig b/.editorconfig index f6b7d3a..8f8512a 100644 --- a/.editorconfig +++ b/.editorconfig @@ -82,6 +82,7 @@ csharp_style_prefer_local_over_anonymous_function = true:silent csharp_style_prefer_extended_property_pattern = true:suggestion csharp_style_implicit_object_creation_when_type_is_apparent = true:silent csharp_style_prefer_tuple_swap = true:silent +csharp_style_prefer_simple_property_accessors = true:suggestion # Field preferences dotnet_style_readonly_field = true:suggestion @@ -299,4 +300,7 @@ dotnet_diagnostic.IDE0010.severity = none dotnet_diagnostic.IDE0072.severity = none # IDE0305: Simplify collection initialization -dotnet_diagnostic.IDE0305.severity = none \ No newline at end of file +dotnet_diagnostic.IDE0305.severity = none + +# CA1873: Avoid potentially expensive logging +dotnet_diagnostic.CA1873.severity = none \ No newline at end of file diff --git a/SqlDatabaseVectorSearch/Data/Entities/DocumentChunk.cs b/SqlDatabaseVectorSearch/Data/Entities/DocumentChunk.cs index bc67963..0f91b28 100644 --- a/SqlDatabaseVectorSearch/Data/Entities/DocumentChunk.cs +++ b/SqlDatabaseVectorSearch/Data/Entities/DocumentChunk.cs @@ -1,4 +1,6 @@ -namespace SqlDatabaseVectorSearch.Data.Entities; +using Microsoft.Data.SqlTypes; + +namespace SqlDatabaseVectorSearch.Data.Entities; public class DocumentChunk { @@ -14,7 +16,7 @@ public class DocumentChunk public required string Content { get; set; } - public required float[] Embedding { get; set; } + public required SqlVector Embedding { get; set; } public virtual Document Document { get; set; } = null!; } \ No newline at end of file diff --git a/SqlDatabaseVectorSearch/Data/Migrations/00000000000000_Initial.Designer.cs b/SqlDatabaseVectorSearch/Data/Migrations/00000000000000_Initial.Designer.cs index c071cf1..e4f78bc 100644 --- a/SqlDatabaseVectorSearch/Data/Migrations/00000000000000_Initial.Designer.cs +++ b/SqlDatabaseVectorSearch/Data/Migrations/00000000000000_Initial.Designer.cs @@ -1,5 +1,6 @@ // using System; +using Microsoft.Data.SqlTypes; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Metadata; @@ -12,7 +13,7 @@ using SqlDatabaseVectorSearch.Data; namespace SqlDatabaseVectorSearch.Migrations { [DbContext(typeof(ApplicationDbContext))] - [Migration("20250606091336_Initial")] + [Migration("00000000000000_Initial")] partial class Initial { /// @@ -20,7 +21,7 @@ namespace SqlDatabaseVectorSearch.Migrations { #pragma warning disable 612, 618 modelBuilder - .HasAnnotation("ProductVersion", "9.0.5") + .HasAnnotation("ProductVersion", "10.0.0-rc.1.25451.107") .HasAnnotation("Relational:MaxIdentifierLength", 128); SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); @@ -57,8 +58,7 @@ namespace SqlDatabaseVectorSearch.Migrations b.Property("DocumentId") .HasColumnType("uniqueidentifier"); - b.PrimitiveCollection("Embedding") - .IsRequired() + b.Property>("Embedding") .HasColumnType("vector(1536)"); b.Property("Index") diff --git a/SqlDatabaseVectorSearch/Data/Migrations/00000000000000_Initial.cs b/SqlDatabaseVectorSearch/Data/Migrations/00000000000000_Initial.cs index 590cad4..4db809e 100644 --- a/SqlDatabaseVectorSearch/Data/Migrations/00000000000000_Initial.cs +++ b/SqlDatabaseVectorSearch/Data/Migrations/00000000000000_Initial.cs @@ -1,4 +1,5 @@ using System; +using Microsoft.Data.SqlTypes; using Microsoft.EntityFrameworkCore.Migrations; #nullable disable @@ -34,7 +35,7 @@ namespace SqlDatabaseVectorSearch.Migrations PageNumber = table.Column(type: "int", nullable: true), IndexOnPage = table.Column(type: "int", nullable: false), Content = table.Column(type: "nvarchar(max)", nullable: false), - Embedding = table.Column(type: "vector(1536)", nullable: false) + Embedding = table.Column>(type: "vector(1536)", nullable: false) }, constraints: table => { diff --git a/SqlDatabaseVectorSearch/Data/Migrations/ApplicationDbContextModelSnapshot.cs b/SqlDatabaseVectorSearch/Data/Migrations/ApplicationDbContextModelSnapshot.cs index aeb0666..0673aca 100644 --- a/SqlDatabaseVectorSearch/Data/Migrations/ApplicationDbContextModelSnapshot.cs +++ b/SqlDatabaseVectorSearch/Data/Migrations/ApplicationDbContextModelSnapshot.cs @@ -1,5 +1,6 @@ // using System; +using Microsoft.Data.SqlTypes; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Metadata; @@ -17,7 +18,7 @@ namespace SqlDatabaseVectorSearch.Migrations { #pragma warning disable 612, 618 modelBuilder - .HasAnnotation("ProductVersion", "9.0.5") + .HasAnnotation("ProductVersion", "10.0.0-rc.1.25451.107") .HasAnnotation("Relational:MaxIdentifierLength", 128); SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); @@ -54,8 +55,7 @@ namespace SqlDatabaseVectorSearch.Migrations b.Property("DocumentId") .HasColumnType("uniqueidentifier"); - b.PrimitiveCollection("Embedding") - .IsRequired() + b.Property>("Embedding") .HasColumnType("vector(1536)"); b.Property("Index") diff --git a/SqlDatabaseVectorSearch/Program.cs b/SqlDatabaseVectorSearch/Program.cs index dfda707..a36ec70 100644 --- a/SqlDatabaseVectorSearch/Program.cs +++ b/SqlDatabaseVectorSearch/Program.cs @@ -11,7 +11,6 @@ using SqlDatabaseVectorSearch.Services; using SqlDatabaseVectorSearch.Settings; using SqlDatabaseVectorSearch.TextChunkers; using TinyHelpers.AspNetCore.Extensions; -using TinyHelpers.AspNetCore.OpenApi; var builder = WebApplication.CreateBuilder(args); builder.Configuration.AddJsonFile("appsettings.local.json", optional: true, reloadOnChange: true); @@ -32,10 +31,7 @@ builder.Services.ConfigureHttpJsonOptions(options => builder.Services.AddSingleton(TimeProvider.System); -builder.Services.AddAzureSql(builder.Configuration.GetConnectionString("SqlConnection"), options => -{ - options.UseVectorSearch(); -}, options => +builder.Services.AddSqlServer(builder.Configuration.GetConnectionString("SqlConnection"), optionsAction: options => { options.UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking); }); @@ -79,8 +75,8 @@ builder.Services.AddScoped(); builder.Services.AddOpenApi(options => { - options.RemoveServerList(); - options.AddDefaultProblemDetailsResponse(); + //options.RemoveServerList(); + //options.AddDefaultProblemDetailsResponse(); }); ValidatorOptions.Global.LanguageManager.Enabled = false; diff --git a/SqlDatabaseVectorSearch/Services/DocumentService.cs b/SqlDatabaseVectorSearch/Services/DocumentService.cs index a8aca12..c376ceb 100644 --- a/SqlDatabaseVectorSearch/Services/DocumentService.cs +++ b/SqlDatabaseVectorSearch/Services/DocumentService.cs @@ -28,7 +28,7 @@ public class DocumentService(ApplicationDbContext dbContext) public async Task GetChunkEmbeddingAsync(Guid documentId, Guid documentChunkId, CancellationToken cancellationToken = default) { var documentChunk = await dbContext.DocumentChunks.Where(c => c.Id == documentChunkId && c.DocumentId == documentId) - .Select(c => new DocumentChunk(c.Id, c.Index, c.Content, c.PageNumber, c.IndexOnPage, c.Embedding)) + .Select(c => new DocumentChunk(c.Id, c.Index, c.Content, c.PageNumber, c.IndexOnPage, c.Embedding.Memory.ToArray())) .FirstOrDefaultAsync(cancellationToken); return documentChunk; diff --git a/SqlDatabaseVectorSearch/Services/VectorSearchService.cs b/SqlDatabaseVectorSearch/Services/VectorSearchService.cs index 18bfa3e..edaeea3 100644 --- a/SqlDatabaseVectorSearch/Services/VectorSearchService.cs +++ b/SqlDatabaseVectorSearch/Services/VectorSearchService.cs @@ -2,6 +2,7 @@ using System.Runtime.CompilerServices; using System.Text; using System.Text.RegularExpressions; +using Microsoft.Data.SqlTypes; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.AI; using Microsoft.Extensions.Options; @@ -66,7 +67,7 @@ public partial class VectorSearchService(IServiceProvider serviceProvider, Appli PageNumber = chunk.PageNumber, IndexOnPage = chunk.IndexOnPage, Content = chunk.Content, - Embedding = embedding.Vector.ToArray() + Embedding = new SqlVector(embedding.Vector) }; dbContext.DocumentChunks.Add(documentChunk); @@ -149,9 +150,10 @@ public partial class VectorSearchService(IServiceProvider serviceProvider, Appli // Perform Vector Search on SQL Database. var questionEmbedding = await embeddingGenerator.GenerateVectorAsync(reformulatedQuestion.Text!, cancellationToken: cancellationToken); + var embeddingVector = new SqlVector(questionEmbedding); var chunks = await dbContext.DocumentChunks.Include(c => c.Document) - .OrderBy(c => EF.Functions.VectorDistance("cosine", c.Embedding, questionEmbedding.ToArray())) + .OrderBy(c => EF.Functions.VectorDistance("cosine", c.Embedding, embeddingVector)) .Take(appSettings.MaxRelevantChunks) .ToListAsync(cancellationToken); diff --git a/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj b/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj index ecd9ad7..984f65a 100644 --- a/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj +++ b/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj @@ -10,7 +10,6 @@ - From d4753ca665eaee5826e952bc6097163cad560476 Mon Sep 17 00:00:00 2001 From: Marco Minerva Date: Thu, 11 Sep 2025 09:24:52 +0200 Subject: [PATCH 03/11] Update Microsoft.SemanticKernel to version 1.65.0 Upgraded the `Microsoft.SemanticKernel` package reference in `SqlDatabaseVectorSearch.csproj` from version 1.64.0 to 1.65.0. This update may include bug fixes, new features, or performance improvements introduced in the newer version. --- SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj b/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj index 984f65a..10a78ae 100644 --- a/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj +++ b/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj @@ -23,7 +23,7 @@ - + From 8788a013e6f0ba2f393b2a11333e876efc4e2014 Mon Sep 17 00:00:00 2001 From: Marco Minerva Date: Mon, 6 Oct 2025 11:54:06 +0200 Subject: [PATCH 04/11] Update README with ModelId and VECTOR size guidelines Enhanced documentation to clarify the configuration of `ModelId` values for `ChatCompletion` and `Embedding` in `appsettings.json`, ensuring compatibility with `Microsoft.ML.Tokenizers`. Added guidance on setting `Dimensions` for embedding models and provided instructions for updating `ApplicationDbContext` and migrations when modifying VECTOR size. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1679e5d..1ffeb63 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ Embeddings and chat completion are powered by [Semantic Kernel](https://github.c 2. Configure the database and OpenAI settings - Edit `SqlDatabaseVectorSearch/appsettings.json` and set your Azure SQL connection string and OpenAI settings. + - **Important**: The `ModelId` values for both `ChatCompletion` and `Embedding` are used for token counting via `Microsoft.ML.Tokenizers`. These values must be valid model identifiers supported by the tokenizer library (e.g., `gpt-4o`, `gpt-4`, `gpt-3.5-turbo`, `text-embedding-3-small`, `text-embedding-3-large`, `text-embedding-ada-002`). The `ModelId` may differ from the actual deployment name you're using in Azure OpenAI. For example, for gpt-4.1 models set the `ModelId` to `gpt-4o` for proper token counting. - If using embedding models with shortening (e.g., `text-embedding-3-small` or `text-embedding-3-large`), set the `Dimensions` property accordingly. For `text-embedding-3-large`, you must specify a value <= 1998. - If you change the VECTOR size, update both the [ApplicationDbContext](SqlDatabaseVectorSearch/Data/ApplicationDbContext.cs) and the [Initial Migration](SqlDatabaseVectorSearch/Data/Migrations/00000000000000_Initial.cs). From edae0c35b98c856aaee9c42ae6ac8f94c0115d75 Mon Sep 17 00:00:00 2001 From: Marco Minerva Date: Tue, 7 Oct 2025 10:13:36 +0200 Subject: [PATCH 05/11] Update ModelId guidance for gpt-4.1 and gpt-5 Updated README.md to specify that for gpt-4.1 and gpt-5 models, the ModelId should be set to gpt-4o for proper token counting. Aligned appsettings.json comments with this guidance to ensure consistency and clarity for users configuring Azure OpenAI settings. --- README.md | 2 +- SqlDatabaseVectorSearch/appsettings.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1ffeb63..84e93d5 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Embeddings and chat completion are powered by [Semantic Kernel](https://github.c 2. Configure the database and OpenAI settings - Edit `SqlDatabaseVectorSearch/appsettings.json` and set your Azure SQL connection string and OpenAI settings. - - **Important**: The `ModelId` values for both `ChatCompletion` and `Embedding` are used for token counting via `Microsoft.ML.Tokenizers`. These values must be valid model identifiers supported by the tokenizer library (e.g., `gpt-4o`, `gpt-4`, `gpt-3.5-turbo`, `text-embedding-3-small`, `text-embedding-3-large`, `text-embedding-ada-002`). The `ModelId` may differ from the actual deployment name you're using in Azure OpenAI. For example, for gpt-4.1 models set the `ModelId` to `gpt-4o` for proper token counting. + - **Important**: The `ModelId` values for both `ChatCompletion` and `Embedding` are used for token counting via `Microsoft.ML.Tokenizers`. These values must be valid model identifiers supported by the tokenizer library (e.g., `gpt-4o`, `gpt-4`, `gpt-3.5-turbo`, `text-embedding-3-small`, `text-embedding-3-large`, `text-embedding-ada-002`). The `ModelId` may differ from the actual deployment name you're using in Azure OpenAI. For example, for gpt-4.1 and gpt-5 models set the `ModelId` to `gpt-4o` for proper token counting. - If using embedding models with shortening (e.g., `text-embedding-3-small` or `text-embedding-3-large`), set the `Dimensions` property accordingly. For `text-embedding-3-large`, you must specify a value <= 1998. - If you change the VECTOR size, update both the [ApplicationDbContext](SqlDatabaseVectorSearch/Data/ApplicationDbContext.cs) and the [Initial Migration](SqlDatabaseVectorSearch/Data/Migrations/00000000000000_Initial.cs). diff --git a/SqlDatabaseVectorSearch/appsettings.json b/SqlDatabaseVectorSearch/appsettings.json index fbe05b7..3c23b98 100644 --- a/SqlDatabaseVectorSearch/appsettings.json +++ b/SqlDatabaseVectorSearch/appsettings.json @@ -6,7 +6,7 @@ "ChatCompletion": { "Endpoint": "", "Deployment": "", - "ModelId": "", // gpt-4o, gpt-4, gpt-3.5, etc. Note that for gpt-4.1 models, the ModelId must be set to gpt-4o. + "ModelId": "", // gpt-4o, gpt-4, gpt-3.5, etc. Note that for gpt-4.1 and gpt-5 models, the ModelId must be set to gpt-4o. "ApiKey": "" }, "Embedding": { From f0873bc9bf16e1a5fae4e46c6f120515920951bc Mon Sep 17 00:00:00 2001 From: Marco Minerva Date: Wed, 15 Oct 2025 11:33:14 +0200 Subject: [PATCH 06/11] Update package dependencies to latest versions Updated several package dependencies in the SqlDatabaseVectorSearch.csproj file to their latest versions: - Microsoft.AspNetCore.OpenApi, Microsoft.EntityFrameworkCore.SqlServer, and Microsoft.EntityFrameworkCore.Tools to 10.0.0-rc.2.25502.107 - Microsoft.Extensions.Caching.Hybrid and Microsoft.Extensions.Http.Resilience to 9.10.0 - Microsoft.SemanticKernel to 1.66.0 - TinyHelpers.AspNetCore to 4.1.8 These updates may include bug fixes, performance improvements, or new features. --- .../SqlDatabaseVectorSearch.csproj | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj b/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj index 187b4f4..87e0de0 100644 --- a/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj +++ b/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj @@ -12,24 +12,24 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - + + - + - + From 6361bfd8d1162a90890257a958e97be0aef7fbf4 Mon Sep 17 00:00:00 2001 From: Marco Minerva Date: Thu, 16 Oct 2025 12:15:00 +0200 Subject: [PATCH 07/11] Add ReconnectModal component and optimize resource loading Introduced a new `` component in `App.razor` to handle reconnection scenarios with a user interface. Added a `` to optimize resource loading. Updated `_Imports.razor` with a new namespace for layout components. Enhanced error handling in `Program.cs` by modifying `UseExceptionHandler` to create a new scope for errors. Added `ReconnectModal.razor`, `ReconnectModal.razor.css`, and `ReconnectModal.razor.js` to implement the modal's HTML, CSS, and JavaScript logic. --- SqlDatabaseVectorSearch/Components/App.razor | 4 +- .../Components/Layout/ReconnectModal.razor | 31 ++++ .../Layout/ReconnectModal.razor.css | 157 ++++++++++++++++++ .../Components/Layout/ReconnectModal.razor.js | 63 +++++++ .../Components/_Imports.razor | 1 + SqlDatabaseVectorSearch/Program.cs | 2 +- 6 files changed, 256 insertions(+), 2 deletions(-) create mode 100644 SqlDatabaseVectorSearch/Components/Layout/ReconnectModal.razor create mode 100644 SqlDatabaseVectorSearch/Components/Layout/ReconnectModal.razor.css create mode 100644 SqlDatabaseVectorSearch/Components/Layout/ReconnectModal.razor.js diff --git a/SqlDatabaseVectorSearch/Components/App.razor b/SqlDatabaseVectorSearch/Components/App.razor index 6b1533a..389a89c 100644 --- a/SqlDatabaseVectorSearch/Components/App.razor +++ b/SqlDatabaseVectorSearch/Components/App.razor @@ -5,7 +5,8 @@ - + + @@ -18,6 +19,7 @@ + diff --git a/SqlDatabaseVectorSearch/Components/Layout/ReconnectModal.razor b/SqlDatabaseVectorSearch/Components/Layout/ReconnectModal.razor new file mode 100644 index 0000000..a55bcc1 --- /dev/null +++ b/SqlDatabaseVectorSearch/Components/Layout/ReconnectModal.razor @@ -0,0 +1,31 @@ + + + +
+ +

+ Rejoining the server... +

+

+ Rejoin failed... Trying again in seconds. +

+

+ Failed to rejoin.
Please retry or reload the page. +

+ +

+ The session has been paused by the server. +

+ +

+ Failed to resume the session.
Please reload the page. +

+
+ diff --git a/SqlDatabaseVectorSearch/Components/Layout/ReconnectModal.razor.css b/SqlDatabaseVectorSearch/Components/Layout/ReconnectModal.razor.css new file mode 100644 index 0000000..3ad3773 --- /dev/null +++ b/SqlDatabaseVectorSearch/Components/Layout/ReconnectModal.razor.css @@ -0,0 +1,157 @@ +.components-reconnect-first-attempt-visible, +.components-reconnect-repeated-attempt-visible, +.components-reconnect-failed-visible, +.components-pause-visible, +.components-resume-failed-visible, +.components-rejoining-animation { + display: none; +} + +#components-reconnect-modal.components-reconnect-show .components-reconnect-first-attempt-visible, +#components-reconnect-modal.components-reconnect-show .components-rejoining-animation, +#components-reconnect-modal.components-reconnect-paused .components-pause-visible, +#components-reconnect-modal.components-reconnect-resume-failed .components-resume-failed-visible, +#components-reconnect-modal.components-reconnect-retrying, +#components-reconnect-modal.components-reconnect-retrying .components-reconnect-repeated-attempt-visible, +#components-reconnect-modal.components-reconnect-retrying .components-rejoining-animation, +#components-reconnect-modal.components-reconnect-failed, +#components-reconnect-modal.components-reconnect-failed .components-reconnect-failed-visible { + display: block; +} + + +#components-reconnect-modal { + background-color: white; + width: 20rem; + margin: 20vh auto; + padding: 2rem; + border: 0; + border-radius: 0.5rem; + box-shadow: 0 3px 6px 2px rgba(0, 0, 0, 0.3); + opacity: 0; + transition: display 0.5s allow-discrete, overlay 0.5s allow-discrete; + animation: components-reconnect-modal-fadeOutOpacity 0.5s both; + &[open] + +{ + animation: components-reconnect-modal-slideUp 1.5s cubic-bezier(.05, .89, .25, 1.02) 0.3s, components-reconnect-modal-fadeInOpacity 0.5s ease-in-out 0.3s; + animation-fill-mode: both; +} + +} + +#components-reconnect-modal::backdrop { + background-color: rgba(0, 0, 0, 0.4); + animation: components-reconnect-modal-fadeInOpacity 0.5s ease-in-out; + opacity: 1; +} + +@keyframes components-reconnect-modal-slideUp { + 0% { + transform: translateY(30px) scale(0.95); + } + + 100% { + transform: translateY(0); + } +} + +@keyframes components-reconnect-modal-fadeInOpacity { + 0% { + opacity: 0; + } + + 100% { + opacity: 1; + } +} + +@keyframes components-reconnect-modal-fadeOutOpacity { + 0% { + opacity: 1; + } + + 100% { + opacity: 0; + } +} + +.components-reconnect-container { + display: flex; + flex-direction: column; + align-items: center; + gap: 1rem; +} + +#components-reconnect-modal p { + margin: 0; + text-align: center; +} + +#components-reconnect-modal button { + border: 0; + background-color: #6b9ed2; + color: white; + padding: 4px 24px; + border-radius: 4px; +} + + #components-reconnect-modal button:hover { + background-color: #3b6ea2; + } + + #components-reconnect-modal button:active { + background-color: #6b9ed2; + } + +.components-rejoining-animation { + position: relative; + width: 80px; + height: 80px; +} + + .components-rejoining-animation div { + position: absolute; + border: 3px solid #0087ff; + opacity: 1; + border-radius: 50%; + animation: components-rejoining-animation 1.5s cubic-bezier(0, 0.2, 0.8, 1) infinite; + } + + .components-rejoining-animation div:nth-child(2) { + animation-delay: -0.5s; + } + +@keyframes components-rejoining-animation { + 0% { + top: 40px; + left: 40px; + width: 0; + height: 0; + opacity: 0; + } + + 4.9% { + top: 40px; + left: 40px; + width: 0; + height: 0; + opacity: 0; + } + + 5% { + top: 40px; + left: 40px; + width: 0; + height: 0; + opacity: 1; + } + + 100% { + top: 0px; + left: 0px; + width: 80px; + height: 80px; + opacity: 0; + } +} diff --git a/SqlDatabaseVectorSearch/Components/Layout/ReconnectModal.razor.js b/SqlDatabaseVectorSearch/Components/Layout/ReconnectModal.razor.js new file mode 100644 index 0000000..e52a190 --- /dev/null +++ b/SqlDatabaseVectorSearch/Components/Layout/ReconnectModal.razor.js @@ -0,0 +1,63 @@ +// Set up event handlers +const reconnectModal = document.getElementById("components-reconnect-modal"); +reconnectModal.addEventListener("components-reconnect-state-changed", handleReconnectStateChanged); + +const retryButton = document.getElementById("components-reconnect-button"); +retryButton.addEventListener("click", retry); + +const resumeButton = document.getElementById("components-resume-button"); +resumeButton.addEventListener("click", resume); + +function handleReconnectStateChanged(event) { + if (event.detail.state === "show") { + reconnectModal.showModal(); + } else if (event.detail.state === "hide") { + reconnectModal.close(); + } else if (event.detail.state === "failed") { + document.addEventListener("visibilitychange", retryWhenDocumentBecomesVisible); + } else if (event.detail.state === "rejected") { + location.reload(); + } +} + +async function retry() { + document.removeEventListener("visibilitychange", retryWhenDocumentBecomesVisible); + + try { + // Reconnect will asynchronously return: + // - true to mean success + // - false to mean we reached the server, but it rejected the connection (e.g., unknown circuit ID) + // - exception to mean we didn't reach the server (this can be sync or async) + const successful = await Blazor.reconnect(); + if (!successful) { + // We have been able to reach the server, but the circuit is no longer available. + // We'll reload the page so the user can continue using the app as quickly as possible. + const resumeSuccessful = await Blazor.resumeCircuit(); + if (!resumeSuccessful) { + location.reload(); + } else { + reconnectModal.close(); + } + } + } catch (err) { + // We got an exception, server is currently unavailable + document.addEventListener("visibilitychange", retryWhenDocumentBecomesVisible); + } +} + +async function resume() { + try { + const successful = await Blazor.resumeCircuit(); + if (!successful) { + location.reload(); + } + } catch { + location.reload(); + } +} + +async function retryWhenDocumentBecomesVisible() { + if (document.visibilityState === "visible") { + await retry(); + } +} diff --git a/SqlDatabaseVectorSearch/Components/_Imports.razor b/SqlDatabaseVectorSearch/Components/_Imports.razor index 06b940f..ef23f97 100644 --- a/SqlDatabaseVectorSearch/Components/_Imports.razor +++ b/SqlDatabaseVectorSearch/Components/_Imports.razor @@ -9,6 +9,7 @@ @using Microsoft.JSInterop @using SqlDatabaseVectorSearch @using SqlDatabaseVectorSearch.Components +@using SqlDatabaseVectorSearch.Components.Layout @using SqlDatabaseVectorSearch.Extensions @using SqlDatabaseVectorSearch.Models @using SqlDatabaseVectorSearch.Services diff --git a/SqlDatabaseVectorSearch/Program.cs b/SqlDatabaseVectorSearch/Program.cs index a36ec70..6762bf5 100644 --- a/SqlDatabaseVectorSearch/Program.cs +++ b/SqlDatabaseVectorSearch/Program.cs @@ -95,7 +95,7 @@ app.UseWhen(context => context.IsWebRequest(), builder => { if (!app.Environment.IsDevelopment()) { - builder.UseExceptionHandler("/error"); + builder.UseExceptionHandler("/error", createScopeForErrors: true); // The default HSTS value is 30 days. builder.UseHsts(); From 0cc969c16462171592bb6855889396f824fa54d4 Mon Sep 17 00:00:00 2001 From: Marco Minerva Date: Thu, 16 Oct 2025 12:32:48 +0200 Subject: [PATCH 08/11] Switch to CountEmbeddingTokens in chunkers Updated the token counting method from CountChatCompletionTokens to CountEmbeddingTokens in VectorSearchService, DefaultTextChunker, and MarkdownTextChunker to align with embedding token counting. Added a new logging configuration for Microsoft.AspNetCore.Watch.BrowserRefresh in appsettings.Development.json to manage log verbosity during development. --- SqlDatabaseVectorSearch/Services/VectorSearchService.cs | 2 +- SqlDatabaseVectorSearch/TextChunkers/DefaultTextChunker.cs | 4 ++-- SqlDatabaseVectorSearch/TextChunkers/MarkdownTextChunker.cs | 4 ++-- SqlDatabaseVectorSearch/appsettings.Development.json | 1 + 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/SqlDatabaseVectorSearch/Services/VectorSearchService.cs b/SqlDatabaseVectorSearch/Services/VectorSearchService.cs index edaeea3..cc49944 100644 --- a/SqlDatabaseVectorSearch/Services/VectorSearchService.cs +++ b/SqlDatabaseVectorSearch/Services/VectorSearchService.cs @@ -58,7 +58,7 @@ public partial class VectorSearchService(IServiceProvider serviceProvider, Appli foreach (var (index, embedding) in embeddings.Index()) { var chunk = chunks.ElementAt(index); - logger.LogDebug("Storing a chunk of {TokenCount} tokens.", tokenizerService.CountChatCompletionTokens(chunk.Content)); + logger.LogDebug("Storing a chunk of {TokenCount} tokens.", tokenizerService.CountEmbeddingTokens(chunk.Content)); var documentChunk = new Entities.DocumentChunk { diff --git a/SqlDatabaseVectorSearch/TextChunkers/DefaultTextChunker.cs b/SqlDatabaseVectorSearch/TextChunkers/DefaultTextChunker.cs index 46ccd49..a5a6854 100644 --- a/SqlDatabaseVectorSearch/TextChunkers/DefaultTextChunker.cs +++ b/SqlDatabaseVectorSearch/TextChunkers/DefaultTextChunker.cs @@ -11,8 +11,8 @@ public class DefaultTextChunker(TokenizerService tokenizerService, IOptions Split(string text) { - var lines = TextChunker.SplitPlainTextLines(text, appSettings.MaxTokensPerLine, tokenizerService.CountChatCompletionTokens); - var paragraphs = TextChunker.SplitPlainTextParagraphs(lines, appSettings.MaxTokensPerParagraph, appSettings.OverlapTokens, tokenCounter: tokenizerService.CountChatCompletionTokens); + var lines = TextChunker.SplitPlainTextLines(text, appSettings.MaxTokensPerLine, tokenizerService.CountEmbeddingTokens); + var paragraphs = TextChunker.SplitPlainTextParagraphs(lines, appSettings.MaxTokensPerParagraph, appSettings.OverlapTokens, tokenCounter: tokenizerService.CountEmbeddingTokens); return paragraphs; } diff --git a/SqlDatabaseVectorSearch/TextChunkers/MarkdownTextChunker.cs b/SqlDatabaseVectorSearch/TextChunkers/MarkdownTextChunker.cs index fd3a8f6..cba6679 100644 --- a/SqlDatabaseVectorSearch/TextChunkers/MarkdownTextChunker.cs +++ b/SqlDatabaseVectorSearch/TextChunkers/MarkdownTextChunker.cs @@ -11,8 +11,8 @@ public class MarkdownTextChunker(TokenizerService tokenizerService, IOptions Split(string text) { - var lines = TextChunker.SplitMarkDownLines(text, appSettings.MaxTokensPerLine, tokenizerService.CountChatCompletionTokens); - var paragraphs = TextChunker.SplitMarkdownParagraphs(lines, appSettings.MaxTokensPerParagraph, appSettings.OverlapTokens, tokenCounter: tokenizerService.CountChatCompletionTokens); + var lines = TextChunker.SplitMarkDownLines(text, appSettings.MaxTokensPerLine, tokenizerService.CountEmbeddingTokens); + var paragraphs = TextChunker.SplitMarkdownParagraphs(lines, appSettings.MaxTokensPerParagraph, appSettings.OverlapTokens, tokenCounter: tokenizerService.CountEmbeddingTokens); return paragraphs; } diff --git a/SqlDatabaseVectorSearch/appsettings.Development.json b/SqlDatabaseVectorSearch/appsettings.Development.json index 19c6237..0fc47fd 100644 --- a/SqlDatabaseVectorSearch/appsettings.Development.json +++ b/SqlDatabaseVectorSearch/appsettings.Development.json @@ -3,6 +3,7 @@ "LogLevel": { "Default": "Information", "Microsoft.AspNetCore": "Warning", + "Microsoft.AspNetCore.Watch.BrowserRefresh": "Warning", "SqlDatabaseVectorSearch": "Debug" } } From b4e47ef552498846fc68a427fad7e691a71462b9 Mon Sep 17 00:00:00 2001 From: Marco Minerva Date: Fri, 31 Oct 2025 17:37:56 +0100 Subject: [PATCH 09/11] Update Microsoft.ML.Tokenizers to version 1.0.3 Updated the `` for `Microsoft.ML.Tokenizers` and its related data packages (`Data.Cl100kBase` and `Data.O200kBase`) from version `1.0.2` to `1.0.3`. These updates may include bug fixes, performance improvements, or new features introduced in the latest version. --- SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj b/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj index 87e0de0..bb0c9e9 100644 --- a/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj +++ b/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj @@ -20,9 +20,9 @@ - - - + + + From bef955831db0557bc8079cc99f8fa9b57f5f14e0 Mon Sep 17 00:00:00 2001 From: Marco Minerva Date: Tue, 4 Nov 2025 09:57:03 +0100 Subject: [PATCH 10/11] Update package versions for FluentValidation and SemanticKernel Updated the `FluentValidation.DependencyInjectionExtensions` package from version 12.0.0 to 12.1.0 to incorporate potential bug fixes and improvements. Updated the `Microsoft.SemanticKernel` package from version 1.66.0 to 1.67.0, which may include enhancements, bug fixes, or new functionality. --- SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj b/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj index bb0c9e9..7e90b5c 100644 --- a/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj +++ b/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj @@ -11,7 +11,7 @@ - + @@ -23,7 +23,7 @@ - + From d6458892f7125a0bf3ef7012db0ce644d739d0ac Mon Sep 17 00:00:00 2001 From: Marco Minerva Date: Thu, 13 Nov 2025 17:57:27 +0100 Subject: [PATCH 11/11] Update package dependencies to stable and latest versions Updated multiple NuGet package dependencies to their latest stable versions, including: - `Microsoft.AspNetCore.OpenApi` to `10.0.0` - `Microsoft.EntityFrameworkCore.SqlServer` to `10.0.0` - `Microsoft.Extensions.Caching.Hybrid` to `10.0.0` - `Microsoft.ML.Tokenizers` to `2.0.0` - `Microsoft.SemanticKernel` to `1.67.1` - `Swashbuckle.AspNetCore.SwaggerUI` to `10.0.1` - Other minor updates to `PdfPig`, `TinyHelpers.AspNetCore`, and more. These updates ensure compatibility with the latest features and improvements. --- .../SqlDatabaseVectorSearch.csproj | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj b/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj index 7e90b5c..bcd3fff 100644 --- a/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj +++ b/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj @@ -12,24 +12,24 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - + + + + + + - + - - - + + +