mirror of
https://github.com/marcominerva/SqlDatabaseVectorSearch.git
synced 2026-06-20 12:23:10 +00:00
Refactor embeddings and add reconnect modal
Enhanced support for SQL Server's `SqlVector<float>` for embeddings, replacing `float[]` to improve vector search compatibility. Added a reconnect modal for better handling of server disconnections, including UI, styles, and JavaScript logic. Upgraded to .NET 10.0 and EF Core 10.0, updated dependencies, and improved token counting logic for embeddings. Updated documentation and configuration files to reflect these changes. Migrated database schema to support `SqlVector<float>`.
This commit is contained in:
Marco Minerva
@@ -62,6 +62,7 @@ Embeddings and chat completion are powered by [Semantic Kernel](https://github.c
|
||||
|
||||
2. Configure the database and OpenAI settings
|
||||
- Edit `SqlDatabaseVectorSearch/appsettings.json` and set your Azure SQL connection string and OpenAI settings.
|
||||
- **Important**: The `ModelId` values for both `ChatCompletion` and `Embedding` are used for token counting via `Microsoft.ML.Tokenizers`. These values must be valid model identifiers supported by the tokenizer library (e.g., `gpt-4o`, `gpt-4`, `gpt-3.5-turbo`, `text-embedding-3-small`, `text-embedding-3-large`, `text-embedding-ada-002`). The `ModelId` may differ from the actual deployment name you're using in Azure OpenAI. For example, for gpt-4.1 and gpt-5 models set the `ModelId` to `gpt-4o` for proper token counting.
|
||||
- If using embedding models with shortening (e.g., `text-embedding-3-small` or `text-embedding-3-large`), set the `Dimensions` property accordingly. For `text-embedding-3-large`, you must specify a value <= 1998.
|
||||
- If you change the VECTOR size, update both the [ApplicationDbContext](SqlDatabaseVectorSearch/Data/ApplicationDbContext.cs) and the [Initial Migration](SqlDatabaseVectorSearch/Data/Migrations/00000000000000_Initial.cs).
|
||||
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<base href="/" />
|
||||
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-QWTKZyjpPEjISv5WaRU9OFeRpok6YctnYmDr5pNlyT2bRjXh0JMhjY6hW+ALEwIH" crossorigin="anonymous">
|
||||
<ResourcePreloader />
|
||||
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-QWTKZyjpPEjISv5WaRU9OFeRpok6YctnYmDr5pNlyT2bRjXh0JMhjY6hW+ALEwIH" crossorigin="anonymous">
|
||||
<link href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.3/font/bootstrap-icons.css" rel="stylesheet" />
|
||||
<link href="_content/Blazor.Bootstrap/blazor.bootstrap.css" rel="stylesheet" />
|
||||
<script src="https://kit.fontawesome.com/f7a7b34f96.js" crossorigin="anonymous"></script>
|
||||
@@ -18,6 +19,7 @@
|
||||
|
||||
<body>
|
||||
<Routes @rendermode="InteractiveServer" />
|
||||
<ReconnectModal />
|
||||
<script src="_framework/blazor.web.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js" integrity="sha384-YvpcrYf0tY3lHB60NNkmXc5s9fDVZLESaAA55NDzOxhy9GkcIdslK1eN7N6jIeHz" crossorigin="anonymous"></script>
|
||||
<!-- Add chart.js reference if chart components are used in your application. -->
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
<script type="module" src="@Assets["Components/Layout/ReconnectModal.razor.js"]"></script>
|
||||
|
||||
<dialog id="components-reconnect-modal" data-nosnippet>
|
||||
<div class="components-reconnect-container">
|
||||
<div class="components-rejoining-animation" aria-hidden="true">
|
||||
<div></div>
|
||||
<div></div>
|
||||
</div>
|
||||
<p class="components-reconnect-first-attempt-visible">
|
||||
Rejoining the server...
|
||||
</p>
|
||||
<p class="components-reconnect-repeated-attempt-visible">
|
||||
Rejoin failed... Trying again in <span id="components-seconds-to-next-attempt"></span> seconds.
|
||||
</p>
|
||||
<p class="components-reconnect-failed-visible">
|
||||
Failed to rejoin.<br />Please retry or reload the page.
|
||||
</p>
|
||||
<button id="components-reconnect-button" class="components-reconnect-failed-visible">
|
||||
Retry
|
||||
</button>
|
||||
<p class="components-pause-visible">
|
||||
The session has been paused by the server.
|
||||
</p>
|
||||
<button id="components-resume-button" class="components-pause-visible">
|
||||
Resume
|
||||
</button>
|
||||
<p class="components-resume-failed-visible">
|
||||
Failed to resume the session.<br />Please reload the page.
|
||||
</p>
|
||||
</div>
|
||||
</dialog>
|
||||
@@ -0,0 +1,157 @@
|
||||
.components-reconnect-first-attempt-visible,
|
||||
.components-reconnect-repeated-attempt-visible,
|
||||
.components-reconnect-failed-visible,
|
||||
.components-pause-visible,
|
||||
.components-resume-failed-visible,
|
||||
.components-rejoining-animation {
|
||||
display: none;
|
||||
}
|
||||
|
||||
#components-reconnect-modal.components-reconnect-show .components-reconnect-first-attempt-visible,
|
||||
#components-reconnect-modal.components-reconnect-show .components-rejoining-animation,
|
||||
#components-reconnect-modal.components-reconnect-paused .components-pause-visible,
|
||||
#components-reconnect-modal.components-reconnect-resume-failed .components-resume-failed-visible,
|
||||
#components-reconnect-modal.components-reconnect-retrying,
|
||||
#components-reconnect-modal.components-reconnect-retrying .components-reconnect-repeated-attempt-visible,
|
||||
#components-reconnect-modal.components-reconnect-retrying .components-rejoining-animation,
|
||||
#components-reconnect-modal.components-reconnect-failed,
|
||||
#components-reconnect-modal.components-reconnect-failed .components-reconnect-failed-visible {
|
||||
display: block;
|
||||
}
|
||||
|
||||
|
||||
#components-reconnect-modal {
|
||||
background-color: white;
|
||||
width: 20rem;
|
||||
margin: 20vh auto;
|
||||
padding: 2rem;
|
||||
border: 0;
|
||||
border-radius: 0.5rem;
|
||||
box-shadow: 0 3px 6px 2px rgba(0, 0, 0, 0.3);
|
||||
opacity: 0;
|
||||
transition: display 0.5s allow-discrete, overlay 0.5s allow-discrete;
|
||||
animation: components-reconnect-modal-fadeOutOpacity 0.5s both;
|
||||
&[open]
|
||||
|
||||
{
|
||||
animation: components-reconnect-modal-slideUp 1.5s cubic-bezier(.05, .89, .25, 1.02) 0.3s, components-reconnect-modal-fadeInOpacity 0.5s ease-in-out 0.3s;
|
||||
animation-fill-mode: both;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#components-reconnect-modal::backdrop {
|
||||
background-color: rgba(0, 0, 0, 0.4);
|
||||
animation: components-reconnect-modal-fadeInOpacity 0.5s ease-in-out;
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
@keyframes components-reconnect-modal-slideUp {
|
||||
0% {
|
||||
transform: translateY(30px) scale(0.95);
|
||||
}
|
||||
|
||||
100% {
|
||||
transform: translateY(0);
|
||||
}
|
||||
}
|
||||
|
||||
@keyframes components-reconnect-modal-fadeInOpacity {
|
||||
0% {
|
||||
opacity: 0;
|
||||
}
|
||||
|
||||
100% {
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
|
||||
@keyframes components-reconnect-modal-fadeOutOpacity {
|
||||
0% {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
100% {
|
||||
opacity: 0;
|
||||
}
|
||||
}
|
||||
|
||||
.components-reconnect-container {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
#components-reconnect-modal p {
|
||||
margin: 0;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
#components-reconnect-modal button {
|
||||
border: 0;
|
||||
background-color: #6b9ed2;
|
||||
color: white;
|
||||
padding: 4px 24px;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
#components-reconnect-modal button:hover {
|
||||
background-color: #3b6ea2;
|
||||
}
|
||||
|
||||
#components-reconnect-modal button:active {
|
||||
background-color: #6b9ed2;
|
||||
}
|
||||
|
||||
.components-rejoining-animation {
|
||||
position: relative;
|
||||
width: 80px;
|
||||
height: 80px;
|
||||
}
|
||||
|
||||
.components-rejoining-animation div {
|
||||
position: absolute;
|
||||
border: 3px solid #0087ff;
|
||||
opacity: 1;
|
||||
border-radius: 50%;
|
||||
animation: components-rejoining-animation 1.5s cubic-bezier(0, 0.2, 0.8, 1) infinite;
|
||||
}
|
||||
|
||||
.components-rejoining-animation div:nth-child(2) {
|
||||
animation-delay: -0.5s;
|
||||
}
|
||||
|
||||
@keyframes components-rejoining-animation {
|
||||
0% {
|
||||
top: 40px;
|
||||
left: 40px;
|
||||
width: 0;
|
||||
height: 0;
|
||||
opacity: 0;
|
||||
}
|
||||
|
||||
4.9% {
|
||||
top: 40px;
|
||||
left: 40px;
|
||||
width: 0;
|
||||
height: 0;
|
||||
opacity: 0;
|
||||
}
|
||||
|
||||
5% {
|
||||
top: 40px;
|
||||
left: 40px;
|
||||
width: 0;
|
||||
height: 0;
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
100% {
|
||||
top: 0px;
|
||||
left: 0px;
|
||||
width: 80px;
|
||||
height: 80px;
|
||||
opacity: 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
// Set up event handlers
|
||||
const reconnectModal = document.getElementById("components-reconnect-modal");
|
||||
reconnectModal.addEventListener("components-reconnect-state-changed", handleReconnectStateChanged);
|
||||
|
||||
const retryButton = document.getElementById("components-reconnect-button");
|
||||
retryButton.addEventListener("click", retry);
|
||||
|
||||
const resumeButton = document.getElementById("components-resume-button");
|
||||
resumeButton.addEventListener("click", resume);
|
||||
|
||||
function handleReconnectStateChanged(event) {
|
||||
if (event.detail.state === "show") {
|
||||
reconnectModal.showModal();
|
||||
} else if (event.detail.state === "hide") {
|
||||
reconnectModal.close();
|
||||
} else if (event.detail.state === "failed") {
|
||||
document.addEventListener("visibilitychange", retryWhenDocumentBecomesVisible);
|
||||
} else if (event.detail.state === "rejected") {
|
||||
location.reload();
|
||||
}
|
||||
}
|
||||
|
||||
async function retry() {
|
||||
document.removeEventListener("visibilitychange", retryWhenDocumentBecomesVisible);
|
||||
|
||||
try {
|
||||
// Reconnect will asynchronously return:
|
||||
// - true to mean success
|
||||
// - false to mean we reached the server, but it rejected the connection (e.g., unknown circuit ID)
|
||||
// - exception to mean we didn't reach the server (this can be sync or async)
|
||||
const successful = await Blazor.reconnect();
|
||||
if (!successful) {
|
||||
// We have been able to reach the server, but the circuit is no longer available.
|
||||
// We'll reload the page so the user can continue using the app as quickly as possible.
|
||||
const resumeSuccessful = await Blazor.resumeCircuit();
|
||||
if (!resumeSuccessful) {
|
||||
location.reload();
|
||||
} else {
|
||||
reconnectModal.close();
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
// We got an exception, server is currently unavailable
|
||||
document.addEventListener("visibilitychange", retryWhenDocumentBecomesVisible);
|
||||
}
|
||||
}
|
||||
|
||||
async function resume() {
|
||||
try {
|
||||
const successful = await Blazor.resumeCircuit();
|
||||
if (!successful) {
|
||||
location.reload();
|
||||
}
|
||||
} catch {
|
||||
location.reload();
|
||||
}
|
||||
}
|
||||
|
||||
async function retryWhenDocumentBecomesVisible() {
|
||||
if (document.visibilityState === "visible") {
|
||||
await retry();
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@
|
||||
@using Microsoft.JSInterop
|
||||
@using SqlDatabaseVectorSearch
|
||||
@using SqlDatabaseVectorSearch.Components
|
||||
@using SqlDatabaseVectorSearch.Components.Layout
|
||||
@using SqlDatabaseVectorSearch.Extensions
|
||||
@using SqlDatabaseVectorSearch.Models
|
||||
@using SqlDatabaseVectorSearch.Services
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
namespace SqlDatabaseVectorSearch.Data.Entities;
|
||||
using Microsoft.Data.SqlTypes;
|
||||
|
||||
namespace SqlDatabaseVectorSearch.Data.Entities;
|
||||
|
||||
public class DocumentChunk
|
||||
{
|
||||
@@ -14,7 +16,7 @@ public class DocumentChunk
|
||||
|
||||
public required string Content { get; set; }
|
||||
|
||||
public required float[] Embedding { get; set; }
|
||||
public required SqlVector<float> Embedding { get; set; }
|
||||
|
||||
public virtual Document Document { get; set; } = null!;
|
||||
}
|
||||
+4
-4
@@ -1,5 +1,6 @@
|
||||
// <auto-generated />
|
||||
using System;
|
||||
using Microsoft.Data.SqlTypes;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Metadata;
|
||||
@@ -12,7 +13,7 @@ using SqlDatabaseVectorSearch.Data;
|
||||
namespace SqlDatabaseVectorSearch.Migrations
|
||||
{
|
||||
[DbContext(typeof(ApplicationDbContext))]
|
||||
[Migration("20250606091336_Initial")]
|
||||
[Migration("00000000000000_Initial")]
|
||||
partial class Initial
|
||||
{
|
||||
/// <inheritdoc />
|
||||
@@ -20,7 +21,7 @@ namespace SqlDatabaseVectorSearch.Migrations
|
||||
{
|
||||
#pragma warning disable 612, 618
|
||||
modelBuilder
|
||||
.HasAnnotation("ProductVersion", "9.0.5")
|
||||
.HasAnnotation("ProductVersion", "10.0.0-rc.1.25451.107")
|
||||
.HasAnnotation("Relational:MaxIdentifierLength", 128);
|
||||
|
||||
SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder);
|
||||
@@ -57,8 +58,7 @@ namespace SqlDatabaseVectorSearch.Migrations
|
||||
b.Property<Guid>("DocumentId")
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.PrimitiveCollection<string>("Embedding")
|
||||
.IsRequired()
|
||||
b.Property<SqlVector<float>>("Embedding")
|
||||
.HasColumnType("vector(1536)");
|
||||
|
||||
b.Property<int>("Index")
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using Microsoft.Data.SqlTypes;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
#nullable disable
|
||||
@@ -34,7 +35,7 @@ namespace SqlDatabaseVectorSearch.Migrations
|
||||
PageNumber = table.Column<int>(type: "int", nullable: true),
|
||||
IndexOnPage = table.Column<int>(type: "int", nullable: false),
|
||||
Content = table.Column<string>(type: "nvarchar(max)", nullable: false),
|
||||
Embedding = table.Column<string>(type: "vector(1536)", nullable: false)
|
||||
Embedding = table.Column<SqlVector<float>>(type: "vector(1536)", nullable: false)
|
||||
},
|
||||
constraints: table =>
|
||||
{
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// <auto-generated />
|
||||
using System;
|
||||
using Microsoft.Data.SqlTypes;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Metadata;
|
||||
@@ -17,7 +18,7 @@ namespace SqlDatabaseVectorSearch.Migrations
|
||||
{
|
||||
#pragma warning disable 612, 618
|
||||
modelBuilder
|
||||
.HasAnnotation("ProductVersion", "9.0.5")
|
||||
.HasAnnotation("ProductVersion", "10.0.0-rc.1.25451.107")
|
||||
.HasAnnotation("Relational:MaxIdentifierLength", 128);
|
||||
|
||||
SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder);
|
||||
@@ -54,8 +55,7 @@ namespace SqlDatabaseVectorSearch.Migrations
|
||||
b.Property<Guid>("DocumentId")
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.PrimitiveCollection<string>("Embedding")
|
||||
.IsRequired()
|
||||
b.Property<SqlVector<float>>("Embedding")
|
||||
.HasColumnType("vector(1536)");
|
||||
|
||||
b.Property<int>("Index")
|
||||
|
||||
@@ -11,7 +11,6 @@ using SqlDatabaseVectorSearch.Services;
|
||||
using SqlDatabaseVectorSearch.Settings;
|
||||
using SqlDatabaseVectorSearch.TextChunkers;
|
||||
using TinyHelpers.AspNetCore.Extensions;
|
||||
using TinyHelpers.AspNetCore.OpenApi;
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
builder.Configuration.AddJsonFile("appsettings.local.json", optional: true, reloadOnChange: true);
|
||||
@@ -32,10 +31,7 @@ builder.Services.ConfigureHttpJsonOptions(options =>
|
||||
|
||||
builder.Services.AddSingleton(TimeProvider.System);
|
||||
|
||||
builder.Services.AddAzureSql<ApplicationDbContext>(builder.Configuration.GetConnectionString("SqlConnection"), options =>
|
||||
{
|
||||
options.UseVectorSearch();
|
||||
}, options =>
|
||||
builder.Services.AddSqlServer<ApplicationDbContext>(builder.Configuration.GetConnectionString("SqlConnection"), optionsAction: options =>
|
||||
{
|
||||
options.UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking);
|
||||
});
|
||||
@@ -79,8 +75,8 @@ builder.Services.AddScoped<VectorSearchService>();
|
||||
|
||||
builder.Services.AddOpenApi(options =>
|
||||
{
|
||||
options.RemoveServerList();
|
||||
options.AddDefaultProblemDetailsResponse();
|
||||
//options.RemoveServerList();
|
||||
//options.AddDefaultProblemDetailsResponse();
|
||||
});
|
||||
|
||||
ValidatorOptions.Global.LanguageManager.Enabled = false;
|
||||
@@ -99,7 +95,7 @@ app.UseWhen(context => context.IsWebRequest(), builder =>
|
||||
{
|
||||
if (!app.Environment.IsDevelopment())
|
||||
{
|
||||
builder.UseExceptionHandler("/error");
|
||||
builder.UseExceptionHandler("/error", createScopeForErrors: true);
|
||||
|
||||
// The default HSTS value is 30 days.
|
||||
builder.UseHsts();
|
||||
|
||||
@@ -28,7 +28,7 @@ public class DocumentService(ApplicationDbContext dbContext)
|
||||
public async Task<DocumentChunk?> GetChunkEmbeddingAsync(Guid documentId, Guid documentChunkId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var documentChunk = await dbContext.DocumentChunks.Where(c => c.Id == documentChunkId && c.DocumentId == documentId)
|
||||
.Select(c => new DocumentChunk(c.Id, c.Index, c.Content, c.PageNumber, c.IndexOnPage, c.Embedding))
|
||||
.Select(c => new DocumentChunk(c.Id, c.Index, c.Content, c.PageNumber, c.IndexOnPage, c.Embedding.Memory.ToArray()))
|
||||
.FirstOrDefaultAsync(cancellationToken);
|
||||
|
||||
return documentChunk;
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using Microsoft.Data.SqlTypes;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.Options;
|
||||
@@ -57,7 +58,7 @@ public partial class VectorSearchService(IServiceProvider serviceProvider, Appli
|
||||
foreach (var (index, embedding) in embeddings.Index())
|
||||
{
|
||||
var chunk = chunks.ElementAt(index);
|
||||
logger.LogDebug("Storing a chunk of {TokenCount} tokens.", tokenizerService.CountChatCompletionTokens(chunk.Content));
|
||||
logger.LogDebug("Storing a chunk of {TokenCount} tokens.", tokenizerService.CountEmbeddingTokens(chunk.Content));
|
||||
|
||||
var documentChunk = new Entities.DocumentChunk
|
||||
{
|
||||
@@ -66,7 +67,7 @@ public partial class VectorSearchService(IServiceProvider serviceProvider, Appli
|
||||
PageNumber = chunk.PageNumber,
|
||||
IndexOnPage = chunk.IndexOnPage,
|
||||
Content = chunk.Content,
|
||||
Embedding = embedding.Vector.ToArray()
|
||||
Embedding = new SqlVector<float>(embedding.Vector)
|
||||
};
|
||||
|
||||
dbContext.DocumentChunks.Add(documentChunk);
|
||||
@@ -149,9 +150,10 @@ public partial class VectorSearchService(IServiceProvider serviceProvider, Appli
|
||||
|
||||
// Perform Vector Search on SQL Database.
|
||||
var questionEmbedding = await embeddingGenerator.GenerateVectorAsync(reformulatedQuestion.Text!, cancellationToken: cancellationToken);
|
||||
var embeddingVector = new SqlVector<float>(questionEmbedding);
|
||||
|
||||
var chunks = await dbContext.DocumentChunks.Include(c => c.Document)
|
||||
.OrderBy(c => EF.Functions.VectorDistance("cosine", c.Embedding, questionEmbedding.ToArray()))
|
||||
.OrderBy(c => EF.Functions.VectorDistance("cosine", c.Embedding, embeddingVector))
|
||||
.Take(appSettings.MaxRelevantChunks)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net9.0</TargetFramework>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<NoWarn>$(NoWarn);SKEXP0010;SKEXP0050</NoWarn>
|
||||
@@ -10,27 +10,26 @@
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Blazor.Bootstrap" Version="3.4.0" />
|
||||
<PackageReference Include="DocumentFormat.OpenXml" Version="3.3.0" />
|
||||
<PackageReference Include="EFCore.SqlServer.VectorSearch" Version="9.0.0" />
|
||||
<PackageReference Include="EntityFrameworkCore.Exceptions.SqlServer" Version="8.1.3" />
|
||||
<PackageReference Include="FluentValidation.DependencyInjectionExtensions" Version="12.1.0" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="9.0.10" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="9.0.10" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="9.0.10">
|
||||
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="10.0.0" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="10.0.0" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="10.0.0">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
<PackageReference Include="Microsoft.Extensions.Caching.Hybrid" Version="9.10.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="9.10.0" />
|
||||
<PackageReference Include="Microsoft.ML.Tokenizers" Version="1.0.3" />
|
||||
<PackageReference Include="Microsoft.ML.Tokenizers.Data.Cl100kBase" Version="1.0.3" />
|
||||
<PackageReference Include="Microsoft.ML.Tokenizers.Data.O200kBase" Version="1.0.3" />
|
||||
<PackageReference Include="Microsoft.SemanticKernel" Version="1.67.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Caching.Hybrid" Version="10.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="10.0.0" />
|
||||
<PackageReference Include="Microsoft.ML.Tokenizers" Version="2.0.0" />
|
||||
<PackageReference Include="Microsoft.ML.Tokenizers.Data.Cl100kBase" Version="2.0.0" />
|
||||
<PackageReference Include="Microsoft.ML.Tokenizers.Data.O200kBase" Version="2.0.0" />
|
||||
<PackageReference Include="Microsoft.SemanticKernel" Version="1.67.1" />
|
||||
<PackageReference Include="MimeMapping" Version="3.1.0" />
|
||||
<PackageReference Include="MinimalHelpers.FluentValidation" Version="1.1.4" />
|
||||
<PackageReference Include="MinimalHelpers.FluentValidation" Version="1.1.7" />
|
||||
<PackageReference Include="MinimalHelpers.Routing.Analyzers" Version="1.2.2" />
|
||||
<PackageReference Include="PdfPig" Version="0.1.11" />
|
||||
<PackageReference Include="Swashbuckle.AspNetCore.SwaggerUI" Version="9.0.6" />
|
||||
<PackageReference Include="TinyHelpers.AspNetCore" Version="4.1.8" />
|
||||
<PackageReference Include="PdfPig" Version="0.1.12" />
|
||||
<PackageReference Include="Swashbuckle.AspNetCore.SwaggerUI" Version="10.0.1" />
|
||||
<PackageReference Include="TinyHelpers.AspNetCore" Version="4.1.10" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -11,8 +11,8 @@ public class DefaultTextChunker(TokenizerService tokenizerService, IOptions<AppS
|
||||
|
||||
public IList<string> Split(string text)
|
||||
{
|
||||
var lines = TextChunker.SplitPlainTextLines(text, appSettings.MaxTokensPerLine, tokenizerService.CountChatCompletionTokens);
|
||||
var paragraphs = TextChunker.SplitPlainTextParagraphs(lines, appSettings.MaxTokensPerParagraph, appSettings.OverlapTokens, tokenCounter: tokenizerService.CountChatCompletionTokens);
|
||||
var lines = TextChunker.SplitPlainTextLines(text, appSettings.MaxTokensPerLine, tokenizerService.CountEmbeddingTokens);
|
||||
var paragraphs = TextChunker.SplitPlainTextParagraphs(lines, appSettings.MaxTokensPerParagraph, appSettings.OverlapTokens, tokenCounter: tokenizerService.CountEmbeddingTokens);
|
||||
|
||||
return paragraphs;
|
||||
}
|
||||
|
||||
@@ -11,8 +11,8 @@ public class MarkdownTextChunker(TokenizerService tokenizerService, IOptions<App
|
||||
|
||||
public IList<string> Split(string text)
|
||||
{
|
||||
var lines = TextChunker.SplitMarkDownLines(text, appSettings.MaxTokensPerLine, tokenizerService.CountChatCompletionTokens);
|
||||
var paragraphs = TextChunker.SplitMarkdownParagraphs(lines, appSettings.MaxTokensPerParagraph, appSettings.OverlapTokens, tokenCounter: tokenizerService.CountChatCompletionTokens);
|
||||
var lines = TextChunker.SplitMarkDownLines(text, appSettings.MaxTokensPerLine, tokenizerService.CountEmbeddingTokens);
|
||||
var paragraphs = TextChunker.SplitMarkdownParagraphs(lines, appSettings.MaxTokensPerParagraph, appSettings.OverlapTokens, tokenCounter: tokenizerService.CountEmbeddingTokens);
|
||||
|
||||
return paragraphs;
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
"LogLevel": {
|
||||
"Default": "Information",
|
||||
"Microsoft.AspNetCore": "Warning",
|
||||
"Microsoft.AspNetCore.Watch.BrowserRefresh": "Warning",
|
||||
"SqlDatabaseVectorSearch": "Debug"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"ChatCompletion": {
|
||||
"Endpoint": "",
|
||||
"Deployment": "",
|
||||
"ModelId": "", // gpt-4o, gpt-4, gpt-3.5, etc. Note that for gpt-4.1 models, the ModelId must be set to gpt-4o.
|
||||
"ModelId": "", // gpt-4o, gpt-4, gpt-3.5, etc. Note that for gpt-4.1 and gpt-5 models, the ModelId must be set to gpt-4o.
|
||||
"ApiKey": ""
|
||||
},
|
||||
"Embedding": {
|
||||
|
||||
Reference in New Issue
Block a user