@@ -9,9 +7,9 @@
Page Not Found
404
-
Ops! Page Not Found.
+
Oops! Page Not Found.
- The page you're looking for does not exists.
+ The page you're looking for does not exist.
}
else if (Code > 0)
@@ -19,9 +17,9 @@
Unexpected Error
500
-
Ops! Unexpected error.
+
Oops! Unexpected error.
- An unexpected error occurred while loading the page. Please, wait a minute and try again.
+ An unexpected error occurred while loading the page. Please wait a moment and try again.
}
diff --git a/SqlDatabaseVectorSearch/Components/Pages/Home.razor b/SqlDatabaseVectorSearch/Components/Pages/Home.razor
index a120b40..76f2545 100644
--- a/SqlDatabaseVectorSearch/Components/Pages/Home.razor
+++ b/SqlDatabaseVectorSearch/Components/Pages/Home.razor
@@ -3,35 +3,113 @@
SQL Database Vector Search
-
SQL Database Vector Search
+
+
+
+ SQL Database Vector Search
+
+
+ A Blazor Web App and Minimal API for Retrieval Augmented Generation (RAG) and vector search using the native
+ VECTOR type in
+ 
Azure SQL Database or
+ SQL Server 2025, with
+ 
Azure OpenAI and
+ Microsoft Agent Framework.
+
+
+
-
- A Blazor Web App and Minimal API for Retrieval Augmented Generation (RAG) and vector search using the native VECTOR type in Azure SQL Database with Azure OpenAI.
-
+
+
+ How it works
+
-
- This application allows you to:
-
- - Load documents (PDF, DOCX, TXT, MD)
- - Generate embeddings and save them as vectors in Azure SQL Database
- - Perform semantic search and RAG using Azure OpenAI
- - Interact via a Blazor Web App or programmatically via Minimal API
-
- Embeddings and chat completion are powered by
Semantic Kernel. Vectors are managed with
EFCore.SqlServer.VectorSearch.
-
+
+
+
+
+
+
1. Import
+
+ Upload a PDF, DOCX, TXT or MD file. Its content is split into chunks, and each chunk is turned into
+ an embedding by an Agent Framework workflow.
+
+
+
+
+
+
+
+
+
2. Store
+
+ Embeddings are persisted in Azure SQL Database or SQL Server 2025 using the native
+ VECTOR type, so no external vector store is required.
+
+
+
+
+
+
+
+
+
3. Ask
+
+ Your question is reformulated with the conversation context, matched against the stored vectors, and
+ answered with citations.
+
+
+
+
+
-
Supported Features
-
- - Conversation History with Question Reformulation: View and reformulate your conversation history for better clarity and understanding.
- - Information about Token Usage: Access detailed information about token usage for transparency and management.
- - Response Streaming: Receive real-time streaming of responses for a seamless and efficient user experience.
- - Citations: Get citations for the sources used to justify each answer, allowing you to verify and understand the origin of the content.
+
+
+ Supported features
+
+
+
+ -
+
+ Microsoft Agent Framework orchestration: documents are imported through an embedding workflow,
+ and questions are answered by dedicated reformulation and RAG agents.
+
+ -
+
+ Conversation history with question reformulation: follow-up questions are rewritten with the
+ current conversation context before the vector search is performed.
+
+ -
+
+ SQL vector search: the most relevant chunks are retrieved from Azure SQL Database or
+ SQL Server 2025 through native
VECTOR cosine-distance search.
+
+ -
+
+ Token usage details: input, output and total tokens are reported for both question
+ reformulation and answer generation.
+
+ -
+
+ Response streaming: answers are streamed token by token in the chat page and through the
+ Server-Sent Events API.
+
+ -
+
+ Markdown source citations: citations are embedded in the answer with the source name, the page
+ number when available, and a short supporting excerpt.
+
-
- Try uploading a document or ask a question to get started!
-
-
-
- For API usage and more details, see the README.
+
+
+ For API usage and more details, see the
+ README.
diff --git a/SqlDatabaseVectorSearch/Components/Pages/Home.razor.css b/SqlDatabaseVectorSearch/Components/Pages/Home.razor.css
new file mode 100644
index 0000000..fd29ea3
--- /dev/null
+++ b/SqlDatabaseVectorSearch/Components/Pages/Home.razor.css
@@ -0,0 +1,14 @@
+.hero {
+ background: linear-gradient(135deg, #eef4ff 0%, #f8f9fa 100%);
+ border: 1px solid #e3e8f0;
+}
+
+.inline-logo {
+ height: 1.5em;
+ vertical-align: middle;
+}
+
+.card:hover {
+ transform: translateY(-2px);
+ transition: transform 0.15s ease-in-out;
+}
diff --git a/SqlDatabaseVectorSearch/Components/_Imports.razor b/SqlDatabaseVectorSearch/Components/_Imports.razor
index ef23f97..6ef6fce 100644
--- a/SqlDatabaseVectorSearch/Components/_Imports.razor
+++ b/SqlDatabaseVectorSearch/Components/_Imports.razor
@@ -13,4 +13,5 @@
@using SqlDatabaseVectorSearch.Extensions
@using SqlDatabaseVectorSearch.Models
@using SqlDatabaseVectorSearch.Services
+@using SqlDatabaseVectorSearch.Workflows
@using BlazorBootstrap
\ No newline at end of file
diff --git a/SqlDatabaseVectorSearch/Endpoints/AskEndpoints.cs b/SqlDatabaseVectorSearch/Endpoints/AskEndpoints.cs
index 5b236ff..d1b2a80 100644
--- a/SqlDatabaseVectorSearch/Endpoints/AskEndpoints.cs
+++ b/SqlDatabaseVectorSearch/Endpoints/AskEndpoints.cs
@@ -1,4 +1,6 @@
using System.ComponentModel;
+using System.Net.ServerSentEvents;
+using System.Runtime.CompilerServices;
using MinimalHelpers.FluentValidation;
using SqlDatabaseVectorSearch.Models;
using SqlDatabaseVectorSearch.Services;
@@ -23,18 +25,18 @@ public class AskEndpoints : IEndpointRouteHandlerBuilder
endpoints.MapPost("/api/ask-streaming", (Question question, VectorSearchService vectorSearchService, CancellationToken cancellationToken,
[Description("If true, the question will be reformulated taking into account the context of the chat identified by the given ConversationId.")] bool reformulate = true) =>
{
- async IAsyncEnumerable Stream()
+ async IAsyncEnumerable> StreamAsync([EnumeratorCancellation] CancellationToken innerCancellationToken)
{
// Requests a streaming response.
- var responseStream = vectorSearchService.AskStreamingAsync(question, reformulate, cancellationToken);
+ var responseStream = vectorSearchService.AskStreamingAsync(question, reformulate, innerCancellationToken);
await foreach (var delta in responseStream)
{
- yield return delta;
+ yield return new(delta, delta.StreamState?.ToString());
}
}
- return Stream();
+ return TypedResults.ServerSentEvents(StreamAsync(cancellationToken));
})
.WithValidation()
.WithSummary("Asks a question and gets the response as streaming")
diff --git a/SqlDatabaseVectorSearch/Endpoints/DocumentEndpoints.cs b/SqlDatabaseVectorSearch/Endpoints/DocumentEndpoints.cs
index 6862f09..a167f3d 100644
--- a/SqlDatabaseVectorSearch/Endpoints/DocumentEndpoints.cs
+++ b/SqlDatabaseVectorSearch/Endpoints/DocumentEndpoints.cs
@@ -1,9 +1,9 @@
using System.ComponentModel;
using Microsoft.AspNetCore.Http.HttpResults;
-using MimeMapping;
using SqlDatabaseVectorSearch.Models;
using SqlDatabaseVectorSearch.Services;
+using SqlDatabaseVectorSearch.Workflows;
namespace SqlDatabaseVectorSearch.Endpoints;
@@ -23,12 +23,8 @@ public class DocumentEndpoints : IEndpointRouteHandlerBuilder
documentsApiGroup.MapPost(string.Empty, async (IFormFile file, VectorSearchService vectorSearchService, CancellationToken cancellationToken,
[Description("The unique identifier of the document. If not provided, a new one will be generated. If you specify an existing documentId, the corresponding document will be overwritten.")] Guid? documentId = null) =>
{
- using var stream = file.OpenReadStream();
-
- // Note: file.ContentType is not 100% reliable (for example, for markdown file).
- var response = await vectorSearchService.ImportAsync(stream, file.FileName, MimeUtility.GetMimeMapping(file.FileName), documentId, cancellationToken);
-
- return TypedResults.Ok(response);
+ var result = await vectorSearchService.ImportAsync(EmbeddingRequest.FromFormFile(file, documentId), cancellationToken);
+ return TypedResults.Ok(result);
})
.DisableAntiforgery()
.ProducesProblem(StatusCodes.Status400BadRequest)
diff --git a/SqlDatabaseVectorSearch/Models/ChatResponse.cs b/SqlDatabaseVectorSearch/Models/ChatResponse.cs
deleted file mode 100644
index 1f67ca9..0000000
--- a/SqlDatabaseVectorSearch/Models/ChatResponse.cs
+++ /dev/null
@@ -1,3 +0,0 @@
-namespace SqlDatabaseVectorSearch.Models;
-
-public record class ChatResponse(string? Text, TokenUsage? TokenUsage = null);
\ No newline at end of file
diff --git a/SqlDatabaseVectorSearch/Models/Citation.cs b/SqlDatabaseVectorSearch/Models/Citation.cs
deleted file mode 100644
index 04fb64b..0000000
--- a/SqlDatabaseVectorSearch/Models/Citation.cs
+++ /dev/null
@@ -1,16 +0,0 @@
-namespace SqlDatabaseVectorSearch.Models;
-
-public class Citation
-{
- public Guid DocumentId { get; set; }
-
- public Guid ChunkId { get; set; }
-
- public string FileName { get; set; } = null!;
-
- public string Quote { get; set; } = null!;
-
- public int? PageNumber { get; set; }
-
- public int IndexOnPage { get; set; }
-}
\ No newline at end of file
diff --git a/SqlDatabaseVectorSearch/Models/ImportDocumentResponse.cs b/SqlDatabaseVectorSearch/Models/ImportDocumentResponse.cs
deleted file mode 100644
index 252018a..0000000
--- a/SqlDatabaseVectorSearch/Models/ImportDocumentResponse.cs
+++ /dev/null
@@ -1,3 +0,0 @@
-namespace SqlDatabaseVectorSearch.Models;
-
-public record class ImportDocumentResponse(Guid DocumentId, int EmbeddingTokenCount);
diff --git a/SqlDatabaseVectorSearch/Models/Response.cs b/SqlDatabaseVectorSearch/Models/Response.cs
index 62bde55..a80b2ec 100644
--- a/SqlDatabaseVectorSearch/Models/Response.cs
+++ b/SqlDatabaseVectorSearch/Models/Response.cs
@@ -1,10 +1,15 @@
namespace SqlDatabaseVectorSearch.Models;
// Question and Answer can be null when using response streaming.
-public record class Response(string? OriginalQuestion, string? ReformulatedQuestion, string? Answer, StreamState? StreamState = null, TokenUsageResponse? TokenUsage = null, IEnumerable? Citations = null)
+public record class Response(Guid ConversationId, string? OriginalQuestion, string? ReformulatedQuestion, string? Answer, StreamState? StreamState = null, TokenUsageResponse? TokenUsage = null)
{
- public Response(string? token, StreamState streamState, TokenUsageResponse? tokenUsageResponse = null, IEnumerable? citations = null)
- : this(null, null, token, streamState, tokenUsageResponse, citations)
+ public Response(Guid conversationId, string? token, StreamState streamState, TokenUsageResponse? tokenUsageResponse = null)
+ : this(conversationId, null, null, token, streamState, tokenUsageResponse)
+ {
+ }
+
+ public Response(Guid conversationId, StreamState streamState, TokenUsageResponse? tokenUsageResponse = null)
+ : this(conversationId, null, null, null, streamState, tokenUsageResponse)
{
}
}
\ No newline at end of file
diff --git a/SqlDatabaseVectorSearch/Models/StreamState.cs b/SqlDatabaseVectorSearch/Models/StreamState.cs
index 2bb25ad..0846911 100644
--- a/SqlDatabaseVectorSearch/Models/StreamState.cs
+++ b/SqlDatabaseVectorSearch/Models/StreamState.cs
@@ -3,6 +3,6 @@
public enum StreamState
{
Start,
- Append,
+ Delta,
End
}
\ No newline at end of file
diff --git a/SqlDatabaseVectorSearch/Models/TokenUsage.cs b/SqlDatabaseVectorSearch/Models/TokenUsage.cs
deleted file mode 100644
index 9a39649..0000000
--- a/SqlDatabaseVectorSearch/Models/TokenUsage.cs
+++ /dev/null
@@ -1,6 +0,0 @@
-namespace SqlDatabaseVectorSearch.Models;
-
-public record class TokenUsage(int PromptTokens, int CompletionTokens)
-{
- public int TotalTokens => PromptTokens + CompletionTokens;
-}
diff --git a/SqlDatabaseVectorSearch/Models/TokenUsageResponse.cs b/SqlDatabaseVectorSearch/Models/TokenUsageResponse.cs
index 500a72e..1dae2e0 100644
--- a/SqlDatabaseVectorSearch/Models/TokenUsageResponse.cs
+++ b/SqlDatabaseVectorSearch/Models/TokenUsageResponse.cs
@@ -1,9 +1,5 @@
-namespace SqlDatabaseVectorSearch.Models;
+using Microsoft.Extensions.AI;
-public record class TokenUsageResponse(TokenUsage? Reformulation, int? EmbeddingTokenCount, TokenUsage? Question)
-{
- public TokenUsageResponse(TokenUsage? question)
- : this(null, null, question)
- {
- }
-}
+namespace SqlDatabaseVectorSearch.Models;
+
+public record class TokenUsageResponse(UsageDetails? Reformulation, UsageDetails? Question);
diff --git a/SqlDatabaseVectorSearch/Program.cs b/SqlDatabaseVectorSearch/Program.cs
index 655e93a..6428050 100644
--- a/SqlDatabaseVectorSearch/Program.cs
+++ b/SqlDatabaseVectorSearch/Program.cs
@@ -1,8 +1,15 @@
+using System.ClientModel;
using System.Net.Mime;
+using System.Text;
using System.Text.Json.Serialization;
using FluentValidation;
+using Microsoft.Agents.AI;
+using Microsoft.Agents.AI.Hosting;
+using Microsoft.Agents.AI.Workflows;
using Microsoft.EntityFrameworkCore;
-using Microsoft.SemanticKernel;
+using Microsoft.Extensions.AI;
+using OpenAI;
+using OpenAI.Responses;
using SqlDatabaseVectorSearch.Components;
using SqlDatabaseVectorSearch.ContentDecoders;
using SqlDatabaseVectorSearch.Data;
@@ -10,6 +17,7 @@ using SqlDatabaseVectorSearch.Extensions;
using SqlDatabaseVectorSearch.Services;
using SqlDatabaseVectorSearch.Settings;
using SqlDatabaseVectorSearch.TextChunkers;
+using SqlDatabaseVectorSearch.Workflows;
using TinyHelpers.AspNetCore.Extensions;
using TinyHelpers.AspNetCore.OpenApi;
@@ -32,17 +40,23 @@ builder.Services.ConfigureHttpJsonOptions(options =>
builder.Services.AddSingleton(TimeProvider.System);
-builder.Services.AddSqlServer(builder.Configuration.GetConnectionString("SqlConnection"), optionsAction: options =>
+builder.Services.AddDbContext(options =>
{
- options.UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking);
-});
+ var connectionString = builder.Configuration.GetConnectionString("SqlConnection")!;
-builder.Services.AddHybridCache(options =>
-{
- options.DefaultEntryOptions = new()
+ if (connectionString.Contains("database.windows.net"))
{
- LocalCacheExpiration = appSettings.MessageExpiration
- };
+ options.UseAzureSql(connectionString);
+ }
+ else
+ {
+ options.UseSqlServer(connectionString, sqlOptions =>
+ {
+ sqlOptions.EnableRetryOnFailure(maxRetryCount: 5, maxRetryDelay: TimeSpan.FromSeconds(10), errorNumbersToAdd: null);
+ });
+ }
+
+ options.UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking);
});
builder.Services.ConfigureHttpClientDefaults(configure =>
@@ -54,11 +68,25 @@ builder.Services.ConfigureHttpClientDefaults(configure =>
});
});
-// Semantic Kernel is used to generate embeddings and to reformulate questions taking into account all the previous interactions,
-// so that embeddings themselves can be generated more accurately.
-builder.Services.AddKernel()
- .AddAzureOpenAIEmbeddingGenerator(aiSettings.Embedding.Deployment, aiSettings.Embedding.Endpoint, aiSettings.Embedding.ApiKey, modelId: aiSettings.Embedding.ModelId, dimensions: aiSettings.Embedding.Dimensions)
- .AddAzureOpenAIChatCompletion(aiSettings.ChatCompletion.Deployment, aiSettings.ChatCompletion.Endpoint, aiSettings.ChatCompletion.ApiKey, modelId: aiSettings.ChatCompletion.ModelId);
+builder.Services.AddSingleton(_ =>
+{
+ var embeddingClient = new OpenAIClient(new ApiKeyCredential(aiSettings.Embedding.ApiKey), new()
+ {
+ Endpoint = new(aiSettings.Embedding.Endpoint),
+ }).GetEmbeddingClient(aiSettings.Embedding.Deployment).AsIEmbeddingGenerator(aiSettings.Embedding.Dimensions);
+
+ return embeddingClient;
+});
+
+builder.Services.AddChatClient(_ =>
+{
+ var chatClient = new OpenAIClient(new ApiKeyCredential(aiSettings.ChatCompletion.ApiKey), new()
+ {
+ Endpoint = new(aiSettings.ChatCompletion.Endpoint)
+ }).GetResponsesClient().AsIChatClientWithStoredOutputDisabled(aiSettings.ChatCompletion.Deployment);
+
+ return chatClient;
+});
builder.Services.AddKeyedSingleton(MediaTypeNames.Application.Pdf);
builder.Services.AddKeyedSingleton("application/vnd.openxmlformats-officedocument.wordprocessingml.document");
@@ -69,10 +97,168 @@ builder.Services.AddKeyedSingleton(KeyedServic
builder.Services.AddKeyedSingleton(MediaTypeNames.Text.Markdown);
builder.Services.AddSingleton();
-builder.Services.AddSingleton();
builder.Services.AddScoped();
builder.Services.AddScoped();
+builder.Services.AddScoped();
+
+builder.Services.AddSingleton();
+builder.Services.AddSingleton();
+builder.Services.AddScoped(); // This executor is registered as scoped because it uses the DbContext, which is also scoped.
+
+builder.AddWorkflow("EmbeddingWorkflow", (services, key) =>
+{
+ var extractChunksExecutor = services.GetRequiredService();
+ var generateEmbeddingExecutor = services.GetRequiredService();
+ var storeEmbeddingExecutor = services.GetRequiredService();
+
+ var workflow = new WorkflowBuilder(extractChunksExecutor).WithName(key)
+ .AddEdge(extractChunksExecutor, generateEmbeddingExecutor)
+ .AddEdge(generateEmbeddingExecutor, storeEmbeddingExecutor)
+ .WithOutputFrom(storeEmbeddingExecutor)
+ .Build(validateOrphans: true);
+
+ return workflow;
+}, ServiceLifetime.Scoped);
+
+builder.Services.AddAIAgent("ReformulationAgent", (services, key) =>
+{
+ var chatClient = services.GetRequiredService();
+
+ return chatClient.AsAIAgent(new()
+ {
+ Id = key.ToLowerInvariant(),
+ Name = key,
+ ChatOptions = new()
+ {
+ Instructions = """
+ You are a helpful assistant that reformulates questions to perform embeddings search.
+ Your task is to reformulate the question taking into account the context of the chat.
+ The reformulated question must always explicitly contain the subject of the question.
+
+ You MUST reformulate the question in the SAME language as the user's question.
+ For example, if the user asks a question in English, the reformulated question MUST be in English. If the user asks in Italian, the reformulated question MUST be in Italian.
+
+ Never add "in this chat", "in the context of this chat", "in the context of our conversation", "search for" or something like that in your answer.
+ Your answer must contain only the reformulated question and nothing else.
+ Never add follow-up messages, clarifications, notes, disclaimers, or requests for more information such as "if you give me more information, I can be more precise".
+ """,
+ Reasoning = new()
+ {
+ Effort = ReasoningEffort.None,
+ Output = ReasoningOutput.None
+ }
+ },
+ ChatHistoryProvider = new InMemoryChatHistoryProvider(new()
+ {
+ // The reformulation agent reads the conversation only to get the context it needs, but its own questions and answers
+ // must not pollute the session: the history is kept clean for the RAG agent, so nothing is stored back.
+ StorageInputRequestMessageFilter = _ => [],
+ StorageInputResponseMessageFilter = _ => []
+ })
+ },
+ loggerFactory: services.GetRequiredService(),
+ services: services);
+});
+
+var textSearchOptions = new TextSearchProviderOptions()
+{
+ ContextFormatter = results =>
+ {
+ var sb = new StringBuilder();
+
+ sb.AppendLine("## Additional Context");
+ sb.AppendLine("Use the excerpts below to answer the user.");
+ sb.AppendLine("Citation rules:");
+ sb.AppendLine("- Do NOT add inline citations.");
+ sb.AppendLine("- At the END of your answer, add a sources section that follows this template exactly, where the sources label and the page label are localized in the same language as the user's question:");
+ sb.AppendLine(" *localized-sources-label*");
+ sb.AppendLine(" 1. **SourceName**, localized-page-label PageNumber: *supporting excerpt of about 20-30 words*");
+ sb.AppendLine("- Omit the page label and the page number when the page number is not available.");
+ sb.AppendLine("- Do NOT use headings or links in the sources section.");
+ sb.AppendLine("- Include ONLY sources you actually used. No duplicates.");
+ sb.AppendLine();
+
+ sb.AppendLine("### Sources");
+ foreach (var (i, r) in results.Index())
+ {
+ sb.AppendLine($"[{i + 1}] {GetSourceName(r, i)}");
+ sb.AppendLine(r.Text);
+ sb.AppendLine("---");
+ }
+
+ return sb.ToString();
+
+ static string GetSourceName(TextSearchProvider.TextSearchResult result, int index)
+ {
+ var name = string.IsNullOrWhiteSpace(result.SourceName) ? $"Source {index + 1}" : result.SourceName;
+ var pageNumber = result.RawRepresentation is int number ? number : (int?)null;
+ var pageText = pageNumber.HasValue ? $", page {pageNumber}" : string.Empty;
+
+ return $"{name}{pageText}";
+ }
+ }
+};
+
+builder.Services.AddHybridCache(options =>
+{
+ options.DefaultEntryOptions = new()
+ {
+ LocalCacheExpiration = appSettings.MessageExpiration
+ };
+});
+builder.Services.AddSingleton();
+
+builder.Services.AddAIAgent("RagAgent", (services, key) =>
+{
+ var chatClient = services.GetRequiredService();
+
+ return chatClient.AsAIAgent(new()
+ {
+ Id = key.ToLowerInvariant(),
+ Name = key,
+ ChatOptions = new()
+ {
+ Instructions = """
+ You are a helpful assistant. Answer questions using the provided context and cite the source document when available.
+ You can use only the information provided in this chat to answer questions. If you don't know the answer, reply suggesting to refine the question.
+
+ For example, if the user asks "What is the capital of Italy?" and in this chat there isn't information about Italy, you should reply something like:
+ - This information isn't available in the given context.
+ - I'm sorry, I don't know the answer to that question.
+ - I don't have that information.
+ - I don't know.
+ - Given the context, I can't answer that question.
+ - I'm sorry, I don't have enough information to answer that question.
+
+ Never answer questions that are not related to this chat.
+ """,
+ Reasoning = new()
+ {
+ Effort = ReasoningEffort.Low,
+ Output = ReasoningOutput.None
+ }
+ },
+ ChatHistoryProvider = new InMemoryChatHistoryProvider(new()
+ {
+ ChatReducer = new MessageCountingChatReducer(appSettings.MessageLimit),
+ ReducerTriggerEvent = InMemoryChatHistoryProviderOptions.ChatReducerTriggerEvent.AfterMessageAdded,
+ StorageInputRequestMessageFilter = messages =>
+ {
+ return messages.Where(m => m.GetAgentRequestMessageSourceType() != AgentRequestMessageSourceType.ChatHistory
+ && m.GetAgentRequestMessageSourceType() != AgentRequestMessageSourceType.AIContextProvider);
+ }
+ }),
+ AIContextProviders = [new TextSearchProvider(services.GetRequiredService().SearchAsync, textSearchOptions)]
+ },
+ loggerFactory: services.GetRequiredService(),
+ services: services);
+}, ServiceLifetime.Scoped)
+.WithSessionStore((services, _) =>
+{
+ var sessionStore = services.GetRequiredService();
+ return sessionStore;
+}, withIsolation: false);
builder.Services.AddOpenApi(options =>
{
@@ -109,6 +295,7 @@ app.UseWhen(context => context.IsApiRequest(), builder =>
{
app.UseExceptionHandler(new ExceptionHandlerOptions
{
+ SuppressDiagnosticsCallback = _ => false,
StatusCodeSelector = exception => exception switch
{
NotSupportedException => StatusCodes.Status501NotImplemented,
diff --git a/SqlDatabaseVectorSearch/Services/ChatService.cs b/SqlDatabaseVectorSearch/Services/ChatService.cs
deleted file mode 100644
index d0c68b3..0000000
--- a/SqlDatabaseVectorSearch/Services/ChatService.cs
+++ /dev/null
@@ -1,247 +0,0 @@
-using System.Runtime.CompilerServices;
-using System.Text;
-using Microsoft.Extensions.Caching.Hybrid;
-using Microsoft.Extensions.Options;
-using Microsoft.SemanticKernel.ChatCompletion;
-using Microsoft.SemanticKernel.Connectors.AzureOpenAI;
-using OpenAI.Chat;
-using SqlDatabaseVectorSearch.Models;
-using SqlDatabaseVectorSearch.Settings;
-using Entities = SqlDatabaseVectorSearch.Data.Entities;
-
-namespace SqlDatabaseVectorSearch.Services;
-
-public class ChatService(IChatCompletionService chatCompletionService, TokenizerService tokenizerService, HybridCache cache, IOptions appSettingsOptions, ILogger logger)
-{
- private readonly AppSettings appSettings = appSettingsOptions.Value;
-
- private static readonly string systemPromptForReformulation = """
- You are a helpful assistant that reformulates questions to perform embeddings search.
- Your task is to reformulate the question taking into account the context of the chat.
- The reformulated question must always explicitly contain the subject of the question.
- You MUST reformulate the question in the SAME language as the user's question. For example, if the user asks a question in English, the reformulated question MUST be in English. If the user asks in Italian, the reformulated question MUST be in Italian.
-
- If asking a clarifying question to the user would help, ask the question.
- Never add "in this chat", "in the context of this chat", "in the context of our conversation", "search for" or something like that in your answer.
- """;
-
- private static readonly string systemPromptForAnswering = """
- You can use only the information provided in this chat to answer questions. If you don't know the answer, reply suggesting to refine the question.
-
- For example, if the user asks "What is the capital of Italy?" and in this chat there isn't information about Italy, you should reply something like:
- - This information isn't available in the given context.
- - I'm sorry, I don't know the answer to that question.
- - I don't have that information.
- - I don't know.
- - Given the context, I can't answer that question.
- - I'm sorry, I don't have enough information to answer that question.
-
- Never answer questions that are not related to this chat.
-
- LANGUAGE RULE: You MUST ALWAYS answer in the SAME language as the user's question. For example, if the user asks a question in English, the answer MUST be in English. If the user asks in Italian, the answer MUST be in Italian. This rule applies NO MATTER what language the documents are written in. The language of your response must match the language of the question, NOT the language of the documents.
-
- FORMATTING REQUIREMENT: Your answer MUST ALWAYS end with a period followed by a space before the citations block.
- If your answer doesn't naturally end with a period, you MUST add one followed by a space.
-
- After the answer, you need to include citations following the XML format below ONLY IF you know the answer and are providing information from the context. If you do NOT know the answer, DO NOT include the citations section at all.
-
- 【exact quote here
- exact quote here】
-
- The entire list of XML citations MUST be enclosed between 【 and 】 (U+3010 and U+3011) and must exactly match the above format.
- The quote in each MUST be MAXIMUM 5 words, taken word-for-word from the search result.
-
- IMPORTANT CITATION RULES:
- 1. NEVER put citations inside your answer text.
- 2. ALWAYS provide your complete answer FIRST.
- 3. ONLY AFTER completing your answer, add ALL citations in a block at the very end.
- 4. The citations block MUST be the last thing in your response, with absolutely nothing (no text, no spaces, no newlines, no punctuation, no comments) after it.
- 5. NEVER reference citations by number or mention them in your answer text.
- 6. The citations MUST ALWAYS follow the XML format exactly as shown below. Any other format is NOT ACCEPTED.
- 7. If you add anything after the citations block, your answer will be considered invalid.
- 8. If you do NOT know the answer, DO NOT include the citations block at all.
- 9. ALWAYS check that your answer ends with a period followed by a space before adding citations.
-
- ---
- Example of a correct answer:
- The capital of Italy is Rome.
- 【capital of Italy is Rome】
-
- Example of a correct answer when you do NOT know the answer:
- I'm sorry, I don't know the answer to that question.
-
- Example of an incorrect answer (NOT ACCEPTED):
- The capital of Italy is Rome
- 【capital of Italy is Rome】
- Thank you for your question.
-
- Another incorrect example (NOT ACCEPTED):
- The capital of Italy is Rome.
- 【capital of Italy is Rome】
- [1] italy.pdf, page 1
- ---
-
- Only the correct format is accepted. If you do not follow the XML format exactly, or if you add anything after the citations block, your answer will be considered invalid.
- If you do NOT know the answer, DO NOT include the citations block at all.
- Remember to ALWAYS end your answer with a period followed by a space before adding citations.
- """;
-
- public async Task CreateReformulateQuestionAsync(Guid conversationId, string question, CancellationToken cancellationToken = default)
- {
- var chat = await GetChatHistoryAsync(conversationId, cancellationToken);
-
- var settings = new AzureOpenAIPromptExecutionSettings
- {
- ChatSystemPrompt = systemPromptForReformulation
- };
-
- var embeddingQuestion = $"""
- Reformulate the following question:
- ---
- {question}
- """;
-
- chat.AddUserMessage(embeddingQuestion);
-
- var reformulatedQuestion = await chatCompletionService.GetChatMessageContentAsync(chat, settings, cancellationToken: cancellationToken);
-
- chat.AddAssistantMessage(reformulatedQuestion.Content!);
-
- await UpdateCacheAsync(conversationId, chat, cancellationToken);
-
- var tokenUsage = GetTokenUsage(reformulatedQuestion);
- logger.LogDebug("Reformulation: {TokenUsage}", tokenUsage);
-
- return new(reformulatedQuestion.Content!, tokenUsage);
- }
-
- public async Task AskQuestionAsync(Guid conversationId, IEnumerable chunks, string question, CancellationToken cancellationToken = default)
- {
- var (chat, settings) = CreateChatAsync(chunks, question);
-
- var answer = await chatCompletionService.GetChatMessageContentAsync(chat, settings, cancellationToken: cancellationToken);
-
- // Add question and answer to the chat history.
- await SetChatHistoryAsync(conversationId, question, answer.Content!, cancellationToken);
-
- var tokenUsage = GetTokenUsage(answer);
- logger.LogDebug("Ask question: {TokenUsage}", tokenUsage);
-
- return new(answer.Content!, tokenUsage);
- }
-
- public async IAsyncEnumerable AskStreamingAsync(Guid conversationId, IEnumerable chunks, string question, [EnumeratorCancellation] CancellationToken cancellationToken = default)
- {
- var (chat, settings) = CreateChatAsync(chunks, question);
-
- var answer = new StringBuilder();
- await foreach (var token in chatCompletionService.GetStreamingChatMessageContentsAsync(chat, settings, cancellationToken: cancellationToken))
- {
- if (!string.IsNullOrEmpty(token.Content))
- {
- yield return new(token.Content);
- answer.Append(token.Content);
- }
- else if (token.Content is null)
- {
- // Token usage is returned in the last message, when the Content is null.
- var tokenUsage = GetTokenUsage(token);
- if (tokenUsage is not null)
- {
- logger.LogDebug("Ask streaming: {TokenUsage}", tokenUsage);
- yield return new(null, tokenUsage);
- }
- }
- }
-
- // Add question and answer to the chat history.
- await SetChatHistoryAsync(conversationId, question, answer.ToString(), cancellationToken).ConfigureAwait(false);
- }
-
- private static TokenUsage? GetTokenUsage(Microsoft.SemanticKernel.ChatMessageContent message) =>
- message.InnerContent is ChatCompletion content && content.Usage is not null
- ? new(content.Usage.InputTokenCount, content.Usage.OutputTokenCount) : null;
-
- private static TokenUsage? GetTokenUsage(Microsoft.SemanticKernel.StreamingChatMessageContent message) =>
- message.InnerContent is StreamingChatCompletionUpdate content && content.Usage is not null
- ? new(content.Usage.InputTokenCount, content.Usage.OutputTokenCount) : null;
-
- private (ChatHistory Chat, AzureOpenAIPromptExecutionSettings Settings) CreateChatAsync(IEnumerable chunks, string question)
- {
- var settings = new AzureOpenAIPromptExecutionSettings
- {
- MaxTokens = appSettings.MaxOutputTokens,
- ChatSystemPrompt = systemPromptForAnswering
- };
-
- var prompt = new StringBuilder($"""
- Answer the following question:
- ---
- {question}
- =====
- Using the following information:
-
- """);
-
- var availableTokens = appSettings.MaxInputTokens
- - tokenizerService.CountChatCompletionTokens(systemPromptForAnswering) // System prompt.
- - tokenizerService.CountChatCompletionTokens(prompt.ToString()) // Initial user prompt.
- - appSettings.MaxOutputTokens; // To ensure there is enough space for the answer.
-
- foreach (var chunk in chunks)
- {
- var text = $"--- {chunk.Document.Name} (Document ID: {chunk.Document.Id} | Chunk ID: {chunk.Id} | Page Number: {chunk.PageNumber} | Index on Page: {chunk.IndexOnPage}) {Environment.NewLine}{chunk.Content}{Environment.NewLine}";
-
- var tokenCount = tokenizerService.CountChatCompletionTokens(text);
- if (tokenCount > availableTokens)
- {
- // There isn't enough space to add the current chunk.
- break;
- }
-
- prompt.Append(text);
-
- availableTokens -= tokenCount;
- if (availableTokens <= 0)
- {
- // There isn't enough space to add more chunks.
- break;
- }
- }
-
- var chat = new ChatHistory();
- chat.AddUserMessage(prompt.ToString());
-
- return (chat, settings);
- }
-
- private async Task UpdateCacheAsync(Guid conversationId, ChatHistory chat, CancellationToken cancellationToken)
- {
- if (chat.Count > appSettings.MessageLimit)
- {
- chat.RemoveRange(0, chat.Count - appSettings.MessageLimit);
- }
-
- await cache.SetAsync(conversationId.ToString(), chat, cancellationToken: cancellationToken);
- }
-
- private async Task GetChatHistoryAsync(Guid conversationId, CancellationToken cancellationToken)
- {
- var chat = await cache.GetOrCreateAsync(conversationId.ToString(), (cancellationToken) =>
- {
- return ValueTask.FromResult([]);
- }, cancellationToken: cancellationToken);
-
- return chat;
- }
-
- private async Task SetChatHistoryAsync(Guid conversationId, string question, string answer, CancellationToken cancellationToken)
- {
- var chat = await GetChatHistoryAsync(conversationId, cancellationToken);
-
- chat.AddUserMessage(question);
- chat.AddAssistantMessage(answer);
-
- await UpdateCacheAsync(conversationId, chat, cancellationToken);
- }
-}
diff --git a/SqlDatabaseVectorSearch/Services/DocumentContextProviderService.cs b/SqlDatabaseVectorSearch/Services/DocumentContextProviderService.cs
new file mode 100644
index 0000000..4b29b5d
--- /dev/null
+++ b/SqlDatabaseVectorSearch/Services/DocumentContextProviderService.cs
@@ -0,0 +1,35 @@
+using System.Data;
+using Microsoft.Agents.AI;
+using Microsoft.Data.SqlTypes;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.AI;
+using Microsoft.Extensions.Options;
+using SqlDatabaseVectorSearch.Data;
+using SqlDatabaseVectorSearch.Settings;
+
+namespace SqlDatabaseVectorSearch.Services;
+
+public class DocumentContextProviderService(ApplicationDbContext dbContext, IEmbeddingGenerator> embeddingGenerator, IOptions appSettingsOptions)
+{
+ private readonly AppSettings appSettings = appSettingsOptions.Value;
+
+ public async Task> SearchAsync(string query, CancellationToken cancellationToken)
+ {
+ // Perform Vector Search on SQL Database.
+ var questionEmbedding = await embeddingGenerator.GenerateVectorAsync(query, cancellationToken: cancellationToken);
+ var embeddingVector = new SqlVector(questionEmbedding);
+
+ var chunks = await dbContext.DocumentChunks.Include(c => c.Document)
+ .OrderBy(c => EF.Functions.VectorDistance("cosine", c.Embedding, embeddingVector))
+ .Take(appSettings.MaxRelevantChunks).Select(c => new TextSearchProvider.TextSearchResult
+ {
+ SourceLink = c.Id.ToString().ToLowerInvariant(),
+ SourceName = c.Document.Name,
+ Text = c.Content,
+ RawRepresentation = c.PageNumber
+ })
+ .ToListAsync(cancellationToken);
+
+ return chunks;
+ }
+}
\ No newline at end of file
diff --git a/SqlDatabaseVectorSearch/Services/HybridCacheSessionStoreService.cs b/SqlDatabaseVectorSearch/Services/HybridCacheSessionStoreService.cs
new file mode 100644
index 0000000..e66a48f
--- /dev/null
+++ b/SqlDatabaseVectorSearch/Services/HybridCacheSessionStoreService.cs
@@ -0,0 +1,37 @@
+using Microsoft.Agents.AI;
+using Microsoft.Agents.AI.Hosting;
+using Microsoft.Extensions.Caching.Hybrid;
+
+namespace SqlDatabaseVectorSearch.Services;
+
+public class HybridCacheSessionStoreService(HybridCache cache) : AgentSessionStore
+{
+ public override async ValueTask GetSessionAsync(AIAgent agent, string conversationId, CancellationToken cancellationToken = default)
+ {
+ var key = GetKey(agent, conversationId);
+ var sessionContent = await cache.GetOrCreateAsync(key, async ct =>
+ {
+ var session = await agent.CreateSessionAsync(ct);
+ return await agent.SerializeSessionAsync(session, cancellationToken: ct);
+ }, cancellationToken: cancellationToken);
+
+ return await agent.DeserializeSessionAsync(sessionContent, cancellationToken: cancellationToken);
+ }
+
+ public override async ValueTask SaveSessionAsync(AIAgent agent, string conversationId, AgentSession session, CancellationToken cancellationToken = default)
+ {
+ var key = GetKey(agent, conversationId);
+ var sessionContent = await agent.SerializeSessionAsync(session, cancellationToken: cancellationToken);
+
+ await cache.SetAsync(key, sessionContent, cancellationToken: cancellationToken);
+ }
+
+ public override async ValueTask DeleteSessionAsync(AIAgent agent, string conversationId, CancellationToken cancellationToken = default)
+ {
+ var key = GetKey(agent, conversationId);
+ await cache.RemoveAsync(key, cancellationToken);
+ }
+
+ private static string GetKey(AIAgent agent, string conversationId)
+ => $"{agent.Id}:{conversationId}";
+}
\ No newline at end of file
diff --git a/SqlDatabaseVectorSearch/Services/VectorSearchService.cs b/SqlDatabaseVectorSearch/Services/VectorSearchService.cs
index f18d39e..edcaee8 100644
--- a/SqlDatabaseVectorSearch/Services/VectorSearchService.cs
+++ b/SqlDatabaseVectorSearch/Services/VectorSearchService.cs
@@ -1,200 +1,84 @@
using System.Data;
using System.Runtime.CompilerServices;
-using System.Text;
-using System.Text.RegularExpressions;
-using Microsoft.Data.SqlTypes;
-using Microsoft.EntityFrameworkCore;
+using Microsoft.Agents.AI;
+using Microsoft.Agents.AI.Hosting;
+using Microsoft.Agents.AI.Workflows;
using Microsoft.Extensions.AI;
-using Microsoft.Extensions.Options;
-using SqlDatabaseVectorSearch.ContentDecoders;
-using SqlDatabaseVectorSearch.Data;
using SqlDatabaseVectorSearch.Models;
-using SqlDatabaseVectorSearch.Settings;
-using ChatResponse = SqlDatabaseVectorSearch.Models.ChatResponse;
-using Entities = SqlDatabaseVectorSearch.Data.Entities;
+using SqlDatabaseVectorSearch.Workflows;
namespace SqlDatabaseVectorSearch.Services;
-public partial class VectorSearchService(IServiceProvider serviceProvider, ApplicationDbContext dbContext, DocumentService documentService, IEmbeddingGenerator> embeddingGenerator, TokenizerService tokenizerService, ChatService chatService, TimeProvider timeProvider, IOptions appSettingsOptions, ILogger logger)
+public partial class VectorSearchService([FromKeyedServices("EmbeddingWorkflow")] Workflow workflow, [FromKeyedServices("ReformulationAgent")] AIAgent reformulationAgent, [FromKeyedServices("RagAgent")] AIAgent ragAgent,
+ [FromKeyedServices("RagAgent")] AgentSessionStore sessionStore)
{
- private readonly AppSettings appSettings = appSettingsOptions.Value;
-
- public async Task ImportAsync(Stream stream, string name, string contentType, Guid? documentId, CancellationToken cancellationToken = default)
+ public async Task ImportAsync(EmbeddingRequest request, CancellationToken cancellationToken = default)
{
- // Extract the contents of the file.
- var decoder = serviceProvider.GetKeyedService(contentType) ?? throw new NotSupportedException($"Content type '{contentType}' is not supported.");
- var chunks = await decoder.DecodeAsync(stream, contentType, cancellationToken);
- var chunkContents = chunks.Select(p => p.Content).ToList();
+ await using var run = await InProcessExecution.RunAsync(workflow, request, cancellationToken: cancellationToken);
+ var events = run.NewEvents.ToList();
- // We get the token count of the whole document because it is the total number of token used by embedding (it may be necessary, for example, for cost analysis).
- var tokenCount = tokenizerService.CountEmbeddingTokens(string.Join(" ", chunkContents));
-
- var strategy = dbContext.Database.CreateExecutionStrategy();
- var document = await strategy.ExecuteAsync(async (cancellationToken) =>
+ var exception = events.OfType().Select(e => e.Exception).FirstOrDefault();
+ if (exception is not null)
{
- await dbContext.Database.BeginTransactionAsync(cancellationToken);
+ throw exception;
+ }
- if (documentId.HasValue)
- {
- // If the user is importing a document that already exists, delete the previous one.
- await documentService.DeleteAsync(documentId.Value, cancellationToken);
- }
-
- var document = new Entities.Document { Id = documentId.GetValueOrDefault(), Name = name, CreationDate = timeProvider.GetUtcNow() };
- dbContext.Documents.Add(document);
-
- // Process paragraphs in batches.
- var embeddings = new List>();
- foreach (var batch in chunkContents.Chunk(appSettings.EmbeddingBatchSize))
- {
- logger.LogDebug("Processing batch of {Count} chunks for embedding generation...", batch.Length);
-
- // Generate embeddings for this batch.
- var batchEmbeddings = await embeddingGenerator.GenerateAsync(batch, cancellationToken: cancellationToken);
- embeddings.AddRange(batchEmbeddings);
- }
-
- // Save the document chunks and the corresponding embedding in the database.
- foreach (var (index, embedding) in embeddings.Index())
- {
- var chunk = chunks.ElementAt(index);
- logger.LogDebug("Storing a chunk of {TokenCount} tokens.", tokenizerService.CountEmbeddingTokens(chunk.Content));
-
- var documentChunk = new Entities.DocumentChunk
- {
- Document = document,
- Index = index,
- PageNumber = chunk.PageNumber,
- IndexOnPage = chunk.IndexOnPage,
- Content = chunk.Content,
- Embedding = new SqlVector(embedding.Vector)
- };
-
- dbContext.DocumentChunks.Add(documentChunk);
- }
-
- await dbContext.SaveChangesAsync(cancellationToken);
- await dbContext.Database.CommitTransactionAsync(cancellationToken);
-
- return document;
- }, cancellationToken);
-
- return new(document.Id, tokenCount);
+ var result = events.OfType().Select(e => e.Data).OfType().First();
+ return result;
}
public async Task AskQuestionAsync(Question question, bool reformulate = true, CancellationToken cancellationToken = default)
{
- // It the user doesn't want to reforulate the question, CreateContextAsync returns the original one.
- var (reformulatedQuestion, embeddingTokenCount, chunks) = await CreateContextAsync(question, reformulate, cancellationToken);
+ UsageDetails? reformulationUsage = null;
+ var reformulatedQuestion = question.Text;
+ var session = await sessionStore.GetSessionAsync(ragAgent, question.ConversationId.ToString(), cancellationToken);
- var (fullAnswer, tokenUsage) = await chatService.AskQuestionAsync(question.ConversationId, chunks, reformulatedQuestion.Text!, cancellationToken);
+ if (reformulate)
+ {
+ // Reformulates the question taking into account the context of the chat to perform keyword search and embeddings.
+ var reformulationResponse = await reformulationAgent.RunAsync(question.Text, session, cancellationToken: cancellationToken);
+ reformulatedQuestion = reformulationResponse.Text;
+ reformulationUsage = reformulationResponse.Usage;
+ }
- // Extract citations from the answer.
- var (answer, citations) = ExtractCitations(fullAnswer);
+ var response = await ragAgent.RunAsync(reformulatedQuestion, session, cancellationToken: cancellationToken);
- return new(question.Text, reformulatedQuestion.Text!, answer, StreamState.End, new(reformulatedQuestion.TokenUsage, embeddingTokenCount, tokenUsage), citations);
+ await sessionStore.SaveSessionAsync(ragAgent, question.ConversationId.ToString(), session, cancellationToken);
+
+ return new(question.ConversationId, question.Text, reformulatedQuestion, response.Text, null, new TokenUsageResponse(reformulationUsage, response.Usage));
}
public async IAsyncEnumerable AskStreamingAsync(Question question, bool reformulate = true, [EnumeratorCancellation] CancellationToken cancellationToken = default)
{
- // It the user doesn't want to reforulate the question, CreateContextAsync returns the original one.
- var (reformulatedQuestion, embeddingTokenCount, chunks) = await CreateContextAsync(question, reformulate, cancellationToken);
+ UsageDetails? reformulationUsage = null;
+ var reformulatedQuestion = question.Text;
+ var session = await sessionStore.GetSessionAsync(ragAgent, question.ConversationId.ToString(), cancellationToken);
- var answerStream = chatService.AskStreamingAsync(question.ConversationId, chunks, reformulatedQuestion.Text!, cancellationToken: cancellationToken);
+ if (reformulate)
+ {
+ // Reformulates the question taking into account the context of the chat to perform keyword search and embeddings.
+ var reformulationResponse = await reformulationAgent.RunAsync(question.Text, session, cancellationToken: cancellationToken);
+ reformulatedQuestion = reformulationResponse.Text;
+ reformulationUsage = reformulationResponse.Usage;
+ }
// The first message contains the question and the corresponding token usage (if reformulated).
- yield return new(question.Text, reformulatedQuestion.Text!, null, StreamState.Start, new(reformulatedQuestion.TokenUsage, embeddingTokenCount, null));
+ yield return new(question.ConversationId, question.Text, reformulatedQuestion, null, StreamState.Start, new(reformulationUsage, null));
- TokenUsageResponse? tokenUsageResponse = null;
- var fullAnswer = new StringBuilder();
- var citationsStarted = false;
+ var updates = new List();
- // Returns each token as a partial response.
- await foreach (var (token, tokenUsage) in answerStream)
+ await foreach (var update in ragAgent.RunStreamingAsync(reformulatedQuestion, session, cancellationToken: cancellationToken))
{
- if (token is not null) // token can be null when the stream ends.
+ updates.Add(update);
+ if (!string.IsNullOrEmpty(update.Text))
{
- fullAnswer.Append(token);
-
- if (token.Contains('【'))
- {
- // Citations start when we encounter a token containing a 【 character.
- // We need to track it because we don't want to return the citations in the actual response.
- citationsStarted = true;
- }
-
- if (!citationsStarted)
- {
- yield return new(token, StreamState.Append);
- }
- }
- else
- {
- // Token usage is expected in the last message, when token is null.
- tokenUsageResponse ??= tokenUsage is not null ? new(tokenUsage) : null;
+ yield return new(question.ConversationId, update.Text, StreamState.Delta);
}
}
- // Extract citations at the end of streaming.
- var (_, citations) = ExtractCitations(fullAnswer.ToString());
- yield return new(null, StreamState.End, tokenUsageResponse, citations);
+ await sessionStore.SaveSessionAsync(ragAgent, question.ConversationId.ToString(), session, cancellationToken);
+ var response = updates.ToAgentResponse();
+
+ yield return new(question.ConversationId, StreamState.End, new TokenUsageResponse(null, response.Usage));
}
-
- private async Task<(ChatResponse ReformulatedQuestion, int EmbeddingTokenCount, IEnumerable Chunks)> CreateContextAsync(Question question, bool reformulate, CancellationToken cancellationToken)
- {
- // Reformulate the question taking into account the context of the chat to perform keyword search and embeddings.
- var reformulatedQuestion = reformulate ? await chatService.CreateReformulateQuestionAsync(question.ConversationId, question.Text, cancellationToken) : new(question.Text);
-
- var embeddingTokenCount = tokenizerService.CountEmbeddingTokens(reformulatedQuestion.Text!);
- logger.LogDebug("Embedding Token Count: {EmbeddingTokenCount}", embeddingTokenCount);
-
- // Perform Vector Search on SQL Database.
- var questionEmbedding = await embeddingGenerator.GenerateVectorAsync(reformulatedQuestion.Text!, cancellationToken: cancellationToken);
- var embeddingVector = new SqlVector(questionEmbedding);
-
- var chunks = await dbContext.DocumentChunks.Include(c => c.Document)
- .OrderBy(c => EF.Functions.VectorDistance("cosine", c.Embedding, embeddingVector))
- .Take(appSettings.MaxRelevantChunks)
- .ToListAsync(cancellationToken);
-
- return (reformulatedQuestion, embeddingTokenCount, chunks);
- }
-
- private static (string, IEnumerable) ExtractCitations(string? text)
- {
- var citations = new List();
-
- if (string.IsNullOrEmpty(text))
- {
- return (text ?? string.Empty, citations);
- }
-
- var matches = CitationRegEx.Matches(text);
-
- foreach (Match match in matches)
- {
- if (match.Success)
- {
- citations.Add(new Citation
- {
- DocumentId = Guid.Parse(match.Groups["documentId"].Value),
- ChunkId = Guid.Parse(match.Groups["chunkId"].Value),
- FileName = match.Groups["filename"].Value,
- PageNumber = int.TryParse(match.Groups["pageNumber"].Value, out var pageNumber) && pageNumber > 0 ? pageNumber : null,
- IndexOnPage = int.TryParse(match.Groups["indexOnPage"].Value, out var indexOnPage) ? indexOnPage : 0,
- Quote = match.Groups["quote"].Value
- });
- }
- }
-
- // Remove all content between 【 and 】.
- var cleanText = RemoveCitationsRegEx.Replace(text, string.Empty).TrimEnd();
- return (cleanText, citations.OrderBy(c => c.FileName).ThenBy(c => c.PageNumber));
- }
-
- [GeneratedRegex(@"[^""']*)(?:""|'|)\s+chunk-id=(?:""|'|)(?[^""']*)(?:""|'|)\s+filename=(?:""|'|)(?[^""']*)(?:""|'|)\s+page-number=(?:""|'|)(?[^""']*)(?:""|'|)\s+index-on-page=(?:""|'|)(?[^""']*)(?:""|'|)>\s*(?.*?)\s*
", RegexOptions.Singleline)]
- private static partial Regex CitationRegEx { get; }
-
- [GeneratedRegex(@"【.*?】", RegexOptions.Singleline)]
- private static partial Regex RemoveCitationsRegEx { get; }
-}
\ No newline at end of file
+}
diff --git a/SqlDatabaseVectorSearch/Settings/AppSettings.cs b/SqlDatabaseVectorSearch/Settings/AppSettings.cs
index 538fed8..fed324e 100644
--- a/SqlDatabaseVectorSearch/Settings/AppSettings.cs
+++ b/SqlDatabaseVectorSearch/Settings/AppSettings.cs
@@ -12,10 +12,6 @@ public class AppSettings
public int MaxRelevantChunks { get; init; } = 5;
- public int MaxInputTokens { get; init; } = 16385;
-
- public int MaxOutputTokens { get; init; } = 800;
-
public TimeSpan MessageExpiration { get; init; }
public int MessageLimit { get; set; } = 20;
diff --git a/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj b/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj
index c82f9d0..fd24438 100644
--- a/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj
+++ b/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj
@@ -4,7 +4,7 @@
net10.0
enable
enable
- $(NoWarn);SKEXP0010;SKEXP0050
+ $(NoWarn);SKEXP0010;SKEXP0050;OPENAI001;MAAI001;MEAI001
@@ -12,24 +12,34 @@
-
-
-
+
+
+
+
+ all
+ runtime; build; native; contentfiles; analyzers; buildtransitive
+
+
+
+ all
+ runtime; build; native; contentfiles; analyzers; buildtransitive
+
+
+
all
runtime; build; native; contentfiles; analyzers; buildtransitive
-
-
+
+
-
-
-
-
+
+
+
diff --git a/SqlDatabaseVectorSearch/TextChunkers/DefaultTextChunker.cs b/SqlDatabaseVectorSearch/TextChunkers/DefaultTextChunker.cs
index a5a6854..777f48f 100644
--- a/SqlDatabaseVectorSearch/TextChunkers/DefaultTextChunker.cs
+++ b/SqlDatabaseVectorSearch/TextChunkers/DefaultTextChunker.cs
@@ -1,7 +1,7 @@
using Microsoft.Extensions.Options;
-using Microsoft.SemanticKernel.Text;
using SqlDatabaseVectorSearch.Services;
using SqlDatabaseVectorSearch.Settings;
+using SqlDatabaseVectorSearch.TextChunkers.Implementations;
namespace SqlDatabaseVectorSearch.TextChunkers;
@@ -11,8 +11,8 @@ public class DefaultTextChunker(TokenizerService tokenizerService, IOptions Split(string text)
{
- var lines = TextChunker.SplitPlainTextLines(text, appSettings.MaxTokensPerLine, tokenizerService.CountEmbeddingTokens);
- var paragraphs = TextChunker.SplitPlainTextParagraphs(lines, appSettings.MaxTokensPerParagraph, appSettings.OverlapTokens, tokenCounter: tokenizerService.CountEmbeddingTokens);
+ var lines = PlainTextChunker.SplitPlainTextLines(text, appSettings.MaxTokensPerLine, tokenizerService.CountEmbeddingTokens);
+ var paragraphs = PlainTextChunker.SplitPlainTextParagraphs(lines, appSettings.MaxTokensPerParagraph, appSettings.OverlapTokens, tokenCounter: tokenizerService.CountEmbeddingTokens);
return paragraphs;
}
diff --git a/SqlDatabaseVectorSearch/TextChunkers/Implementations/PlainTextChunker.cs b/SqlDatabaseVectorSearch/TextChunkers/Implementations/PlainTextChunker.cs
new file mode 100644
index 0000000..80c25e0
--- /dev/null
+++ b/SqlDatabaseVectorSearch/TextChunkers/Implementations/PlainTextChunker.cs
@@ -0,0 +1,372 @@
+using System.Diagnostics;
+using System.Text;
+
+namespace SqlDatabaseVectorSearch.TextChunkers.Implementations;
+
+///
+/// Split text in chunks, attempting to leave meaning intact.
+/// For plain text, split looking at new lines first, then periods, and so on.
+/// For markdown, split looking at punctuation first, and so on.
+///
+internal static class PlainTextChunker
+{
+ ///
+ /// Represents a list of strings with token count.
+ /// Used to reduce the number of calls to the tokenizer.
+ ///
+ private sealed class StringListWithTokenCount(TokenCounter? tokenCounter)
+ {
+ private readonly TokenCounter? tokenCounter = tokenCounter;
+
+ public void Add(string value) => Values.Add((value, tokenCounter is null ? GetDefaultTokenCount(value.Length) : tokenCounter(value)));
+
+ public void Add(string value, int tokenCount) => Values.Add((value, tokenCount));
+
+ public void AddRange(StringListWithTokenCount range) => Values.AddRange(range.Values);
+
+ public void RemoveRange(int index, int count) => Values.RemoveRange(index, count);
+
+ public int Count => Values.Count;
+
+ public List ToStringList() => Values.Select(v => v.Value).ToList();
+
+ private List<(string Value, int TokenCount)> Values { get; } = [];
+
+ public string ValueAt(int i) => Values[i].Value;
+
+ public int TokenCountAt(int i) => Values[i].TokenCount;
+ }
+
+ ///
+ /// Delegate for counting tokens in a string.
+ ///
+ /// The input string to count tokens in.
+ /// The number of tokens in the input string.
+ public delegate int TokenCounter(string input);
+
+ private static readonly string?[] plainTextSplitOptions = ["\n", ".。．", "?!", ";", ":", ",，、", ")]}", " ", "-", null];
+ private static readonly string?[] markdownSplitOptions = [".\u3002\uFF0E", "?!", ";", ":", ",\uFF0C\u3001", ")]}", " ", "-", "\n\r", null];
+
+ ///
+ /// Split plain text into lines.
+ ///
+ /// Text to split
+ /// Maximum number of tokens per line.
+ /// Function to count tokens in a string. If not supplied, the default counter will be used.
+ /// List of lines.
+ public static List SplitPlainTextLines(string text, int maxTokensPerLine, TokenCounter? tokenCounter = null)
+ {
+ ArgumentNullException.ThrowIfNull(text);
+ ValidateMaxTokens(maxTokensPerLine, nameof(maxTokensPerLine));
+
+ return InternalSplitLines(text, maxTokensPerLine, trim: true, plainTextSplitOptions, tokenCounter);
+ }
+
+ ///
+ /// Split markdown text into lines.
+ ///
+ /// Text to split
+ /// Maximum number of tokens per line.
+ /// Function to count tokens in a string. If not supplied, the default counter will be used.
+ /// List of lines.
+ public static List SplitMarkdownLines(string text, int maxTokensPerLine, TokenCounter? tokenCounter = null)
+ {
+ ArgumentNullException.ThrowIfNull(text);
+ ValidateMaxTokens(maxTokensPerLine, nameof(maxTokensPerLine));
+
+ return InternalSplitLines(text, maxTokensPerLine, trim: true, markdownSplitOptions, tokenCounter);
+ }
+
+ ///
+ /// Split plain text into paragraphs.
+ ///
+ /// Lines of text.
+ /// Maximum number of tokens per paragraph.
+ /// Number of tokens to overlap between paragraphs.
+ /// Text to be prepended to each individual chunk.
+ /// Function to count tokens in a string. If not supplied, the default counter will be used.
+ /// List of paragraphs.
+ public static List SplitPlainTextParagraphs(IEnumerable lines, int maxTokensPerParagraph, int overlapTokens = 0, string? chunkHeader = null, TokenCounter? tokenCounter = null)
+ => InternalSplitTextParagraphs(lines, maxTokensPerParagraph, overlapTokens, chunkHeader,
+ static (text, maxTokens, tokenCounter) => InternalSplitLines(text, maxTokens, trim: false, plainTextSplitOptions, tokenCounter), tokenCounter);
+
+ ///
+ /// Split markdown text into paragraphs.
+ ///
+ /// Lines of text.
+ /// Maximum number of tokens per paragraph.
+ /// Number of tokens to overlap between paragraphs.
+ /// Text to be prepended to each individual chunk.
+ /// Function to count tokens in a string. If not supplied, the default counter will be used.
+ /// List of paragraphs.
+ public static List SplitMarkdownParagraphs(IEnumerable lines, int maxTokensPerParagraph, int overlapTokens = 0, string? chunkHeader = null, TokenCounter? tokenCounter = null)
+ => InternalSplitTextParagraphs(lines, maxTokensPerParagraph, overlapTokens, chunkHeader,
+ static (text, maxTokens, tokenCounter) => InternalSplitLines(text, maxTokens, trim: false, markdownSplitOptions, tokenCounter), tokenCounter);
+
+ private static List InternalSplitTextParagraphs(IEnumerable lines, int maxTokensPerParagraph, int overlapTokens, string? chunkHeader, Func> longLinesSplitter, TokenCounter? tokenCounter)
+ {
+ ArgumentNullException.ThrowIfNull(lines);
+ ValidateMaxTokens(maxTokensPerParagraph, nameof(maxTokensPerParagraph));
+
+ if (overlapTokens < 0)
+ {
+ throw new ArgumentOutOfRangeException(nameof(overlapTokens), "overlapTokens cannot be negative.");
+ }
+
+ if (maxTokensPerParagraph <= overlapTokens)
+ {
+ throw new ArgumentException("overlapTokens cannot be larger than or equal to maxTokensPerParagraph.", nameof(overlapTokens));
+ }
+
+ // Optimize empty inputs if we can efficiently determine they're empty.
+ if (lines is ICollection c && c.Count == 0)
+ {
+ return [];
+ }
+
+ var chunkHeaderTokens = chunkHeader is { Length: > 0 } ? GetTokenCount(chunkHeader, tokenCounter) : 0;
+ var adjustedMaxTokensPerParagraph = maxTokensPerParagraph - overlapTokens - chunkHeaderTokens;
+ if (adjustedMaxTokensPerParagraph <= 0)
+ {
+ throw new ArgumentException("chunkHeader and overlapTokens must leave room for paragraph content.", nameof(chunkHeader));
+ }
+
+ // Split long lines first
+ var truncatedLines = lines.SelectMany(line => longLinesSplitter(NormalizeLineEndings(line), adjustedMaxTokensPerParagraph, tokenCounter));
+
+ var paragraphs = BuildParagraph(truncatedLines, adjustedMaxTokensPerParagraph, tokenCounter);
+ var processedParagraphs = ProcessParagraphs(paragraphs, adjustedMaxTokensPerParagraph, overlapTokens, chunkHeader, longLinesSplitter, tokenCounter);
+
+ return processedParagraphs;
+ }
+
+ private static List BuildParagraph(IEnumerable truncatedLines, int maxTokensPerParagraph, TokenCounter? tokenCounter)
+ {
+ StringBuilder paragraphBuilder = new();
+ List paragraphs = [];
+
+ foreach (var line in truncatedLines)
+ {
+ if (paragraphBuilder.Length > 0)
+ {
+ string? paragraph = null;
+
+ var currentCount = GetTokenCount(line, tokenCounter) + 1;
+ if (currentCount < maxTokensPerParagraph)
+ {
+ currentCount += tokenCounter is null ?
+ GetDefaultTokenCount(paragraphBuilder.Length) :
+ tokenCounter(paragraph = paragraphBuilder.ToString());
+ }
+
+ if (currentCount >= maxTokensPerParagraph)
+ {
+ // Complete the paragraph and prepare for the next
+ paragraph ??= paragraphBuilder.ToString();
+ paragraphs.Add(paragraph.Trim());
+ paragraphBuilder.Clear();
+ }
+ }
+
+ paragraphBuilder.AppendLine(line);
+ }
+
+ if (paragraphBuilder.Length > 0)
+ {
+ // Add the final paragraph if there's anything remaining
+ paragraphs.Add(paragraphBuilder.ToString().Trim());
+ }
+
+ return paragraphs;
+ }
+
+ private static List ProcessParagraphs(List paragraphs, int adjustedMaxTokensPerParagraph, int overlapTokens, string? chunkHeader, Func> longLinesSplitter, TokenCounter? tokenCounter)
+ {
+ // distribute text more evenly in the last paragraphs when the last paragraph is too short.
+ if (paragraphs.Count > 1)
+ {
+ var lastParagraph = paragraphs[^1];
+ var secondLastParagraph = paragraphs[^2];
+
+ if (GetTokenCount(lastParagraph, tokenCounter) < adjustedMaxTokensPerParagraph / 4)
+ {
+ var mergedParagraph = $"{secondLastParagraph} {lastParagraph}";
+
+ if (GetTokenCount(mergedParagraph, tokenCounter) <= adjustedMaxTokensPerParagraph)
+ {
+ paragraphs[^2] = mergedParagraph;
+ paragraphs.RemoveAt(paragraphs.Count - 1);
+ }
+ }
+ }
+
+ var processedParagraphs = new List();
+ var paragraphStringBuilder = new StringBuilder();
+
+ for (var i = 0; i < paragraphs.Count; i++)
+ {
+ paragraphStringBuilder.Clear();
+
+ if (chunkHeader is not null)
+ {
+ paragraphStringBuilder.Append(chunkHeader);
+ }
+
+ var paragraph = paragraphs[i];
+
+ if (overlapTokens > 0 && i < paragraphs.Count - 1)
+ {
+ var nextParagraph = paragraphs[i + 1];
+ var split = longLinesSplitter(nextParagraph, overlapTokens, tokenCounter);
+
+ paragraphStringBuilder.Append(paragraph);
+
+ if (split.Count != 0)
+ {
+ paragraphStringBuilder.Append(' ').Append(split[0]);
+ }
+ }
+ else
+ {
+ paragraphStringBuilder.Append(paragraph);
+ }
+
+ processedParagraphs.Add(paragraphStringBuilder.ToString());
+ }
+
+ return processedParagraphs;
+ }
+
+ private static List InternalSplitLines(string text, int maxTokensPerLine, bool trim, string?[] splitOptions, TokenCounter? tokenCounter)
+ {
+ var result = new StringListWithTokenCount(tokenCounter);
+
+ text = NormalizeLineEndings(text);
+ result.Add(text);
+ for (var i = 0; i < splitOptions.Length; i++)
+ {
+ var count = result.Count; // track where the original input left off
+ var (splits2, inputWasSplit2) = Split(result, maxTokensPerLine, splitOptions[i].AsSpan(), trim, tokenCounter);
+ result.AddRange(splits2);
+ result.RemoveRange(0, count); // remove the original input
+ if (!inputWasSplit2)
+ {
+ break;
+ }
+ }
+
+ return result.ToStringList();
+ }
+
+ private static (StringListWithTokenCount, bool) Split(StringListWithTokenCount input, int maxTokens, ReadOnlySpan separators, bool trim, TokenCounter? tokenCounter)
+ {
+ var inputWasSplit = false;
+ StringListWithTokenCount result = new(tokenCounter);
+ var count = input.Count;
+ for (var i = 0; i < count; i++)
+ {
+ var (splits, split) = Split(input.ValueAt(i).AsSpan(), input.ValueAt(i), maxTokens, separators, trim, tokenCounter, input.TokenCountAt(i));
+ result.AddRange(splits);
+ inputWasSplit |= split;
+ }
+
+ return (result, inputWasSplit);
+ }
+
+ private static (StringListWithTokenCount, bool) Split(ReadOnlySpan input, string? inputString, int maxTokens, ReadOnlySpan separators, bool trim, TokenCounter? tokenCounter, int inputTokenCount)
+ {
+ Debug.Assert(inputString is null || input.SequenceEqual(inputString.AsSpan()));
+ StringListWithTokenCount result = new(tokenCounter);
+ var inputWasSplit = false;
+
+ if (inputTokenCount > maxTokens)
+ {
+ inputWasSplit = true;
+
+ var half = input.Length / 2;
+ var cutPoint = -1;
+
+ if (separators.IsEmpty)
+ {
+ cutPoint = half;
+ }
+ else if (input.Length > 2)
+ {
+ var pos = 0;
+ while (true)
+ {
+ var index = input[pos..^1].IndexOfAny(separators);
+ if (index < 0)
+ {
+ break;
+ }
+
+ index += pos;
+
+ if (Math.Abs(half - index) < Math.Abs(half - cutPoint))
+ {
+ cutPoint = index + 1;
+ }
+
+ pos = index + 1;
+ }
+ }
+
+ if (cutPoint > 0)
+ {
+ var firstHalf = input[..cutPoint];
+ var secondHalf = input[cutPoint..];
+ if (trim)
+ {
+ firstHalf = firstHalf.Trim();
+ secondHalf = secondHalf.Trim();
+ }
+
+ // Recursion
+ var (splits1, split1) = Split(firstHalf, null, maxTokens, separators, trim, tokenCounter, GetTokenCount(firstHalf, tokenCounter));
+ result.AddRange(splits1);
+ var (splits2, split2) = Split(secondHalf, null, maxTokens, separators, trim, tokenCounter, GetTokenCount(secondHalf, tokenCounter));
+ result.AddRange(splits2);
+
+ inputWasSplit = split1 || split2;
+ return (result, inputWasSplit);
+ }
+ }
+
+ var resultString = inputString ?? input.ToString();
+ var resultTokenCount = inputTokenCount;
+ if (trim)
+ {
+ var trimmedResult = resultString.Trim();
+ if (!trimmedResult.Equals(resultString, StringComparison.Ordinal))
+ {
+ resultString = trimmedResult;
+ resultTokenCount = GetTokenCount(resultString, tokenCounter);
+ }
+ }
+
+ result.Add(resultString, resultTokenCount);
+
+ return (result, inputWasSplit);
+ }
+
+ private static int GetTokenCount(string input, TokenCounter? tokenCounter) => tokenCounter is null ? GetDefaultTokenCount(input.Length) : tokenCounter(input);
+
+ private static int GetTokenCount(ReadOnlySpan input, TokenCounter? tokenCounter) => tokenCounter is null ? GetDefaultTokenCount(input.Length) : tokenCounter(input.ToString());
+
+ private static string NormalizeLineEndings(string text) => text.Replace("\r\n", "\n").Replace('\r', '\n');
+
+ private static void ValidateMaxTokens(int maxTokens, string parameterName)
+ {
+ if (maxTokens <= 0)
+ {
+ throw new ArgumentOutOfRangeException(parameterName, "The maximum token count must be a positive number.");
+ }
+ }
+
+ private static int GetDefaultTokenCount(int length)
+ {
+ Debug.Assert(length >= 0);
+ return length == 0 ? 0 : Math.Max(1, length >> 2);
+ }
+}
\ No newline at end of file
diff --git a/SqlDatabaseVectorSearch/TextChunkers/MarkdownTextChunker.cs b/SqlDatabaseVectorSearch/TextChunkers/MarkdownTextChunker.cs
index cba6679..ebea231 100644
--- a/SqlDatabaseVectorSearch/TextChunkers/MarkdownTextChunker.cs
+++ b/SqlDatabaseVectorSearch/TextChunkers/MarkdownTextChunker.cs
@@ -1,7 +1,7 @@
using Microsoft.Extensions.Options;
-using Microsoft.SemanticKernel.Text;
using SqlDatabaseVectorSearch.Services;
using SqlDatabaseVectorSearch.Settings;
+using SqlDatabaseVectorSearch.TextChunkers.Implementations;
namespace SqlDatabaseVectorSearch.TextChunkers;
@@ -11,8 +11,8 @@ public class MarkdownTextChunker(TokenizerService tokenizerService, IOptions Split(string text)
{
- var lines = TextChunker.SplitMarkDownLines(text, appSettings.MaxTokensPerLine, tokenizerService.CountEmbeddingTokens);
- var paragraphs = TextChunker.SplitMarkdownParagraphs(lines, appSettings.MaxTokensPerParagraph, appSettings.OverlapTokens, tokenCounter: tokenizerService.CountEmbeddingTokens);
+ var lines = PlainTextChunker.SplitMarkdownLines(text, appSettings.MaxTokensPerLine, tokenizerService.CountEmbeddingTokens);
+ var paragraphs = PlainTextChunker.SplitMarkdownParagraphs(lines, appSettings.MaxTokensPerParagraph, appSettings.OverlapTokens, tokenCounter: tokenizerService.CountEmbeddingTokens);
return paragraphs;
}
diff --git a/SqlDatabaseVectorSearch/Workflows/EmbeddingRequest.cs b/SqlDatabaseVectorSearch/Workflows/EmbeddingRequest.cs
new file mode 100644
index 0000000..9837162
--- /dev/null
+++ b/SqlDatabaseVectorSearch/Workflows/EmbeddingRequest.cs
@@ -0,0 +1,26 @@
+namespace SqlDatabaseVectorSearch.Workflows;
+
+public record class EmbeddingRequest(Stream Content, string FileName, string ContentType, Guid? DocumentId)
+{
+ ///
+ /// Creates an from an uploaded .
+ ///
+ /// The uploaded file.
+ /// The optional identifier of the document to overwrite.
+ public static EmbeddingRequest FromFormFile(IFormFile file, Guid? documentId = null) => Create(file.OpenReadStream(), Path.GetFileName(file.FileName), documentId);
+
+ ///
+ /// Creates an from a content stream, inferring the content type from the file name.
+ ///
+ /// The stream that contains the document content.
+ /// The name of the document.
+ /// The optional identifier of the document to overwrite.
+ ///
+ /// The content type is inferred from the file name because the content type declared by the client is not always reliable (for example, for Markdown files).
+ ///
+ public static EmbeddingRequest Create(Stream content, string fileName, Guid? documentId = null)
+ {
+ var name = Path.GetFileName(fileName);
+ return new EmbeddingRequest(content, name, MimeMapping.MimeUtility.GetMimeMapping(name), documentId);
+ }
+}
\ No newline at end of file
diff --git a/SqlDatabaseVectorSearch/Workflows/ExtractChunksExecutor.cs b/SqlDatabaseVectorSearch/Workflows/ExtractChunksExecutor.cs
new file mode 100644
index 0000000..a43ea19
--- /dev/null
+++ b/SqlDatabaseVectorSearch/Workflows/ExtractChunksExecutor.cs
@@ -0,0 +1,21 @@
+using Microsoft.Agents.AI.Workflows;
+using SqlDatabaseVectorSearch.ContentDecoders;
+
+namespace SqlDatabaseVectorSearch.Workflows;
+
+public partial class ExtractChunksExecutor(IServiceProvider serviceProvider, ILogger logger) : Executor(nameof(ExtractChunksExecutor))
+{
+ [MessageHandler]
+ private async ValueTask HandleAsync(EmbeddingRequest request, IWorkflowContext context, CancellationToken cancellationToken)
+ {
+ // Extract the contents of the file.
+ var decoder = serviceProvider.GetKeyedService(request.ContentType) ?? throw new NotSupportedException($"Content type '{request.ContentType}' is not supported.");
+ var chunks = await decoder.DecodeAsync(request.Content, request.ContentType, cancellationToken);
+
+ logger.LogDebug("Extracted {Count} chunks from '{FileName}'.", chunks.Count(), request.FileName);
+
+ return new(request, chunks);
+ }
+}
+
+public record class ExtractChunksResponse(EmbeddingRequest Request, IEnumerable Chunks);
diff --git a/SqlDatabaseVectorSearch/Workflows/GenerateEmbeddingExecutor.cs b/SqlDatabaseVectorSearch/Workflows/GenerateEmbeddingExecutor.cs
new file mode 100644
index 0000000..8907c67
--- /dev/null
+++ b/SqlDatabaseVectorSearch/Workflows/GenerateEmbeddingExecutor.cs
@@ -0,0 +1,37 @@
+using Microsoft.Agents.AI.Workflows;
+using Microsoft.Extensions.AI;
+using Microsoft.Extensions.Options;
+using SqlDatabaseVectorSearch.ContentDecoders;
+using SqlDatabaseVectorSearch.Services;
+using SqlDatabaseVectorSearch.Settings;
+
+namespace SqlDatabaseVectorSearch.Workflows;
+
+public partial class GenerateEmbeddingExecutor(IEmbeddingGenerator> embeddingGenerator, TokenizerService tokenizerService, IOptions appSettingsOptions, ILogger logger) : Executor(nameof(GenerateEmbeddingExecutor))
+{
+ private readonly AppSettings appSettings = appSettingsOptions.Value;
+
+ [MessageHandler]
+ private async ValueTask HandleAsync(ExtractChunksResponse chunks, IWorkflowContext context, CancellationToken cancellationToken)
+ {
+ var chunkContents = chunks.Chunks.Select(p => p.Content).ToList();
+
+ // We get the token count of the whole document because it is the total number of tokens used by the embedding (it may be necessary, for example, for cost analysis).
+ var tokenCount = tokenizerService.CountEmbeddingTokens(string.Join(" ", chunkContents));
+
+ // Process paragraphs in batches.
+ var embeddings = new List>();
+ foreach (var batch in chunkContents.Chunk(appSettings.EmbeddingBatchSize))
+ {
+ logger.LogDebug("Processing batch of {Count} chunks for embedding generation...", batch.Length);
+
+ // Generate embeddings for this batch.
+ var batchEmbeddings = await embeddingGenerator.GenerateAsync(batch, cancellationToken: cancellationToken);
+ embeddings.AddRange(batchEmbeddings);
+ }
+
+ return new EmbeddingResponse(chunks.Request, chunks.Chunks, embeddings, tokenCount);
+ }
+}
+
+public record class EmbeddingResponse(EmbeddingRequest Request, IEnumerable Chunks, IEnumerable> Embeddings, int TokenCount);
\ No newline at end of file
diff --git a/SqlDatabaseVectorSearch/Workflows/StoreEmbeddingExecutor.cs b/SqlDatabaseVectorSearch/Workflows/StoreEmbeddingExecutor.cs
new file mode 100644
index 0000000..aaec48d
--- /dev/null
+++ b/SqlDatabaseVectorSearch/Workflows/StoreEmbeddingExecutor.cs
@@ -0,0 +1,58 @@
+using Microsoft.Agents.AI.Workflows;
+using Microsoft.Data.SqlTypes;
+using Microsoft.EntityFrameworkCore;
+using SqlDatabaseVectorSearch.Data;
+using SqlDatabaseVectorSearch.Services;
+using Entities = SqlDatabaseVectorSearch.Data.Entities;
+
+namespace SqlDatabaseVectorSearch.Workflows;
+
+public partial class StoreEmbeddingExecutor(ApplicationDbContext dbContext, DocumentService documentService, TokenizerService tokenizerService, TimeProvider timeProvider, ILogger logger) : Executor(nameof(StoreEmbeddingExecutor))
+{
+ [MessageHandler]
+ private async ValueTask HandleAsync(EmbeddingResponse embeddingData, IWorkflowContext context, CancellationToken cancellationToken)
+ {
+ var strategy = dbContext.Database.CreateExecutionStrategy();
+ var document = await strategy.ExecuteAsync(async (cancellationToken) =>
+ {
+ await dbContext.Database.BeginTransactionAsync(cancellationToken);
+
+ if (embeddingData.Request.DocumentId.HasValue)
+ {
+ // If the user is importing a document that already exists, delete the previous one.
+ await documentService.DeleteAsync(embeddingData.Request.DocumentId.Value, cancellationToken);
+ }
+
+ var document = new Entities.Document { Id = embeddingData.Request.DocumentId.GetValueOrDefault(), Name = embeddingData.Request.FileName, CreationDate = timeProvider.GetUtcNow() };
+ dbContext.Documents.Add(document);
+
+ // Save the document chunks and the corresponding embedding in the database.
+ foreach (var (index, embedding) in embeddingData.Embeddings.Index())
+ {
+ var chunk = embeddingData.Chunks.ElementAt(index);
+ logger.LogDebug("Storing a chunk of {TokenCount} tokens.", tokenizerService.CountEmbeddingTokens(chunk.Content));
+
+ var documentChunk = new Entities.DocumentChunk
+ {
+ Document = document,
+ Index = index,
+ PageNumber = chunk.PageNumber,
+ IndexOnPage = chunk.IndexOnPage,
+ Content = chunk.Content,
+ Embedding = new SqlVector(embedding.Vector)
+ };
+
+ dbContext.DocumentChunks.Add(documentChunk);
+ }
+
+ await dbContext.SaveChangesAsync(cancellationToken);
+ await dbContext.Database.CommitTransactionAsync(cancellationToken);
+
+ return document;
+ }, cancellationToken);
+
+ return new(document.Id, embeddingData.TokenCount);
+ }
+}
+
+public record class StoreEmbeddingResponse(Guid DocumentId, int TokenCount);
\ No newline at end of file
diff --git a/SqlDatabaseVectorSearch/appsettings.json b/SqlDatabaseVectorSearch/appsettings.json
index 3c23b98..37d6be3 100644
--- a/SqlDatabaseVectorSearch/appsettings.json
+++ b/SqlDatabaseVectorSearch/appsettings.json
@@ -6,7 +6,7 @@
"ChatCompletion": {
"Endpoint": "",
"Deployment": "",
- "ModelId": "", // gpt-4o, gpt-4, gpt-3.5, etc. Note that for gpt-4.1 and gpt-5 models, the ModelId must be set to gpt-4o.
+ "ModelId": "", // Used only for token counting: gpt-5, gpt-4.1, gpt-4o, gpt-4, gpt-3.5-turbo, etc. If the model isn't supported by Microsoft.ML.Tokenizers, use the closest one (for example gpt-4o).
"ApiKey": ""
},
"Embedding": {
@@ -24,9 +24,7 @@
"MaxTokensPerLine": 300,
"MaxTokensPerParagraph": 1000,
"OverlapTokens": 100,
- "MaxRelevantChunks": 50,
- "MaxInputTokens": 32768,
- "MaxOutputTokens": 800,
+ "MaxRelevantChunks": 30,
"MessageExpiration": "00:05:00",
"MessageLimit": 20
},
diff --git a/SqlDatabaseVectorSearch/wwwroot/css/app.css b/SqlDatabaseVectorSearch/wwwroot/css/app.css
index 3c0b751..2fe472f 100644
--- a/SqlDatabaseVectorSearch/wwwroot/css/app.css
+++ b/SqlDatabaseVectorSearch/wwwroot/css/app.css
@@ -7,34 +7,65 @@ body, html {
:root {
--bb-sidebar2-width: 270px;
--bb-sidebar2-collapsed-width: 50px;
- --bb-sidebar2-background-color: rgba(234, 234, 234, 1);
- --bb-sidebar2-top-row-background-color: rgba(0,0,0,0.08);
- --bb-sidebar2-top-row-border-color: rgb(194,192,192);
- --bb-sidebar2-title-text-color: rgb(0,0,0);
- --bb-sidebar2-brand-icon-color: rgb(0,0,0);
+ --bb-sidebar2-background-color: transparent;
+ --bb-sidebar2-top-row-background-color: transparent;
+ --bb-sidebar2-top-row-border-color: rgba(13,110,253,0.12);
+ --bb-sidebar2-title-text-color: #1b2436;
+ --bb-sidebar2-brand-icon-color: #0d6efd;
--bb-sidebar2-brand-image-width: 24px;
--bb-sidebar2-brand-image-height: 24px;
--bb-sidebar2-title-badge-text-color: rgb(255,255,255);
--bb-sidebar2-title-badge-background-color: rgba(25,135,84,var(--bs-bg-opacity,1));
- --bb-sidebar2-navbar-toggler-icon-color: rgb(0,0,0);
- --bb-sidebar2-navbar-toggler-background-color: rgba(0,0,0,0.08);
- --bb-sidebar2-content-border-color: rgb(194,192,192);
- --bb-sidebar2-nav-item-text-color: rgba(0,0,0,0.9);
- --bb-sidebar2-nav-item-text-active-color-rgb: 0,0,0;
- --bb-sidebar2-nav-item-text-hover-color: rgba(var(--bb-sidebar-nav-item-text-active-color-rgb),0.9);
- --bb-sidebar2-nav-item-text-active-color: rgba(var(--bb-sidebar-nav-item-text-active-color-rgb),0.9);
- --bb-sidebar2-nav-item-background-hover-color: rgba(var(--bb-sidebar-nav-item-text-active-color-rgb),0.08);
- --bb-sidebar2-nav-item-group-background-color: rgba(var(--bb-sidebar-nav-item-text-active-color-rgb),0.08);
+ --bb-sidebar2-navbar-toggler-icon-color: #1b2436;
+ --bb-sidebar2-navbar-toggler-background-color: rgba(13,110,253,0.08);
+ --bb-sidebar2-content-border-color: rgba(13,110,253,0.12);
+ --bb-sidebar2-nav-item-text-color: #495867;
+ --bb-sidebar2-nav-item-text-active-color-rgb: 27,36,54;
+ --bb-sidebar2-nav-item-text-hover-color: #0d6efd;
+ --bb-sidebar2-nav-item-text-active-color: #0d6efd;
+ --bb-sidebar2-nav-item-background-hover-color: rgba(13,110,253,0.08);
+ --bb-sidebar2-nav-item-group-background-color: rgba(13,110,253,0.06);
}
-.bb-sidebar2 nav .nav-item a:hover {
- background-color: rgba(0,0,0,0.08) !important;
- color: rgba(0,0,0,0.9) !important;
+/* A soft light gradient gives the sidebar a modern look while keeping the content area in focus. */
+.bb-sidebar2 {
+ background: linear-gradient(180deg, #ffffff 0%, #eef2f8 100%);
+ border-right: 1px solid rgba(13,110,253,0.1);
+ box-shadow: 2px 0 12px rgba(27,36,54,0.06);
}
-.bb-sidebar2 nav .nav-item a.active {
- background-color: rgb(194,192,192) !important;
- color: rgba(0,0,0,0.9) !important;
+ .bb-sidebar2 nav .nav-item {
+ margin: 0.125rem 0.5rem;
+ }
+
+ .bb-sidebar2 nav .nav-item a {
+ border-radius: 0.5rem;
+ transition: background-color 0.15s ease-in-out, color 0.15s ease-in-out;
+ }
+
+ .bb-sidebar2 nav .nav-item a:hover {
+ background-color: rgba(13,110,253,0.08) !important;
+ color: #0d6efd !important;
+ }
+
+ .bb-sidebar2 nav .nav-item a.active {
+ background: linear-gradient(90deg, rgba(13,110,253,0.16) 0%, rgba(13,110,253,0.04) 100%) !important;
+ color: #0d6efd !important;
+ font-weight: 600;
+ box-shadow: inset 3px 0 0 #0d6efd;
+ }
+
+/* The nav item icons inherit the link color, so they are colored individually. */
+.bb-sidebar2 nav .nav-item a .bi-house-door-fill {
+ color: #0d6efd;
+}
+
+.bb-sidebar2 nav .nav-item a .bi-file-earmark-text-fill {
+ color: #0aa2c0;
+}
+
+.bb-sidebar2 nav .nav-item a .bi-chat-dots-fill {
+ color: #198754;
}
h1:focus {
diff --git a/assets/SqlDatabaseVectorSearch.mp4 b/assets/SqlDatabaseVectorSearch.mp4
deleted file mode 100644
index 0a48f50..0000000
Binary files a/assets/SqlDatabaseVectorSearch.mp4 and /dev/null differ
diff --git a/assets/SqlDatabaseVectorSearch_WebApp.png b/assets/SqlDatabaseVectorSearch_WebApp.png
index f3124a2..c1f1b6a 100644
Binary files a/assets/SqlDatabaseVectorSearch_WebApp.png and b/assets/SqlDatabaseVectorSearch_WebApp.png differ