From 5b4303125112b9c36d78a57d1a7f07a7df721663 Mon Sep 17 00:00:00 2001 From: Marco Minerva Date: Wed, 6 Nov 2024 17:20:05 +0100 Subject: [PATCH] Add TokenizerService and update settings configuration Updated Program.cs to use ConfigureAndGet method for settings, changed ChatService to singleton, and added TokenizerService singleton. Modified ChatService to use TokenizerService for token counting. Updated AppSettings and AzureOpenAISettings with new properties. Added new package references in SqlDatabaseVectorSearch.csproj. Updated appsettings.json with new properties. Added TokenizerService class for token counting. --- SqlDatabaseVectorSearch/Program.cs | 5 +- .../Services/ChatService.cs | 53 +++++++++++++------ .../Services/TokenizerService.cs | 13 +++++ .../Settings/AppSettings.cs | 4 ++ .../Settings/AzureOpenAISettings.cs | 8 +-- .../SqlDatabaseVectorSearch.csproj | 4 +- SqlDatabaseVectorSearch/appsettings.json | 6 ++- 7 files changed, 69 insertions(+), 24 deletions(-) create mode 100644 SqlDatabaseVectorSearch/Services/TokenizerService.cs diff --git a/SqlDatabaseVectorSearch/Program.cs b/SqlDatabaseVectorSearch/Program.cs index a798f73..6381ef6 100644 --- a/SqlDatabaseVectorSearch/Program.cs +++ b/SqlDatabaseVectorSearch/Program.cs @@ -14,7 +14,7 @@ var builder = WebApplication.CreateBuilder(args); builder.Configuration.AddJsonFile("appsettings.local.json", optional: true, reloadOnChange: true); // Add services to the container. -var aiSettings = builder.Configuration.GetSection("AzureOpenAI")!; +var aiSettings = builder.Services.ConfigureAndGet(builder.Configuration, "AzureOpenAI")!; var appSettings = builder.Services.ConfigureAndGet(builder.Configuration, nameof(AppSettings))!; builder.Services.AddSingleton(TimeProvider.System); @@ -35,7 +35,8 @@ builder.Services.AddKernel() .AddAzureOpenAITextEmbeddingGeneration(aiSettings.Embedding.Deployment, aiSettings.Embedding.Endpoint, aiSettings.Embedding.ApiKey, dimensions: aiSettings.Embedding.Dimensions) .AddAzureOpenAIChatCompletion(aiSettings.ChatCompletion.Deployment, aiSettings.ChatCompletion.Endpoint, aiSettings.ChatCompletion.ApiKey); -builder.Services.AddScoped(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddScoped(); builder.Services.AddEndpointsApiExplorer(); diff --git a/SqlDatabaseVectorSearch/Services/ChatService.cs b/SqlDatabaseVectorSearch/Services/ChatService.cs index ecb9ec6..80fb965 100644 --- a/SqlDatabaseVectorSearch/Services/ChatService.cs +++ b/SqlDatabaseVectorSearch/Services/ChatService.cs @@ -2,11 +2,12 @@ using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Options; using Microsoft.SemanticKernel.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.AzureOpenAI; using SqlDatabaseVectorSearch.Settings; namespace SqlDatabaseVectorSearch.Services; -public class ChatService(IMemoryCache cache, IChatCompletionService chatCompletionService, IOptions appSettingsOptions) +public class ChatService(IMemoryCache cache, IChatCompletionService chatCompletionService, TokenizerService tokenizerService, IOptions appSettingsOptions) { private readonly AppSettings appSettings = appSettingsOptions.Value; @@ -35,36 +36,54 @@ public class ChatService(IMemoryCache cache, IChatCompletionService chatCompleti public async Task AskQuestionAsync(Guid conversationId, IEnumerable chunks, string question) { - var chat = new ChatHistory("""" - """ + var chat = new ChatHistory(""" You can use only the information provided in this chat to answer questions. If you don't know the answer, reply suggesting to refine the question. For example, if the user asks "What is the capital of France?" and in this chat there isn't information about France, you should reply something like "This information isn't available in the given context". Never answer to questions that are not related to this chat. You must answer in the same language of the user's question. - """"); + """); - var prompt = new StringBuilder(""" + var prompt = new StringBuilder($""" + Answer the following question: + --- + {question} + --- Using the following information: --- """); - // TODO: Ensure that chunks are not too long, according to the model max token. - foreach (var result in chunks) - { - prompt.AppendLine(result); - prompt.AppendLine("---"); - } + var tokensAvailable = appSettings.MaxInputTokens + - tokenizerService.CountTokens(chat[0].ToString()) - tokenizerService.CountTokens(prompt.ToString()) + - appSettings.MaxOutputTokens; // To ensure there is enough space for the answer. - prompt.AppendLine($""" - Answer the following question: - --- - {question} - """); + foreach (var chunk in chunks) + { + var text = $"{chunk}---"; + + var tokenCount = tokenizerService.CountTokens(text); + if (tokenCount > tokensAvailable) + { + // There isn't enough space to add the text. + break; + } + + prompt.AppendLine(text); + + tokensAvailable -= tokenCount; + if (tokensAvailable <= 0) + { + // There isn't enough space to add more chunks. + break; + } + } chat.AddUserMessage(prompt.ToString()); - var answer = await chatCompletionService.GetChatMessageContentAsync(chat)!; + var answer = await chatCompletionService.GetChatMessageContentAsync(chat, new AzureOpenAIPromptExecutionSettings + { + MaxTokens = appSettings.MaxOutputTokens + }); // Add question and answer to the chat history. var history = new ChatHistory(cache.Get(conversationId) ?? []); diff --git a/SqlDatabaseVectorSearch/Services/TokenizerService.cs b/SqlDatabaseVectorSearch/Services/TokenizerService.cs new file mode 100644 index 0000000..9115b3d --- /dev/null +++ b/SqlDatabaseVectorSearch/Services/TokenizerService.cs @@ -0,0 +1,13 @@ +using Microsoft.Extensions.Options; +using Microsoft.ML.Tokenizers; +using SqlDatabaseVectorSearch.Settings; + +namespace SqlDatabaseVectorSearch.Services; + +public class TokenizerService(IOptions settingsOptions) +{ + private readonly TiktokenTokenizer tokenizer = TiktokenTokenizer.CreateForModel(settingsOptions.Value.ChatCompletion.ModelId); + + public int CountTokens(string input) + => tokenizer.CountTokens(input); +} diff --git a/SqlDatabaseVectorSearch/Settings/AppSettings.cs b/SqlDatabaseVectorSearch/Settings/AppSettings.cs index 4466e1a..56300a0 100644 --- a/SqlDatabaseVectorSearch/Settings/AppSettings.cs +++ b/SqlDatabaseVectorSearch/Settings/AppSettings.cs @@ -10,6 +10,10 @@ public class AppSettings public int MaxRelevantChunks { get; init; } = 5; + public int MaxInputTokens { get; init; } = 16385; + + public int MaxOutputTokens { get; init; } = 800; + public int MessageLimit { get; init; } public TimeSpan MessageExpiration { get; init; } diff --git a/SqlDatabaseVectorSearch/Settings/AzureOpenAISettings.cs b/SqlDatabaseVectorSearch/Settings/AzureOpenAISettings.cs index e85d51d..2faac44 100644 --- a/SqlDatabaseVectorSearch/Settings/AzureOpenAISettings.cs +++ b/SqlDatabaseVectorSearch/Settings/AzureOpenAISettings.cs @@ -4,7 +4,7 @@ public class AzureOpenAISettings { public required ServiceSettings ChatCompletion { get; init; } - public required EmbeddingServiceSettings Embedding { get; init; } + public required EmbeddingSettings Embedding { get; init; } } public class ServiceSettings @@ -13,10 +13,12 @@ public class ServiceSettings public required string Deployment { get; init; } + public required string ModelId { get; init; } + public required string ApiKey { get; init; } } -public class EmbeddingServiceSettings : ServiceSettings +public class EmbeddingSettings : ServiceSettings { public int? Dimensions { get; set; } -} +} \ No newline at end of file diff --git a/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj b/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj index 7fc93f7..9773a95 100644 --- a/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj +++ b/SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj @@ -12,7 +12,9 @@ - + + + diff --git a/SqlDatabaseVectorSearch/appsettings.json b/SqlDatabaseVectorSearch/appsettings.json index 79648fe..d3834ee 100644 --- a/SqlDatabaseVectorSearch/appsettings.json +++ b/SqlDatabaseVectorSearch/appsettings.json @@ -6,12 +6,14 @@ "ChatCompletion": { "Endpoint": "", "Deployment": "", - "ApiKey": "" + "ApiKey": "", + "ModelId": "" }, "Embedding": { "Endpoint": "", "Deployment": "", "ApiKey": "", + "ModelId": "", // Set this value only if you're using a model that allows to specify the dimensions of the embeddings // (e.g. text-embedding-3-small or text-embedding-3-large). Currently, a maximum value of 1998 is supported. "Dimensions": null @@ -22,6 +24,8 @@ "MaxTokensPerParagraph": 1024, "OverlapTokens": 100, "MaxRelevantChunks": 10, + "MaxInputTokens": 16385, + "MaxOutputTokens": 800, "MessageLimit": 20, "MessageExpiration": "00:05:00" },