Add TokenizerService and update settings for token limits

Updated Program.cs to use ConfigureAndGet for settings and added TokenizerService singleton. Modified ChatService to include TokenizerService and updated prompt logic for token limits. Added TokenizerService class using TiktokenTokenizer. Updated AppSettings and AzureOpenAISettings with new token and model settings. Updated SqlDatabaseVectorSearch.csproj with new package references and version bump. Updated appsettings.json with new settings.

SqlDatabaseVectorSearch/Program.cs

@@ -13,7 +13,7 @@ var builder = WebApplication.CreateBuilder(args);
 builder.Configuration.AddJsonFile("appsettings.local.json", optional: true, reloadOnChange: true);
 
 // Add services to the container.
-var aiSettings = builder.Configuration.GetSection<AzureOpenAISettings>("AzureOpenAI")!;
+var aiSettings = builder.Services.ConfigureAndGet<AzureOpenAISettings>(builder.Configuration, "AzureOpenAI")!;
 var appSettings = builder.Services.ConfigureAndGet<AppSettings>(builder.Configuration, nameof(AppSettings))!;
 
 builder.Services.AddSingleton(TimeProvider.System);
@@ -40,6 +40,7 @@ builder.Services.AddKernel()
     .AddAzureOpenAITextEmbeddingGeneration(aiSettings.Embedding.Deployment, aiSettings.Embedding.Endpoint, aiSettings.Embedding.ApiKey, dimensions: aiSettings.Embedding.Dimensions)
     .AddAzureOpenAIChatCompletion(aiSettings.ChatCompletion.Deployment, aiSettings.ChatCompletion.Endpoint, aiSettings.ChatCompletion.ApiKey);
 
+builder.Services.AddSingleton<TokenizerService>();
 builder.Services.AddSingleton<ChatService>();
 builder.Services.AddScoped<VectorSearchService>();
 

SqlDatabaseVectorSearch/Services/ChatService.cs

@@ -1,10 +1,13 @@
 using System.Text;
 using Microsoft.Extensions.Caching.Hybrid;
+using Microsoft.Extensions.Options;
 using Microsoft.SemanticKernel.ChatCompletion;
+using Microsoft.SemanticKernel.Connectors.AzureOpenAI;
+using SqlDatabaseVectorSearch.Settings;
 
 namespace SqlDatabaseVectorSearch.Services;
 
-public class ChatService(IChatCompletionService chatCompletionService, HybridCache cache)
+public class ChatService(IChatCompletionService chatCompletionService, TokenizerService tokenizerService, HybridCache cache, IOptions<AppSettings> appSettingsOptions)
 {
     public async Task<string> CreateQuestionAsync(Guid conversationId, string question)
     {
@@ -38,29 +41,47 @@ public class ChatService(IChatCompletionService chatCompletionService, HybridCac
             You must answer in the same language of the user's question.
             """);
 
-        var prompt = new StringBuilder("""
+        var prompt = new StringBuilder($"""
+            Answer the following question:
+            ---
+            {question}
+            =====          
             Using the following information:
 
             """);
 
-        // TODO: Ensure that chunks are not too long, according to the model max token.
+        var tokensAvailable = appSettings.MaxInputTokens
-        foreach (var text in chunks)
+                              - tokenizerService.CountTokens(chat[0].ToString())    // System prompt.
+                              - tokenizerService.CountTokens(prompt.ToString()) // Initial user prompt.
+                              - appSettings.MaxOutputTokens;    // To ensure there is enough space for the answer.
+
+        foreach (var chunk in chunks)
         {
-            prompt.AppendLine("---");
+            var text = $"---{Environment.NewLine}{chunk}";
+
+            var tokenCount = tokenizerService.CountTokens(text);
+            if (tokenCount > tokensAvailable)
+            {
+                // There isn't enough space to add the text.
+                break;
+            }
+
             prompt.Append(text);
+
+            tokensAvailable -= tokenCount;
+            if (tokensAvailable <= 0)
+            {
+                // There isn't enough space to add more chunks.
+                break;
+            }
         }
 
-        prompt.AppendLine($"""
-
-            =====
-            Answer the following question:
-            ---
-            {question}
-            """);
-
         chat.AddUserMessage(prompt.ToString());
 
-        var answer = await chatCompletionService.GetChatMessageContentAsync(chat)!;
+        var answer = await chatCompletionService.GetChatMessageContentAsync(chat, new AzureOpenAIPromptExecutionSettings
+        {
+            MaxTokens = appSettings.MaxOutputTokens
+        });
 
         // Add question and answer to the chat history.
         await SetChatHistoryAsync(conversationId, question, answer.Content!);

SqlDatabaseVectorSearch/Services/TokenizerService.cs

@@ -0,0 +1,13 @@
+using Microsoft.Extensions.Options;
+using Microsoft.ML.Tokenizers;
+using SqlDatabaseVectorSearch.Settings;
+
+namespace SqlDatabaseVectorSearch.Services;
+
+public class TokenizerService(IOptions<AzureOpenAISettings> settingsOptions)
+{
+    private readonly TiktokenTokenizer tokenizer = TiktokenTokenizer.CreateForModel(settingsOptions.Value.ChatCompletion.ModelId);
+
+    public int CountTokens(string input)
+        => tokenizer.CountTokens(input);
+}

SqlDatabaseVectorSearch/Settings/AppSettings.cs

@@ -10,5 +10,9 @@ public class AppSettings
 
     public int MaxRelevantChunks { get; init; } = 5;
 
+    public int MaxInputTokens { get; init; } = 16385;
+
+    public int MaxOutputTokens { get; init; } = 800;
+
     public TimeSpan MessageExpiration { get; init; }
 }

SqlDatabaseVectorSearch/Settings/AzureOpenAISettings.cs

@@ -4,7 +4,7 @@ public class AzureOpenAISettings
 {
     public required ServiceSettings ChatCompletion { get; init; }
 
-    public required EmbeddingServiceSettings Embedding { get; init; }
+    public required EmbeddingSettings Embedding { get; init; }
 }
 
 public class ServiceSettings
@@ -13,10 +13,12 @@ public class ServiceSettings
 
     public required string Deployment { get; init; }
 
+    public required string ModelId { get; init; }
+
     public required string ApiKey { get; init; }
 }
 
-public class EmbeddingServiceSettings : ServiceSettings
+public class EmbeddingSettings : ServiceSettings
 {
     public int? Dimensions { get; set; }
 }

SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj

@@ -13,11 +13,14 @@
         <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="9.0.0" />
         <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="[8.0.11,9.0.0)" />
         <PackageReference Include="Microsoft.Extensions.Caching.Hybrid" Version="9.0.0-preview.9.24556.5" />
-        <PackageReference Include="Microsoft.SemanticKernel" Version="1.32.0" />
+        <PackageReference Include="Microsoft.ML.Tokenizers" Version="1.0.0" />
-		<PackageReference Include="MinimalHelpers.OpenApi" Version="2.1.2" />
+        <PackageReference Include="Microsoft.ML.Tokenizers.Data.Cl100kBase" Version="1.0.0" />
-		<PackageReference Include="PdfPig" Version="0.1.9" />
+        <PackageReference Include="Microsoft.ML.Tokenizers.Data.O200kBase" Version="1.0.0" />
-		<PackageReference Include="Swashbuckle.AspNetCore.SwaggerUI" Version="7.2.0" />
+        <PackageReference Include="Microsoft.SemanticKernel" Version="1.33.0" />
-		<PackageReference Include="TinyHelpers.AspNetCore" Version="4.0.6" />
+        <PackageReference Include="MinimalHelpers.OpenApi" Version="2.1.2" />
+        <PackageReference Include="PdfPig" Version="0.1.9" />
+        <PackageReference Include="Swashbuckle.AspNetCore.SwaggerUI" Version="7.2.0" />
+        <PackageReference Include="TinyHelpers.AspNetCore" Version="4.0.6" />
 	</ItemGroup>
 
 </Project>

SqlDatabaseVectorSearch/appsettings.json

@@ -6,12 +6,14 @@
         "ChatCompletion": {
             "Endpoint": "",
             "Deployment": "",
-            "ApiKey": ""
+            "ApiKey": "",
+            "ModelId": "" // o1, gpt-4o, gpt-4, gpt-3.5 
         },
         "Embedding": {
             "Endpoint": "",
             "Deployment": "",
             "ApiKey": "",
+            "ModelId": "",
             // Set this value only if you're using a model that allows to specify the dimensions of the embeddings
             // (e.g. text-embedding-3-small or text-embedding-3-large). Currently, a maximum value of 1998 is supported.
             "Dimensions": null
@@ -22,6 +24,8 @@
         "MaxTokensPerParagraph": 1024,
         "OverlapTokens": 100,
         "MaxRelevantChunks": 10,
+        "MaxInputTokens": 16385,
+        "MaxOutputTokens": 800,
         "MessageExpiration": "00:05:00"
     },
     "Logging": {