Update VectorSearchService and appsettings.json

- Clarified comment in `ChatService.cs`.
- Added `TokenizerService` and `ILogger` parameters to `VectorSearchService` constructor.
- Updated paragraph splitting to use `tokenizerService.CountTokens`.
- Added logging for token count of each paragraph in `VectorSearchService`.
- Updated `ModelId` comment in `appsettings.json` to include "gpt-4o-mini".
- Changed `MaxTokensPerParagraph` in `appsettings.json` from 1024 to 1000.

SqlDatabaseVectorSearch/Services/ChatService.cs

@@ -100,7 +100,7 @@ public class ChatService(IChatCompletionService chatCompletionService, Tokenizer
             var tokenCount = tokenizerService.CountTokens(text);
             if (tokenCount > tokensAvailable)
             {
-                // There isn't enough space to add the chunks.
+                // There isn't enough space to add the current chunk.
                 break;
             }
 

SqlDatabaseVectorSearch/Services/VectorSearchService.cs

@@ -13,7 +13,7 @@ using Entities = SqlDatabaseVectorSearch.DataAccessLayer.Entities;
 
 namespace SqlDatabaseVectorSearch.Services;
 
-public class VectorSearchService(ApplicationDbContext dbContext, ITextEmbeddingGenerationService textEmbeddingGenerationService, ChatService chatService, TimeProvider timeProvider, IOptions<AppSettings> appSettingsOptions)
+public class VectorSearchService(ApplicationDbContext dbContext, ITextEmbeddingGenerationService textEmbeddingGenerationService, ChatService chatService, TokenizerService tokenizerService, TimeProvider timeProvider, IOptions<AppSettings> appSettingsOptions, ILogger<VectorSearchService> logger)
 {
     private readonly AppSettings appSettings = appSettingsOptions.Value;
 
@@ -34,12 +34,15 @@ public class VectorSearchService(ApplicationDbContext dbContext, ITextEmbeddingG
         dbContext.Documents.Add(document);
 
         // Split the content into chunks and generate the embeddings for each one.
-        var paragraphs = TextChunker.SplitPlainTextParagraphs(TextChunker.SplitPlainTextLines(content, appSettings.MaxTokensPerLine), appSettings.MaxTokensPerParagraph, appSettings.OverlapTokens);
+        var lines = TextChunker.SplitPlainTextLines(content, appSettings.MaxTokensPerLine, tokenizerService.CountTokens);
+        var paragraphs = TextChunker.SplitPlainTextParagraphs(lines, appSettings.MaxTokensPerParagraph, appSettings.OverlapTokens, tokenCounter: tokenizerService.CountTokens);
         var embeddings = await textEmbeddingGenerationService.GenerateEmbeddingsAsync(paragraphs);
 
         // Save the document chunks and the corresponding embedding in the database.
         foreach (var (index, paragraph) in paragraphs.Index())
         {
+            logger.LogInformation("Storing a paragraph of {TokenCount} tokens.", tokenizerService.CountTokens(paragraph));
+
             var documentChunk = new Entities.DocumentChunk { Document = document, Index = index, Content = paragraph!, Embedding = embeddings[index].ToArray() };
             dbContext.DocumentChunks.Add(documentChunk);
         }

SqlDatabaseVectorSearch/appsettings.json

@@ -6,7 +6,7 @@
         "ChatCompletion": {
             "Endpoint": "",
             "Deployment": "",
-            "ModelId": "", // o1, gpt-4o, gpt-4, gpt-3.5 
+            "ModelId": "", // o1, gpt-4o, gpt-4o-mini, gpt-4, gpt-3.5 
             "ApiKey": ""
         },
         "Embedding": {
@@ -20,8 +20,8 @@
         }
     },
     "AppSettings": {
-        "MaxTokenPerLine": 300,
-        "MaxTokensPerParagraph": 1024,
+        "MaxTokensPerLine": 300,
+        "MaxTokensPerParagraph": 1000,
         "OverlapTokens": 100,
         "MaxRelevantChunks": 10,
         "MaxInputTokens": 16385,