diff --git a/SqlDatabaseVectorSearch/Services/ChatService.cs b/SqlDatabaseVectorSearch/Services/ChatService.cs index ba1bee6..d75db78 100644 --- a/SqlDatabaseVectorSearch/Services/ChatService.cs +++ b/SqlDatabaseVectorSearch/Services/ChatService.cs @@ -100,7 +100,7 @@ public class ChatService(IChatCompletionService chatCompletionService, Tokenizer var tokenCount = tokenizerService.CountTokens(text); if (tokenCount > tokensAvailable) { - // There isn't enough space to add the chunks. + // There isn't enough space to add the current chunk. break; } diff --git a/SqlDatabaseVectorSearch/Services/VectorSearchService.cs b/SqlDatabaseVectorSearch/Services/VectorSearchService.cs index 62493bc..362762c 100644 --- a/SqlDatabaseVectorSearch/Services/VectorSearchService.cs +++ b/SqlDatabaseVectorSearch/Services/VectorSearchService.cs @@ -13,7 +13,7 @@ using Entities = SqlDatabaseVectorSearch.DataAccessLayer.Entities; namespace SqlDatabaseVectorSearch.Services; -public class VectorSearchService(ApplicationDbContext dbContext, ITextEmbeddingGenerationService textEmbeddingGenerationService, ChatService chatService, TimeProvider timeProvider, IOptions appSettingsOptions) +public class VectorSearchService(ApplicationDbContext dbContext, ITextEmbeddingGenerationService textEmbeddingGenerationService, ChatService chatService, TokenizerService tokenizerService, TimeProvider timeProvider, IOptions appSettingsOptions, ILogger logger) { private readonly AppSettings appSettings = appSettingsOptions.Value; @@ -34,12 +34,15 @@ public class VectorSearchService(ApplicationDbContext dbContext, ITextEmbeddingG dbContext.Documents.Add(document); // Split the content into chunks and generate the embeddings for each one. - var paragraphs = TextChunker.SplitPlainTextParagraphs(TextChunker.SplitPlainTextLines(content, appSettings.MaxTokensPerLine), appSettings.MaxTokensPerParagraph, appSettings.OverlapTokens); + var lines = TextChunker.SplitPlainTextLines(content, appSettings.MaxTokensPerLine, tokenizerService.CountTokens); + var paragraphs = TextChunker.SplitPlainTextParagraphs(lines, appSettings.MaxTokensPerParagraph, appSettings.OverlapTokens, tokenCounter: tokenizerService.CountTokens); var embeddings = await textEmbeddingGenerationService.GenerateEmbeddingsAsync(paragraphs); // Save the document chunks and the corresponding embedding in the database. foreach (var (index, paragraph) in paragraphs.Index()) { + logger.LogInformation("Storing a paragraph of {TokenCount} tokens.", tokenizerService.CountTokens(paragraph)); + var documentChunk = new Entities.DocumentChunk { Document = document, Index = index, Content = paragraph!, Embedding = embeddings[index].ToArray() }; dbContext.DocumentChunks.Add(documentChunk); } diff --git a/SqlDatabaseVectorSearch/appsettings.json b/SqlDatabaseVectorSearch/appsettings.json index 1dc56e4..4df5bad 100644 --- a/SqlDatabaseVectorSearch/appsettings.json +++ b/SqlDatabaseVectorSearch/appsettings.json @@ -6,7 +6,7 @@ "ChatCompletion": { "Endpoint": "", "Deployment": "", - "ModelId": "", // o1, gpt-4o, gpt-4, gpt-3.5 + "ModelId": "", // o1, gpt-4o, gpt-4o-mini, gpt-4, gpt-3.5 "ApiKey": "" }, "Embedding": { @@ -20,8 +20,8 @@ } }, "AppSettings": { - "MaxTokenPerLine": 300, - "MaxTokensPerParagraph": 1024, + "MaxTokensPerLine": 300, + "MaxTokensPerParagraph": 1000, "OverlapTokens": 100, "MaxRelevantChunks": 10, "MaxInputTokens": 16385,