Mirrors/SqlDatabaseVectorSearch
1
0
Fork 0
mirror of https://github.com/marcominerva/SqlDatabaseVectorSearch.git synced 2026-06-20 12:23:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor to use EF Core for database operations

Browse Source 
Refactored the codebase to replace raw SQL connections and Dapper with Entity Framework Core (EF Core). Modified `Program.cs` to configure EF Core services. Refactored `VectorSearchService` to use EF Core for all database operations. Updated project dependencies to remove Dapper and `Microsoft.Data.SqlClient`, and add EF Core packages. Added `ApplicationDbContext` for EF Core context and new `Document` and `DocumentChunk` classes for entity models.
This commit is contained in:
Marco Minerva
2024-10-31 15:16:38 +01:00
parent 1d8fa7db12
commit 0435f042f1
7 changed files with 121 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
SqlDatabaseVectorSearch/DataAccessLayer/ApplicationDbContext.cs
+51
View File
@@ -0,0 +1,51 @@
using EntityFramework.Exceptions.SqlServer;
using Microsoft.EntityFrameworkCore;
using SqlDatabaseVectorSearch.DataAccessLayer.Entities;
namespace SqlDatabaseVectorSearch.DataAccessLayer;
public class ApplicationDbContext(DbContextOptions<ApplicationDbContext> options) : DbContext(options)
{
public virtual DbSet<Document> Documents { get; set; }
public virtual DbSet<DocumentChunk> DocumentChunks { get; set; }
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
{
base.OnConfiguring(optionsBuilder);
optionsBuilder.UseExceptionProcessor();
//optionsBuilder.EnableSensitiveDataLogging();
}
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
modelBuilder.Entity<Document>(entity =>
{
entity.ToTable("Documents");
entity.HasKey(e => e.Id);
entity.Property(e => e.Id).ValueGeneratedOnAdd();
entity.Property(e => e.Name)
.IsRequired()
.HasMaxLength(255);
});
modelBuilder.Entity<DocumentChunk>(entity =>
{
entity.ToTable("DocumentChunks");
entity.HasKey(e => e.Id);
entity.Property(e => e.Id).ValueGeneratedOnAdd();
entity.Property(e => e.Content).IsRequired();
entity.Property(e => e.Embedding)
.IsRequired()
.HasColumnType("vector(1536)");
entity.HasOne(d => d.Document).WithMany(p => p.Chunks)
.HasForeignKey(d => d.DocumentId)
.OnDelete(DeleteBehavior.Cascade)
.HasConstraintName("FK_DocumentChunks_Documents");
});
}
}
SqlDatabaseVectorSearch/DataAccessLayer/Entities/Document.cs
+12
View File
@@ -0,0 +1,12 @@
namespace SqlDatabaseVectorSearch.DataAccessLayer.Entities;
public class Document
{
public Guid Id { get; set; }
public required string Name { get; set; }
public DateTimeOffset CreationDate { get; set; }
public virtual ICollection<DocumentChunk> Chunks { get; set; } = [];
}
SqlDatabaseVectorSearch/DataAccessLayer/Entities/DocumentChunk.cs
+16
View File
@@ -0,0 +1,16 @@
namespace SqlDatabaseVectorSearch.DataAccessLayer.Entities;
public class DocumentChunk
{
public Guid Id { get; set; }
public Guid DocumentId { get; set; }
public int Index { get; set; }
public required string Content { get; set; }
public required float[] Embedding { get; set; }
public virtual Document Document { get; set; } = null!;
}
SqlDatabaseVectorSearch/Models/DocumentChunk.cs
+2 -13
View File
@@ -1,14 +1,3 @@
using System.Text.Json;
namespace SqlDatabaseVectorSearch.Models;
namespace SqlDatabaseVectorSearch.Models;
public record class DocumentChunk(Guid Id, int Index, string Content, float[]? Embedding)
{
public DocumentChunk(Guid Id, int Index, string Content) : this(Id, Index, Content, (float[]?)null)
{
}
public DocumentChunk(Guid Id, int Index, string Content, string Embedding) : this(Id, Index, Content, JsonSerializer.Deserialize<float[]?>(Embedding))
{
}
}
public record class DocumentChunk(Guid Id, int Index, string Content, float[]? Embedding = null);
SqlDatabaseVectorSearch/Program.cs
+7 -4
View File
@@ -1,8 +1,9 @@
using Microsoft.AspNetCore.Http.HttpResults;
using Microsoft.Data.SqlClient;
using Microsoft.EntityFrameworkCore;
using Microsoft.OpenApi.Models;
using Microsoft.SemanticKernel;
using MinimalHelpers.OpenApi;
using SqlDatabaseVectorSearch.DataAccessLayer;
using SqlDatabaseVectorSearch.Models;
using SqlDatabaseVectorSearch.Services;
using SqlDatabaseVectorSearch.Settings;
@@ -18,10 +19,12 @@ var appSettings = builder.Services.ConfigureAndGet<AppSettings>(builder.Configur
builder.Services.AddSingleton(TimeProvider.System);
builder.Services.AddScoped(_ =>
builder.Services.AddSqlServer<ApplicationDbContext>(builder.Configuration.GetConnectionString("SqlConnection"), options =>
{
var sqlConnection = new SqlConnection(builder.Configuration.GetConnectionString("SqlConnection"));
return sqlConnection;
options.UseVectorSearch();
}, options =>
{
options.UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking);
});
builder.Services.AddMemoryCache();
SqlDatabaseVectorSearch/Services/VectorSearchService.cs
+29 -44
View File
@@ -1,21 +1,20 @@
using System.Data;
using System.Data.Common;
using System.Text;
using System.Text.Json;
using Dapper;
using Microsoft.Data.SqlClient;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
using Microsoft.SemanticKernel.Embeddings;
using Microsoft.SemanticKernel.Text;
using SqlDatabaseVectorSearch.DataAccessLayer;
using SqlDatabaseVectorSearch.Models;
using SqlDatabaseVectorSearch.Settings;
using TinyHelpers.Extensions;
using UglyToad.PdfPig;
using UglyToad.PdfPig.DocumentLayoutAnalysis.TextExtractor;
using Entities = SqlDatabaseVectorSearch.DataAccessLayer.Entities;
namespace SqlDatabaseVectorSearch.Services;
public class VectorSearchService(SqlConnection sqlConnection, ITextEmbeddingGenerationService textEmbeddingGenerationService, ChatService chatService, TimeProvider timeProvider, IOptions<AppSettings> appSettingsOptions)
public class VectorSearchService(ApplicationDbContext dbContext, ITextEmbeddingGenerationService textEmbeddingGenerationService, ChatService chatService, TimeProvider timeProvider, IOptions<AppSettings> appSettingsOptions)
{
private readonly AppSettings appSettings = appSettingsOptions.Value;
@@ -24,21 +23,16 @@ public class VectorSearchService(SqlConnection sqlConnection, ITextEmbeddingGene
// Extract the contents of the file (currently, only PDF files are supported).
var content = await GetContentAsync(stream);
await sqlConnection.OpenAsync();
await using var transaction = await sqlConnection.BeginTransactionAsync();
await dbContext.Database.BeginTransactionAsync();
if (documentId.HasValue)
{
// If the user is importing a document that already exists, delete the previous one.
await DeleteDocumentAsync(documentId.Value, transaction);
await DeleteDocumentAsync(documentId.Value);
}
documentId = await sqlConnection.ExecuteScalarAsync<Guid>($"""
INSERT INTO Documents (Id, [Name], CreationDate)
OUTPUT INSERTED.Id
VALUES (@Id, @Name, @CreationDate);
""", new { Id = documentId.GetValueOrDefault(Guid.NewGuid()), Name = name, CreationDate = timeProvider.GetUtcNow() },
transaction);
var document = new Entities.Document { Id = documentId.GetValueOrDefault(), Name = name, CreationDate = timeProvider.GetUtcNow() };
dbContext.Documents.Add(document);
// Split the content into chunks and generate the embeddings for each one.
var paragraphs = TextChunker.SplitPlainTextParagraphs(TextChunker.SplitPlainTextLines(content, appSettings.MaxTokensPerLine), appSettings.MaxTokensPerParagraph, appSettings.OverlapTokens);
@@ -47,54 +41,45 @@ public class VectorSearchService(SqlConnection sqlConnection, ITextEmbeddingGene
// Save the document chunks and the corresponding embedding in the database.
foreach (var (paragraph, index) in paragraphs.WithIndex())
{
await sqlConnection.ExecuteAsync($"""
INSERT INTO DocumentChunks (DocumentId, [Index], Content, Embedding)
VALUES (@DocumentId, @Index, @Content, CAST(@Embedding AS VECTOR({embeddings[index].Length})));
""", new { DocumentId = documentId, Index = index, Content = paragraph, Embedding = JsonSerializer.Serialize(embeddings[index]) },
transaction);
var documentChunk = new Entities.DocumentChunk { Document = document, Index = index, Content = paragraph!, Embedding = embeddings[index].ToArray() };
dbContext.DocumentChunks.Add(documentChunk);
}
await transaction.CommitAsync();
await dbContext.SaveChangesAsync();
await dbContext.Database.CommitTransactionAsync();
return documentId.Value;
return document.Id;
}
public async Task<IEnumerable<Document>> GetDocumentsAsync()
{
var documents = await sqlConnection.QueryAsync<Document>("""
SELECT Id, [Name], CreationDate, ChunkCount = (SELECT COUNT(*) FROM DocumentChunks WHERE DocumentId = Documents.Id)
FROM Documents
ORDER BY [Name];
""");
var documents = await dbContext.Documents.OrderBy(d => d.Name)
.Select(d => new Document(d.Id, d.Name, d.CreationDate, d.Chunks.Count))
.ToListAsync();
return documents;
}
public async Task<IEnumerable<DocumentChunk>> GetDocumentChunksAsync(Guid documentId)
{
var documentChunks = await sqlConnection.QueryAsync<DocumentChunk>("""
SELECT Id, [Index], Content
FROM DocumentChunks
WHERE DocumentId = @DocumentId
ORDER BY [Index];
""", new { documentId });
var documentChunks = await dbContext.DocumentChunks.Where(c => c.DocumentId == documentId).OrderBy(c => c.Index)
.Select(c => new DocumentChunk(c.Id, c.Index, c.Content, null))
.ToListAsync();
return documentChunks;
}
public async Task<DocumentChunk?> GetDocumentChunkEmbeddingAsync(Guid documentId, Guid documentChunkId)
{
var documentChunk = await sqlConnection.QueryFirstOrDefaultAsync<DocumentChunk>("""
SELECT Id, [Index], Content, CAST(Embedding AS NVARCHAR(MAX)) AS Embedding
FROM DocumentChunks
WHERE Id = @DocumentChunkId AND DocumentId = @DocumentId;
""", new { documentId, documentChunkId });
var documentChunk = await dbContext.DocumentChunks.Where(c => c.Id == documentChunkId && c.DocumentId == documentId)
.Select(c => new DocumentChunk(c.Id, c.Index, c.Content, c.Embedding))
.FirstOrDefaultAsync();
return documentChunk;
}
public Task DeleteDocumentAsync(Guid documentId, DbTransaction? transaction = null)
=> sqlConnection.ExecuteAsync("DELETE FROM Documents WHERE Id = @DocumentId", new { DocumentId = documentId }, transaction);
public Task DeleteDocumentAsync(Guid documentId)
=> dbContext.Documents.Where(d => d.Id == documentId).ExecuteDeleteAsync();
public async Task<Response> AskQuestionAsync(Question question, bool reformulate = true)
{
@@ -104,11 +89,11 @@ public class VectorSearchService(SqlConnection sqlConnection, ITextEmbeddingGene
// Perform Vector Search on SQL Database.
var questionEmbedding = await textEmbeddingGenerationService.GenerateEmbeddingAsync(reformulatedQuestion);
var chunks = await sqlConnection.QueryAsync<string>($"""
SELECT TOP (@MaxRelevantChunks) Content
FROM DocumentChunks
ORDER BY VECTOR_DISTANCE('cosine', Embedding, CAST(@QuestionEmbedding AS VECTOR({questionEmbedding.Length})));
""", new { appSettings.MaxRelevantChunks, QuestionEmbedding = JsonSerializer.Serialize(questionEmbedding) });
var chunks = await dbContext.DocumentChunks
.OrderBy(c => EF.Functions.VectorDistance("cosine", c.Embedding, questionEmbedding.ToArray()))
.Select(c => c.Content)
.Take(appSettings.MaxRelevantChunks)
.ToListAsync();
var answer = await chatService.AskQuestionAsync(question.ConversationId, chunks, reformulatedQuestion);
return new Response(reformulatedQuestion, answer);
SqlDatabaseVectorSearch/SqlDatabaseVectorSearch.csproj
+3 -2
View File
@@ -8,9 +8,10 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Dapper" Version="2.1.35" />
<PackageReference Include="EFCore.SqlServer.VectorSearch" Version="0.2.0" />
<PackageReference Include="EntityFrameworkCore.Exceptions.SqlServer" Version="8.1.3" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.10" />
<PackageReference Include="Microsoft.Data.SqlClient" Version="5.2.2" />
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="8.0.10" />
<PackageReference Include="Microsoft.SemanticKernel" Version="1.25.0" />
<PackageReference Include="MinimalHelpers.OpenApi" Version="2.0.17" />
<PackageReference Include="PdfPig" Version="0.1.9" />