Mirrors/SqlDatabaseVectorSearch
1
0
Fork 0
mirror of https://github.com/marcominerva/SqlDatabaseVectorSearch.git synced 2026-06-20 12:23:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Data access optimizations

Browse Source
This commit is contained in:
Marco Minerva
2024-09-02 10:42:30 +02:00
parent e4570008e0
commit b6a09d0926
4 changed files with 23 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Scripts.sql
+4 -9
View File
@@ -1,7 +1,7 @@
CREATE TABLE [dbo].[DocumentChunks]( CREATE TABLE [dbo].[DocumentChunks](
[Id] [uniqueidentifier] NOT NULL, [Id] [uniqueidentifier] NOT NULL,
[DocumentId] [uniqueidentifier] NOT NULL, [DocumentId] [uniqueidentifier] NOT NULL,
[Index] INT NOT NULL, [Index] [int] NOT NULL,
[Content] [nvarchar](max) NOT NULL, [Content] [nvarchar](max) NOT NULL,
[Embedding] [varbinary](8000) NOT NULL, [Embedding] [varbinary](8000) NOT NULL,
CONSTRAINT [PK_DocumentChunks] PRIMARY KEY CLUSTERED CONSTRAINT [PK_DocumentChunks] PRIMARY KEY CLUSTERED
@@ -17,15 +17,10 @@ CREATE TABLE [dbo].[Documents](
CONSTRAINT [PK_Documents] PRIMARY KEY CLUSTERED CONSTRAINT [PK_Documents] PRIMARY KEY CLUSTERED
( (
[Id] ASC [Id] ASC
)) ))
GO
ALTER TABLE [dbo].[DocumentChunks] ADD CONSTRAINT [DF_DocumentChunks_Id] DEFAULT (newid()) FOR [Id]
GO
ALTER TABLE [dbo].[Documents] ADD CONSTRAINT [DF_Documents_Id] DEFAULT (newid()) FOR [Id]
GO GO
ALTER TABLE [dbo].[DocumentChunks] WITH CHECK ADD CONSTRAINT [FK_DocumentChunks_Documents] FOREIGN KEY([DocumentId]) ALTER TABLE [dbo].[DocumentChunks] WITH CHECK ADD CONSTRAINT [FK_DocumentChunks_Documents] FOREIGN KEY([DocumentId])
REFERENCES [dbo].[Documents] ([Id]) REFERENCES [dbo].[Documents] ([Id])
GO ON DELETE CASCADE
GO
SqlDatabaseVectorSearch/DataAccessLayer/ApplicationDbContext.cs
+10 -3
View File
@@ -15,13 +15,17 @@ public class ApplicationDbContext(DbContextOptions<ApplicationDbContext> options
base.OnConfiguring(optionsBuilder); base.OnConfiguring(optionsBuilder);
optionsBuilder.UseExceptionProcessor(); optionsBuilder.UseExceptionProcessor();
//optionsBuilder.EnableSensitiveDataLogging();
} }
protected override void OnModelCreating(ModelBuilder modelBuilder) protected override void OnModelCreating(ModelBuilder modelBuilder)
{ {
modelBuilder.Entity<Document>(entity => modelBuilder.Entity<Document>(entity =>
{ {
entity.Property(e => e.Id).HasDefaultValueSql("(newid())"); entity.ToTable("Documents");
entity.HasKey(e => e.Id);
entity.Property(e => e.Id).ValueGeneratedOnAdd();
entity.Property(e => e.Name) entity.Property(e => e.Name)
.IsRequired() .IsRequired()
.HasMaxLength(255); .HasMaxLength(255);
@@ -29,7 +33,10 @@ public class ApplicationDbContext(DbContextOptions<ApplicationDbContext> options
modelBuilder.Entity<DocumentChunk>(entity => modelBuilder.Entity<DocumentChunk>(entity =>
{ {
entity.Property(e => e.Id).HasDefaultValueSql("(newid())"); entity.ToTable("DocumentChunks");
entity.HasKey(e => e.Id);
entity.Property(e => e.Id).ValueGeneratedOnAdd();
entity.Property(e => e.Content).IsRequired(); entity.Property(e => e.Content).IsRequired();
entity.Property(e => e.Embedding) entity.Property(e => e.Embedding)
.IsRequired() .IsRequired()
@@ -38,7 +45,7 @@ public class ApplicationDbContext(DbContextOptions<ApplicationDbContext> options
entity.HasOne(d => d.Document).WithMany(p => p.Chunks) entity.HasOne(d => d.Document).WithMany(p => p.Chunks)
.HasForeignKey(d => d.DocumentId) .HasForeignKey(d => d.DocumentId)
.OnDelete(DeleteBehavior.NoAction) .OnDelete(DeleteBehavior.Cascade)
.HasConstraintName("FK_DocumentChunks_Documents"); .HasConstraintName("FK_DocumentChunks_Documents");
}); });
} }
SqlDatabaseVectorSearch/Program.cs
+2
View File
@@ -17,6 +17,8 @@ builder.Configuration.AddJsonFile("appsettings.local.json", optional: true, relo
var aiSettings = builder.Configuration.GetSection<AzureOpenAISettings>("AzureOpenAI")!; var aiSettings = builder.Configuration.GetSection<AzureOpenAISettings>("AzureOpenAI")!;
var appSettings = builder.Services.ConfigureAndGet<AppSettings>(builder.Configuration, nameof(AppSettings))!; var appSettings = builder.Services.ConfigureAndGet<AppSettings>(builder.Configuration, nameof(AppSettings))!;
builder.Services.AddSingleton(TimeProvider.System);
builder.Services.AddSqlServer<ApplicationDbContext>(builder.Configuration.GetConnectionString("SqlConnection"), options => builder.Services.AddSqlServer<ApplicationDbContext>(builder.Configuration.GetConnectionString("SqlConnection"), options =>
{ {
options.UseVectorSearch(); options.UseVectorSearch();
SqlDatabaseVectorSearch/Services/VectorSearchService.cs
+7 -20
View File
@@ -12,7 +12,7 @@ using Entities = SqlDatabaseVectorSearch.DataAccessLayer.Entities;
namespace SqlDatabaseVectorSearch.Services; namespace SqlDatabaseVectorSearch.Services;
public class VectorSearchService(ApplicationDbContext dbContext, ITextEmbeddingGenerationService textEmbeddingGenerationService, ChatService chatService, IOptions<AppSettings> appSettingsOptions) public class VectorSearchService(ApplicationDbContext dbContext, ITextEmbeddingGenerationService textEmbeddingGenerationService, ChatService chatService, TimeProvider timeProvider, IOptions<AppSettings> appSettingsOptions)
{ {
private readonly AppSettings appSettings = appSettingsOptions.Value; private readonly AppSettings appSettings = appSettingsOptions.Value;
@@ -26,16 +26,10 @@ public class VectorSearchService(ApplicationDbContext dbContext, ITextEmbeddingG
if (documentId.HasValue) if (documentId.HasValue)
{ {
// If the user is importing a document that already exists, delete the previous one. // If the user is importing a document that already exists, delete the previous one.
await dbContext.DocumentChunks.Where(c => c.DocumentId == documentId).ExecuteDeleteAsync(); await DeleteDocumentAsync(documentId.Value);
await dbContext.Documents.Where(d => d.Id == documentId).ExecuteDeleteAsync();
}
else
{
// Create a new document.
documentId = Guid.NewGuid();
} }
var document = new Entities.Document { Id = documentId.Value, Name = name, CreationDate = DateTimeOffset.UtcNow }; var document = new Entities.Document { Id = documentId.GetValueOrDefault(), Name = name, CreationDate = timeProvider.GetUtcNow() };
dbContext.Documents.Add(document); dbContext.Documents.Add(document);
// Split the content into chunks and generate the embeddings for each one. // Split the content into chunks and generate the embeddings for each one.
@@ -45,14 +39,14 @@ public class VectorSearchService(ApplicationDbContext dbContext, ITextEmbeddingG
var index = 0; var index = 0;
foreach (var (paragraph, embedding) in paragraphs.Zip(embeddings, (p, e) => (p, e.ToArray()))) foreach (var (paragraph, embedding) in paragraphs.Zip(embeddings, (p, e) => (p, e.ToArray())))
{ {
var documentChunk = new Entities.DocumentChunk { DocumentId = documentId.Value, Index = index++, Content = paragraph, Embedding = embedding }; var documentChunk = new Entities.DocumentChunk { Document = document, Index = index++, Content = paragraph, Embedding = embedding };
dbContext.DocumentChunks.Add(documentChunk); dbContext.DocumentChunks.Add(documentChunk);
} }
await dbContext.SaveChangesAsync(); await dbContext.SaveChangesAsync();
await dbContext.Database.CommitTransactionAsync(); await dbContext.Database.CommitTransactionAsync();
return documentId.Value; return document.Id;
} }
public async Task<IEnumerable<Document>> GetDocumentsAsync() public async Task<IEnumerable<Document>> GetDocumentsAsync()
@@ -82,15 +76,8 @@ public class VectorSearchService(ApplicationDbContext dbContext, ITextEmbeddingG
return documentChunk; return documentChunk;
} }
public async Task DeleteDocumentAsync(Guid documentId) public Task DeleteDocumentAsync(Guid documentId)
{ => dbContext.Documents.Where(d => d.Id == documentId).ExecuteDeleteAsync();
await dbContext.Database.BeginTransactionAsync();
await dbContext.DocumentChunks.Where(c => c.DocumentId == documentId).ExecuteDeleteAsync();
await dbContext.Documents.Where(d => d.Id == documentId).ExecuteDeleteAsync();
await dbContext.Database.CommitTransactionAsync();
}
public async Task<Response> AskQuestionAsync(Question question, bool reformulate = true) public async Task<Response> AskQuestionAsync(Question question, bool reformulate = true)
{ {