Mirrors/SqlDatabaseVectorSearch
1
0
Fork 0
mirror of https://github.com/marcominerva/SqlDatabaseVectorSearch.git synced 2026-06-20 12:23:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Data access optimizations

Browse Source
This commit is contained in:
Marco Minerva
2024-09-02 10:42:30 +02:00
parent e4570008e0
commit b6a09d0926
4 changed files with 23 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Scripts.sql
+2 -7
View File
@@ -1,7 +1,7 @@
CREATE TABLE [dbo].[DocumentChunks](
[Id] [uniqueidentifier] NOT NULL,
[DocumentId] [uniqueidentifier] NOT NULL,
[Index] INT NOT NULL,
[Index] [int] NOT NULL,
[Content] [nvarchar](max) NOT NULL,
[Embedding] [varbinary](8000) NOT NULL,
CONSTRAINT [PK_DocumentChunks] PRIMARY KEY CLUSTERED
@@ -20,12 +20,7 @@ CREATE TABLE [dbo].[Documents](
))
GO
ALTER TABLE [dbo].[DocumentChunks] ADD CONSTRAINT [DF_DocumentChunks_Id] DEFAULT (newid()) FOR [Id]
GO
ALTER TABLE [dbo].[Documents] ADD CONSTRAINT [DF_Documents_Id] DEFAULT (newid()) FOR [Id]
GO
ALTER TABLE [dbo].[DocumentChunks] WITH CHECK ADD CONSTRAINT [FK_DocumentChunks_Documents] FOREIGN KEY([DocumentId])
REFERENCES [dbo].[Documents] ([Id])
ON DELETE CASCADE
GO
SqlDatabaseVectorSearch/DataAccessLayer/ApplicationDbContext.cs
+10 -3
View File
@@ -15,13 +15,17 @@ public class ApplicationDbContext(DbContextOptions<ApplicationDbContext> options
base.OnConfiguring(optionsBuilder);
optionsBuilder.UseExceptionProcessor();
//optionsBuilder.EnableSensitiveDataLogging();
}
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
modelBuilder.Entity<Document>(entity =>
{
entity.Property(e => e.Id).HasDefaultValueSql("(newid())");
entity.ToTable("Documents");
entity.HasKey(e => e.Id);
entity.Property(e => e.Id).ValueGeneratedOnAdd();
entity.Property(e => e.Name)
.IsRequired()
.HasMaxLength(255);
@@ -29,7 +33,10 @@ public class ApplicationDbContext(DbContextOptions<ApplicationDbContext> options
modelBuilder.Entity<DocumentChunk>(entity =>
{
entity.Property(e => e.Id).HasDefaultValueSql("(newid())");
entity.ToTable("DocumentChunks");
entity.HasKey(e => e.Id);
entity.Property(e => e.Id).ValueGeneratedOnAdd();
entity.Property(e => e.Content).IsRequired();
entity.Property(e => e.Embedding)
.IsRequired()
@@ -38,7 +45,7 @@ public class ApplicationDbContext(DbContextOptions<ApplicationDbContext> options
entity.HasOne(d => d.Document).WithMany(p => p.Chunks)
.HasForeignKey(d => d.DocumentId)
.OnDelete(DeleteBehavior.NoAction)
.OnDelete(DeleteBehavior.Cascade)
.HasConstraintName("FK_DocumentChunks_Documents");
});
}
SqlDatabaseVectorSearch/Program.cs
+2
View File
@@ -17,6 +17,8 @@ builder.Configuration.AddJsonFile("appsettings.local.json", optional: true, relo
var aiSettings = builder.Configuration.GetSection<AzureOpenAISettings>("AzureOpenAI")!;
var appSettings = builder.Services.ConfigureAndGet<AppSettings>(builder.Configuration, nameof(AppSettings))!;
builder.Services.AddSingleton(TimeProvider.System);
builder.Services.AddSqlServer<ApplicationDbContext>(builder.Configuration.GetConnectionString("SqlConnection"), options =>
{
options.UseVectorSearch();
SqlDatabaseVectorSearch/Services/VectorSearchService.cs
+7 -20
View File
@@ -12,7 +12,7 @@ using Entities = SqlDatabaseVectorSearch.DataAccessLayer.Entities;
namespace SqlDatabaseVectorSearch.Services;
public class VectorSearchService(ApplicationDbContext dbContext, ITextEmbeddingGenerationService textEmbeddingGenerationService, ChatService chatService, IOptions<AppSettings> appSettingsOptions)
public class VectorSearchService(ApplicationDbContext dbContext, ITextEmbeddingGenerationService textEmbeddingGenerationService, ChatService chatService, TimeProvider timeProvider, IOptions<AppSettings> appSettingsOptions)
{
private readonly AppSettings appSettings = appSettingsOptions.Value;
@@ -26,16 +26,10 @@ public class VectorSearchService(ApplicationDbContext dbContext, ITextEmbeddingG
if (documentId.HasValue)
{
// If the user is importing a document that already exists, delete the previous one.
await dbContext.DocumentChunks.Where(c => c.DocumentId == documentId).ExecuteDeleteAsync();
await dbContext.Documents.Where(d => d.Id == documentId).ExecuteDeleteAsync();
}
else
{
// Create a new document.
documentId = Guid.NewGuid();
await DeleteDocumentAsync(documentId.Value);
}
var document = new Entities.Document { Id = documentId.Value, Name = name, CreationDate = DateTimeOffset.UtcNow };
var document = new Entities.Document { Id = documentId.GetValueOrDefault(), Name = name, CreationDate = timeProvider.GetUtcNow() };
dbContext.Documents.Add(document);
// Split the content into chunks and generate the embeddings for each one.
@@ -45,14 +39,14 @@ public class VectorSearchService(ApplicationDbContext dbContext, ITextEmbeddingG
var index = 0;
foreach (var (paragraph, embedding) in paragraphs.Zip(embeddings, (p, e) => (p, e.ToArray())))
{
var documentChunk = new Entities.DocumentChunk { DocumentId = documentId.Value, Index = index++, Content = paragraph, Embedding = embedding };
var documentChunk = new Entities.DocumentChunk { Document = document, Index = index++, Content = paragraph, Embedding = embedding };
dbContext.DocumentChunks.Add(documentChunk);
}
await dbContext.SaveChangesAsync();
await dbContext.Database.CommitTransactionAsync();
return documentId.Value;
return document.Id;
}
public async Task<IEnumerable<Document>> GetDocumentsAsync()
@@ -82,15 +76,8 @@ public class VectorSearchService(ApplicationDbContext dbContext, ITextEmbeddingG
return documentChunk;
}
public async Task DeleteDocumentAsync(Guid documentId)
{
await dbContext.Database.BeginTransactionAsync();
await dbContext.DocumentChunks.Where(c => c.DocumentId == documentId).ExecuteDeleteAsync();
await dbContext.Documents.Where(d => d.Id == documentId).ExecuteDeleteAsync();
await dbContext.Database.CommitTransactionAsync();
}
public Task DeleteDocumentAsync(Guid documentId)
=> dbContext.Documents.Where(d => d.Id == documentId).ExecuteDeleteAsync();
public async Task<Response> AskQuestionAsync(Question question, bool reformulate = true)
{