Add TokenizerService and update settings configuration

Updated Program.cs to use ConfigureAndGet method for settings,
changed ChatService to singleton, and added TokenizerService
singleton. Modified ChatService to use TokenizerService for
token counting. Updated AppSettings and AzureOpenAISettings
with new properties. Added new package references in
SqlDatabaseVectorSearch.csproj. Updated appsettings.json with
new properties. Added TokenizerService class for token counting.
This commit is contained in:
Marco Minerva
2024-11-06 17:20:05 +01:00
parent c18a6b4e03
commit 5b43031251
7 changed files with 69 additions and 24 deletions
@@ -12,7 +12,9 @@
<PackageReference Include="EntityFrameworkCore.Exceptions.SqlServer" Version="8.1.3" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.10" />
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="8.0.10" />
<PackageReference Include="Microsoft.SemanticKernel" Version="1.26.0" />
<PackageReference Include="Microsoft.ML.Tokenizers" Version="0.22.0-preview.24526.1" />
<PackageReference Include="Microsoft.ML.Tokenizers.Data.O200kBase" Version="0.22.0-preview.24526.1" />
<PackageReference Include="Microsoft.SemanticKernel" Version="1.27.0" />
<PackageReference Include="MinimalHelpers.OpenApi" Version="2.0.17" />
<PackageReference Include="PdfPig" Version="0.1.9" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.9.0" />