Files
myAi/Apis/cv-matcher-api/Program.cs
T
claude 898dd09d50 feat: add page-fetcher-api — centralised Playwright page fetcher
Introduces page-fetcher-api, a new internal ASP.NET Core service that
centralises all web-page fetching through a single Playwright (headless
Chromium) browser instance. All fetches are persisted to the pageFetcher
SQL schema for auditing.

New projects:
- Apis/page-fetcher-api-models: FetchPageRequest, FetchPageResponse, IPageFetcherApiClient
- Apis/page-fetcher-data: PageFetchDbContext, PageFetchEntity, InitialSchema migration (schema: pageFetcher)
- Apis/page-fetcher-api: PlaywrightBrowserService (singleton), PageFetcherService, PageController

Changes to existing services:
- cv-matcher-api: JobTextExtractor now calls IPageFetcherApiClient instead of HttpClient
- cv-search-job: HtmlJobSearcher uses IPageFetcherApiClient (removes inline Playwright);
  CvSearchJobTask fetches individual job pages and applies keyword pre-filter before
  LLM call; passes pre-fetched JobDescription to cv-matcher-api to skip re-fetch
- common: add PageFetcherApiSettings
- docker-compose.yml, build.yml: add new service + env vars for callers

Closes #43

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-08 17:43:56 +03:00

137 lines
5.2 KiB
C#

using Api.Clients.Ai;
using Api.Clients.Ai.Contracts;
using Api.Clients.Api;
using Api.Clients.Api.Contracts;
using CvMatcher.Data;
using CvMatcher.Data.Repositories;
using CvMatcher.Data.Repositories.Contracts;
using Api.Services;
using Api.Services.Contracts;
using CvMatcher.Models.Settings;
using CvSearch.Data;
using Microsoft.EntityFrameworkCore;
using Refit;
using Serilog;
using Common.Settings;
using PageFetcher.Models;
using StartupHelpers;
using System.Reflection;
StartupExtensions.LoadDotEnvFile();
const string ServiceName = "cv-matcher-api";
var appVersion = StartupExtensions.GetApplicationVersion(Assembly.GetExecutingAssembly());
try
{
var builder = WebApplication.CreateBuilder(args);
builder.ConfigureJsonSerilog(ServiceName, appVersion);
Log.Information("Starting {Service} version {AppVersion}", ServiceName, appVersion);
builder.AddAzureKeyVaultIfConfigured();
builder.Services.Configure<DatabaseSettings>(builder.Configuration.GetSection("Database"));
builder.Services.Configure<RagApiSettings>(builder.Configuration.GetSection("RagApi"));
builder.Services.Configure<InternalApiSettings>(builder.Configuration.GetSection("InternalApi"));
builder.Services.Configure<CvMatcher.Models.Settings.AiSettings>(builder.Configuration.GetSection("Ai"));
builder.Services.Configure<MatcherSettings>(builder.Configuration.GetSection("Matcher"));
builder.Services.Configure<JobSearchSettings>(builder.Configuration.GetSection("JobSearch"));
builder.Services.Configure<PageFetcherApiSettings>(builder.Configuration.GetSection("PageFetcherApi"));
builder.Services.AddRefitClient<IPageFetcherApiClient>()
.ConfigureHttpClient((sp, c) =>
{
var settings = sp.GetRequiredService<Microsoft.Extensions.Options.IOptions<PageFetcherApiSettings>>().Value;
c.BaseAddress = new Uri(settings.BaseUrl.TrimEnd('/') + "/");
if (!string.IsNullOrWhiteSpace(settings.InternalApiKey))
c.DefaultRequestHeaders.Add("X-Internal-Api-Key", settings.InternalApiKey);
});
builder.Services.AddRefitClient<IRefitRagApi>()
.ConfigureHttpClient((sp, c) =>
{
var settings = sp.GetRequiredService<Microsoft.Extensions.Options.IOptions<RagApiSettings>>().Value;
c.BaseAddress = new Uri(settings.BaseUrl.TrimEnd('/') + "/");
if (!string.IsNullOrWhiteSpace(settings.InternalApiKey))
{
c.DefaultRequestHeaders.Add("X-Internal-Api-Key", settings.InternalApiKey);
}
});
builder.Services.AddScoped<IRagApiClient, RagApiClient>();
builder.Services.AddHttpClient<IMatcherAiClient, MatcherAiClient>();
builder.Services.AddScoped<IJobTextExtractor, JobTextExtractor>();
builder.Services.AddDbContext<CvMatcherDbContext>(options =>
{
var configuration = builder.Configuration;
var connectionString = builder.Services.GetConfiguredDbConnectionString(configuration);
options.UseSqlServer(connectionString, sql =>
{
sql.MigrationsHistoryTable(CvMatcherDbContext.MigrationTableName, CvMatcherDbContext.SchemaName);
sql.MigrationsAssembly("cv-matcher-data");
});
});
builder.Services.AddDbContext<CvSearchDbContext>(options =>
{
var connectionString = builder.Services.GetConfiguredDbConnectionString(builder.Configuration);
options.UseSqlServer(connectionString, sql =>
{
sql.MigrationsAssembly("cv-search-data");
sql.MigrationsHistoryTable(CvSearchDbContext.MigrationTableName, CvSearchDbContext.SchemaName);
});
});
builder.Services.AddScoped<IMatcherRepository, EfMatcherRepository>();
builder.Services.AddScoped<IAiPromptsRepository, EfAiPromptsRepository>();
builder.Services.AddScoped<ICvMatcherService, CvMatcherService>();
builder.Services.AddScoped<IJobTokenService, JobTokenService>();
builder.Services.AddControllers();
builder.Services.AddSwaggerWithXmlComments(Assembly.GetExecutingAssembly(), ServiceName);
var app = builder.Build();
app.LogStartupDiagnostics(ServiceName);
using (var scope = app.Services.CreateScope())
{
var repository = scope.ServiceProvider.GetRequiredService<IMatcherRepository>();
await repository.InitializeAsync(CancellationToken.None);
}
app.UseDefaultSerilogRequestLogging();
app.UseJsonExceptionHandler(ServiceName);
app.UseInternalApiKeyProtection();
app.UseSwaggerInDevelopment(ServiceName, ServiceName);
app.MapControllers();
Log.Information("Running EF Core migrations if any");
using (var scope = app.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService<CvMatcherDbContext>();
db.Database.Migrate();
}
using (var scope = app.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService<CvSearchDbContext>();
db.Database.Migrate();
}
Log.Information("{Service} startup complete", ServiceName);
app.Run();
}
catch (Exception ex)
{
Log.Fatal(ex, "{Service} terminated unexpectedly", ServiceName);
}
finally
{
Log.Information("Shutting down {Service}", ServiceName);
Log.CloseAndFlush();
}