Files
myAi/Jobs/cv-search-job/Program.cs
T
claude 898dd09d50 feat: add page-fetcher-api — centralised Playwright page fetcher
Introduces page-fetcher-api, a new internal ASP.NET Core service that
centralises all web-page fetching through a single Playwright (headless
Chromium) browser instance. All fetches are persisted to the pageFetcher
SQL schema for auditing.

New projects:
- Apis/page-fetcher-api-models: FetchPageRequest, FetchPageResponse, IPageFetcherApiClient
- Apis/page-fetcher-data: PageFetchDbContext, PageFetchEntity, InitialSchema migration (schema: pageFetcher)
- Apis/page-fetcher-api: PlaywrightBrowserService (singleton), PageFetcherService, PageController

Changes to existing services:
- cv-matcher-api: JobTextExtractor now calls IPageFetcherApiClient instead of HttpClient
- cv-search-job: HtmlJobSearcher uses IPageFetcherApiClient (removes inline Playwright);
  CvSearchJobTask fetches individual job pages and applies keyword pre-filter before
  LLM call; passes pre-fetched JobDescription to cv-matcher-api to skip re-fetch
- common: add PageFetcherApiSettings
- docker-compose.yml, build.yml: add new service + env vars for callers

Closes #43

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-08 17:43:56 +03:00

130 lines
4.8 KiB
C#

using System.Reflection;
using CvMatcher.Models.Settings;
using CvSearch.Data;
using CvSearchJob.Clients;
using CvSearchJob.Services;
using Email.Data;
using Email.Data.Repositories;
using Email.Data.Repositories.Contracts;
using Email.Data.Services;
using EmailApi.Models.Clients;
using CvSearchJob.Tasks;
using JobScheduler.Scheduling;
using JobScheduler.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using PageFetcher.Models;
using Refit;
using Serilog;
using Common.Settings;
using StartupHelpers;
const string ServiceName = "cv-search-job";
StartupExtensions.LoadDotEnvFile();
var appVersion = StartupExtensions.GetApplicationVersion(Assembly.GetExecutingAssembly());
try
{
var builder = Host.CreateApplicationBuilder(args);
builder.ConfigureJsonSerilog(ServiceName, appVersion);
Log.Information("Starting {Service} version {AppVersion}", ServiceName, appVersion);
builder.Services.Configure<JobSearchSettings>(builder.Configuration.GetSection("JobSearch"));
builder.Services.Configure<DatabaseSettings>(builder.Configuration.GetSection("Database"));
builder.Services.AddDbContext<CvSearchDbContext>(options =>
{
var connectionString = builder.Services.GetConfiguredDbConnectionString(builder.Configuration);
options.UseSqlServer(connectionString, sql =>
{
sql.MigrationsAssembly("cv-search-data");
sql.MigrationsHistoryTable(CvSearchDbContext.MigrationTableName, CvSearchDbContext.SchemaName);
});
});
builder.Services.AddRefitClient<ICvMatcherInternalApi>()
.ConfigureHttpClient((sp, client) =>
{
var config = sp.GetRequiredService<Microsoft.Extensions.Configuration.IConfiguration>();
var baseUrl = config["CvMatcherApi:BaseUrl"] ?? string.Empty;
if (!string.IsNullOrWhiteSpace(baseUrl))
client.BaseAddress = new Uri(baseUrl.TrimEnd('/') + "/");
var key = config["CvMatcherApi:InternalApiKey"];
if (!string.IsNullOrWhiteSpace(key))
client.DefaultRequestHeaders.Add("X-Internal-Api-Key", key);
});
builder.Services.AddDbContext<EmailDbContext>(options =>
{
var connectionString = builder.Services.GetConfiguredDbConnectionString(builder.Configuration);
options.UseSqlServer(connectionString, sql =>
{
sql.MigrationsHistoryTable(EmailDbContext.MigrationTableName, EmailDbContext.SchemaName);
sql.MigrationsAssembly("email-data");
});
});
builder.Services.AddScoped<IEmailTemplateRepository, EfEmailTemplateRepository>();
builder.Services.AddSingleton<IEmailTemplateService, EmailTemplateService>();
builder.Services.AddRefitClient<IEmailApiClient>()
.ConfigureHttpClient((sp, client) =>
{
var config = sp.GetRequiredService<Microsoft.Extensions.Configuration.IConfiguration>();
var baseUrl = config["EmailApi:BaseUrl"] ?? string.Empty;
if (!string.IsNullOrWhiteSpace(baseUrl))
client.BaseAddress = new Uri(baseUrl.TrimEnd('/') + "/");
var key = config["EmailApi:InternalApiKey"];
if (!string.IsNullOrWhiteSpace(key))
client.DefaultRequestHeaders.Add("X-Internal-Api-Key", key);
});
builder.Services.AddRefitClient<IPageFetcherApiClient>()
.ConfigureHttpClient((sp, client) =>
{
var config = sp.GetRequiredService<Microsoft.Extensions.Configuration.IConfiguration>();
var baseUrl = config["PageFetcherApi:BaseUrl"] ?? string.Empty;
if (!string.IsNullOrWhiteSpace(baseUrl))
client.BaseAddress = new Uri(baseUrl.TrimEnd('/') + "/");
var key = config["PageFetcherApi:InternalApiKey"];
if (!string.IsNullOrWhiteSpace(key))
client.DefaultRequestHeaders.Add("X-Internal-Api-Key", key);
});
builder.Services.AddSingleton<HtmlJobSearcher>();
builder.Services.AddSingleton<CvSearchEmailSender>();
builder.Services.AddSingleton<CvSearchJobTask>();
builder.Services.AddSingleton<IEnumerable<IJobTask>>(sp => new IJobTask[]
{
sp.GetRequiredService<CvSearchJobTask>(),
});
builder.Services.AddHostedService<JobSchedulerHostedService>();
var host = builder.Build();
host.LogHostStartupDiagnostics(ServiceName);
Log.Information("Running EF Core migrations");
using (var scope = host.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService<CvSearchDbContext>();
db.Database.Migrate();
}
Log.Information("{Service} startup complete. Background scheduler is running.", ServiceName);
await host.RunAsync();
}
catch (Exception ex)
{
Log.Fatal(ex, "{Service} terminated unexpectedly", ServiceName);
}
finally
{
Log.CloseAndFlush();
}