Files
myAi/Apis/page-fetcher-api/Program.cs
T
claude 898dd09d50 feat: add page-fetcher-api — centralised Playwright page fetcher
Introduces page-fetcher-api, a new internal ASP.NET Core service that
centralises all web-page fetching through a single Playwright (headless
Chromium) browser instance. All fetches are persisted to the pageFetcher
SQL schema for auditing.

New projects:
- Apis/page-fetcher-api-models: FetchPageRequest, FetchPageResponse, IPageFetcherApiClient
- Apis/page-fetcher-data: PageFetchDbContext, PageFetchEntity, InitialSchema migration (schema: pageFetcher)
- Apis/page-fetcher-api: PlaywrightBrowserService (singleton), PageFetcherService, PageController

Changes to existing services:
- cv-matcher-api: JobTextExtractor now calls IPageFetcherApiClient instead of HttpClient
- cv-search-job: HtmlJobSearcher uses IPageFetcherApiClient (removes inline Playwright);
  CvSearchJobTask fetches individual job pages and applies keyword pre-filter before
  LLM call; passes pre-fetched JobDescription to cv-matcher-api to skip re-fetch
- common: add PageFetcherApiSettings
- docker-compose.yml, build.yml: add new service + env vars for callers

Closes #43

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-08 17:43:56 +03:00

75 lines
2.4 KiB
C#

using System.Reflection;
using Microsoft.EntityFrameworkCore;
using PageFetcher.Data;
using PageFetcherApi.Services;
using Serilog;
using StartupHelpers;
StartupExtensions.LoadDotEnvFile();
const string ServiceName = "page-fetcher-api";
var appVersion = StartupExtensions.GetApplicationVersion(Assembly.GetExecutingAssembly());
try
{
var builder = WebApplication.CreateBuilder(args);
builder.ConfigureJsonSerilog(ServiceName, appVersion);
Log.Information("Starting {Service} version {AppVersion}", ServiceName, appVersion);
builder.AddAzureKeyVaultIfConfigured();
builder.Services.Configure<PageFetcherSettings>(builder.Configuration.GetSection("PageFetcher"));
builder.Services.AddDbContext<PageFetchDbContext>(options =>
{
var connectionString = builder.Services.GetConfiguredDbConnectionString(builder.Configuration);
options.UseSqlServer(connectionString, sql =>
{
sql.MigrationsHistoryTable(PageFetchDbContext.MigrationTableName, PageFetchDbContext.SchemaName);
sql.MigrationsAssembly("page-fetcher-data");
});
});
// Playwright browser: singleton hosted service, shared across all requests
builder.Services.AddSingleton<PlaywrightBrowserService>();
builder.Services.AddHostedService(sp => sp.GetRequiredService<PlaywrightBrowserService>());
builder.Services.AddScoped<PageFetcherService>();
builder.Services.AddControllers();
builder.Services.AddSwaggerWithXmlComments(Assembly.GetExecutingAssembly(), "Page Fetcher API");
var app = builder.Build();
app.LogStartupDiagnostics(ServiceName);
app.UseDefaultSerilogRequestLogging();
app.UseJsonExceptionHandler(ServiceName);
app.UseInternalApiKeyProtection();
app.UseSwaggerInDevelopment("Page Fetcher API", "PageFetcherAPI");
app.UseRouting();
app.UseAuthorization();
app.MapControllers();
Log.Information("Running EF Core migrations if any");
using (var scope = app.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService<PageFetchDbContext>();
db.Database.Migrate();
}
Log.Information("{Service} startup complete. Listening for requests...", ServiceName);
app.Run();
}
catch (Exception ex)
{
Log.Fatal(ex, "{Service} terminated unexpectedly", ServiceName);
}
finally
{
Log.Information("Shutting down {Service}", ServiceName);
Log.CloseAndFlush();
}