feat: add page-fetcher-api — centralised Playwright page fetcher

Introduces page-fetcher-api, a new internal ASP.NET Core service that centralises all web-page fetching through a single Playwright (headless Chromium) browser instance. All fetches are persisted to the pageFetcher SQL schema for auditing. New projects: - Apis/page-fetcher-api-models: FetchPageRequest, FetchPageResponse, IPageFetcherApiClient - Apis/page-fetcher-data: PageFetchDbContext, PageFetchEntity, InitialSchema migration (schema: pageFetcher) - Apis/page-fetcher-api: PlaywrightBrowserService (singleton), PageFetcherService, PageController Changes to existing services: - cv-matcher-api: JobTextExtractor now calls IPageFetcherApiClient instead of HttpClient - cv-search-job: HtmlJobSearcher uses IPageFetcherApiClient (removes inline Playwright); CvSearchJobTask fetches individual job pages and applies keyword pre-filter before LLM call; passes pre-fetched JobDescription to cv-matcher-api to skip re-fetch - common: add PageFetcherApiSettings - docker-compose.yml, build.yml: add new service + env vars for callers Closes #43 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-08 17:43:56 +03:00
parent 1222a86eb7
commit 898dd09d50
31 changed files with 1121 additions and 110 deletions
@@ -0,0 +1,74 @@
+using System.Reflection;
+using Microsoft.EntityFrameworkCore;
+using PageFetcher.Data;
+using PageFetcherApi.Services;
+using Serilog;
+using StartupHelpers;
+
+StartupExtensions.LoadDotEnvFile();
+
+const string ServiceName = "page-fetcher-api";
+var appVersion = StartupExtensions.GetApplicationVersion(Assembly.GetExecutingAssembly());
+
+try
+{
+    var builder = WebApplication.CreateBuilder(args);
+
+    builder.ConfigureJsonSerilog(ServiceName, appVersion);
+    Log.Information("Starting {Service} version {AppVersion}", ServiceName, appVersion);
+
+    builder.AddAzureKeyVaultIfConfigured();
+
+    builder.Services.Configure<PageFetcherSettings>(builder.Configuration.GetSection("PageFetcher"));
+
+    builder.Services.AddDbContext<PageFetchDbContext>(options =>
+    {
+        var connectionString = builder.Services.GetConfiguredDbConnectionString(builder.Configuration);
+        options.UseSqlServer(connectionString, sql =>
+        {
+            sql.MigrationsHistoryTable(PageFetchDbContext.MigrationTableName, PageFetchDbContext.SchemaName);
+            sql.MigrationsAssembly("page-fetcher-data");
+        });
+    });
+
+    // Playwright browser: singleton hosted service, shared across all requests
+    builder.Services.AddSingleton<PlaywrightBrowserService>();
+    builder.Services.AddHostedService(sp => sp.GetRequiredService<PlaywrightBrowserService>());
+
+    builder.Services.AddScoped<PageFetcherService>();
+
+    builder.Services.AddControllers();
+    builder.Services.AddSwaggerWithXmlComments(Assembly.GetExecutingAssembly(), "Page Fetcher API");
+
+    var app = builder.Build();
+
+    app.LogStartupDiagnostics(ServiceName);
+
+    app.UseDefaultSerilogRequestLogging();
+    app.UseJsonExceptionHandler(ServiceName);
+    app.UseInternalApiKeyProtection();
+    app.UseSwaggerInDevelopment("Page Fetcher API", "PageFetcherAPI");
+
+    app.UseRouting();
+    app.UseAuthorization();
+    app.MapControllers();
+
+    Log.Information("Running EF Core migrations if any");
+    using (var scope = app.Services.CreateScope())
+    {
+        var db = scope.ServiceProvider.GetRequiredService<PageFetchDbContext>();
+        db.Database.Migrate();
+    }
+
+    Log.Information("{Service} startup complete. Listening for requests...", ServiceName);
+    app.Run();
+}
+catch (Exception ex)
+{
+    Log.Fatal(ex, "{Service} terminated unexpectedly", ServiceName);
+}
+finally
+{
+    Log.Information("Shutting down {Service}", ServiceName);
+    Log.CloseAndFlush();
+}