feat: add page-fetcher-api — centralised Playwright page fetcher
Introduces page-fetcher-api, a new internal ASP.NET Core service that centralises all web-page fetching through a single Playwright (headless Chromium) browser instance. All fetches are persisted to the pageFetcher SQL schema for auditing. New projects: - Apis/page-fetcher-api-models: FetchPageRequest, FetchPageResponse, IPageFetcherApiClient - Apis/page-fetcher-data: PageFetchDbContext, PageFetchEntity, InitialSchema migration (schema: pageFetcher) - Apis/page-fetcher-api: PlaywrightBrowserService (singleton), PageFetcherService, PageController Changes to existing services: - cv-matcher-api: JobTextExtractor now calls IPageFetcherApiClient instead of HttpClient - cv-search-job: HtmlJobSearcher uses IPageFetcherApiClient (removes inline Playwright); CvSearchJobTask fetches individual job pages and applies keyword pre-filter before LLM call; passes pre-fetched JobDescription to cv-matcher-api to skip re-fetch - common: add PageFetcherApiSettings - docker-compose.yml, build.yml: add new service + env vars for callers Closes #43 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -14,6 +14,7 @@ using JobScheduler.Tasks;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using PageFetcher.Models;
|
||||
using Refit;
|
||||
using Serilog;
|
||||
using Common.Settings;
|
||||
@@ -81,7 +82,19 @@ try
|
||||
client.DefaultRequestHeaders.Add("X-Internal-Api-Key", key);
|
||||
});
|
||||
|
||||
builder.Services.AddHttpClient<HtmlJobSearcher>();
|
||||
builder.Services.AddRefitClient<IPageFetcherApiClient>()
|
||||
.ConfigureHttpClient((sp, client) =>
|
||||
{
|
||||
var config = sp.GetRequiredService<Microsoft.Extensions.Configuration.IConfiguration>();
|
||||
var baseUrl = config["PageFetcherApi:BaseUrl"] ?? string.Empty;
|
||||
if (!string.IsNullOrWhiteSpace(baseUrl))
|
||||
client.BaseAddress = new Uri(baseUrl.TrimEnd('/') + "/");
|
||||
var key = config["PageFetcherApi:InternalApiKey"];
|
||||
if (!string.IsNullOrWhiteSpace(key))
|
||||
client.DefaultRequestHeaders.Add("X-Internal-Api-Key", key);
|
||||
});
|
||||
|
||||
builder.Services.AddSingleton<HtmlJobSearcher>();
|
||||
builder.Services.AddSingleton<CvSearchEmailSender>();
|
||||
|
||||
builder.Services.AddSingleton<CvSearchJobTask>();
|
||||
|
||||
Reference in New Issue
Block a user