feat: add page-fetcher-api — centralised Playwright page fetcher

Introduces page-fetcher-api, a new internal ASP.NET Core service that
centralises all web-page fetching through a single Playwright (headless
Chromium) browser instance. All fetches are persisted to the pageFetcher
SQL schema for auditing.

New projects:
- Apis/page-fetcher-api-models: FetchPageRequest, FetchPageResponse, IPageFetcherApiClient
- Apis/page-fetcher-data: PageFetchDbContext, PageFetchEntity, InitialSchema migration (schema: pageFetcher)
- Apis/page-fetcher-api: PlaywrightBrowserService (singleton), PageFetcherService, PageController

Changes to existing services:
- cv-matcher-api: JobTextExtractor now calls IPageFetcherApiClient instead of HttpClient
- cv-search-job: HtmlJobSearcher uses IPageFetcherApiClient (removes inline Playwright);
  CvSearchJobTask fetches individual job pages and applies keyword pre-filter before
  LLM call; passes pre-fetched JobDescription to cv-matcher-api to skip re-fetch
- common: add PageFetcherApiSettings
- docker-compose.yml, build.yml: add new service + env vars for callers

Closes #43

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-06-08 17:43:56 +03:00
parent 1222a86eb7
commit 898dd09d50
31 changed files with 1121 additions and 110 deletions
+12 -1
View File
@@ -13,6 +13,7 @@ using Microsoft.EntityFrameworkCore;
using Refit;
using Serilog;
using Common.Settings;
using PageFetcher.Models;
using StartupHelpers;
using System.Reflection;
@@ -36,6 +37,16 @@ try
builder.Services.Configure<CvMatcher.Models.Settings.AiSettings>(builder.Configuration.GetSection("Ai"));
builder.Services.Configure<MatcherSettings>(builder.Configuration.GetSection("Matcher"));
builder.Services.Configure<JobSearchSettings>(builder.Configuration.GetSection("JobSearch"));
builder.Services.Configure<PageFetcherApiSettings>(builder.Configuration.GetSection("PageFetcherApi"));
builder.Services.AddRefitClient<IPageFetcherApiClient>()
.ConfigureHttpClient((sp, c) =>
{
var settings = sp.GetRequiredService<Microsoft.Extensions.Options.IOptions<PageFetcherApiSettings>>().Value;
c.BaseAddress = new Uri(settings.BaseUrl.TrimEnd('/') + "/");
if (!string.IsNullOrWhiteSpace(settings.InternalApiKey))
c.DefaultRequestHeaders.Add("X-Internal-Api-Key", settings.InternalApiKey);
});
builder.Services.AddRefitClient<IRefitRagApi>()
.ConfigureHttpClient((sp, c) =>
@@ -50,7 +61,7 @@ try
builder.Services.AddScoped<IRagApiClient, RagApiClient>();
builder.Services.AddHttpClient<IMatcherAiClient, MatcherAiClient>();
builder.Services.AddHttpClient<IJobTextExtractor, JobTextExtractor>();
builder.Services.AddScoped<IJobTextExtractor, JobTextExtractor>();
builder.Services.AddDbContext<CvMatcherDbContext>(options =>
{
@@ -1,26 +1,23 @@
using System.Net;
using System.Text.RegularExpressions;
using CvMatcher.Models.Settings;
using Api.Services.Contracts;
using Microsoft.Extensions.Options;
using PageFetcher.Models;
namespace Api.Services;
/// <summary>
/// Extracts normalised plain text from a job posting, either from a pasted description or by
/// fetching and stripping the HTML of the job page URL.
/// fetching the job page text via <c>page-fetcher-api</c> (headless Chromium rendering).
/// </summary>
public sealed class JobTextExtractor : IJobTextExtractor
{
private readonly HttpClient _http;
private readonly IPageFetcherApiClient _pageFetcher;
private readonly MatcherSettings _settings;
public JobTextExtractor(HttpClient http, IOptions<MatcherSettings> options)
public JobTextExtractor(IPageFetcherApiClient pageFetcher, IOptions<MatcherSettings> options)
{
_http = http;
_pageFetcher = pageFetcher;
_settings = options.Value;
_http.Timeout = TimeSpan.FromSeconds(25);
_http.DefaultRequestHeaders.UserAgent.ParseAdd("MyAi.ro CV Matcher/1.0");
}
/// <inheritdoc />
@@ -31,15 +28,18 @@ public sealed class JobTextExtractor : IJobTextExtractor
if (string.IsNullOrWhiteSpace(jobUrl)) return string.Empty;
if (!Uri.TryCreate(jobUrl, UriKind.Absolute, out var uri) || uri.Scheme is not ("http" or "https"))
{
throw new InvalidOperationException("Invalid job URL.");
}
var html = await _http.GetStringAsync(uri, ct);
html = Regex.Replace(html, "<script[\\s\\S]*?</script>", " ", RegexOptions.IgnoreCase);
html = Regex.Replace(html, "<style[\\s\\S]*?</style>", " ", RegexOptions.IgnoreCase);
html = Regex.Replace(html, "<[^>]+>", " ");
return Limit(Normalize(WebUtility.HtmlDecode(html)));
var response = await _pageFetcher.FetchAsync(new FetchPageRequest
{
Url = jobUrl,
CallerService = "cv-matcher-api"
}, ct);
if (!response.Success)
throw new InvalidOperationException($"Failed to fetch job page: {response.Error}");
return Limit(Normalize(response.Text));
}
/// <summary>Truncates text to the configured maximum character count.</summary>
@@ -82,6 +82,7 @@
<ProjectReference Include="..\cv-search-data\cv-search-data.csproj" />
<ProjectReference Include="..\cv-matcher-data\cv-matcher-data.csproj" />
<ProjectReference Include="..\common\common.csproj" />
<ProjectReference Include="..\page-fetcher-api-models\page-fetcher-api-models.csproj" />
<ProjectReference Include="..\..\Helpers\startup-helpers\startup-helpers.csproj" />
</ItemGroup>
</Project>