feat: add page-fetcher-api — centralised Playwright page fetcher
Introduces page-fetcher-api, a new internal ASP.NET Core service that centralises all web-page fetching through a single Playwright (headless Chromium) browser instance. All fetches are persisted to the pageFetcher SQL schema for auditing. New projects: - Apis/page-fetcher-api-models: FetchPageRequest, FetchPageResponse, IPageFetcherApiClient - Apis/page-fetcher-data: PageFetchDbContext, PageFetchEntity, InitialSchema migration (schema: pageFetcher) - Apis/page-fetcher-api: PlaywrightBrowserService (singleton), PageFetcherService, PageController Changes to existing services: - cv-matcher-api: JobTextExtractor now calls IPageFetcherApiClient instead of HttpClient - cv-search-job: HtmlJobSearcher uses IPageFetcherApiClient (removes inline Playwright); CvSearchJobTask fetches individual job pages and applies keyword pre-filter before LLM call; passes pre-fetched JobDescription to cv-matcher-api to skip re-fetch - common: add PageFetcherApiSettings - docker-compose.yml, build.yml: add new service + env vars for callers Closes #43 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -13,6 +13,7 @@ using Microsoft.EntityFrameworkCore;
|
||||
using Refit;
|
||||
using Serilog;
|
||||
using Common.Settings;
|
||||
using PageFetcher.Models;
|
||||
using StartupHelpers;
|
||||
using System.Reflection;
|
||||
|
||||
@@ -36,6 +37,16 @@ try
|
||||
builder.Services.Configure<CvMatcher.Models.Settings.AiSettings>(builder.Configuration.GetSection("Ai"));
|
||||
builder.Services.Configure<MatcherSettings>(builder.Configuration.GetSection("Matcher"));
|
||||
builder.Services.Configure<JobSearchSettings>(builder.Configuration.GetSection("JobSearch"));
|
||||
builder.Services.Configure<PageFetcherApiSettings>(builder.Configuration.GetSection("PageFetcherApi"));
|
||||
|
||||
builder.Services.AddRefitClient<IPageFetcherApiClient>()
|
||||
.ConfigureHttpClient((sp, c) =>
|
||||
{
|
||||
var settings = sp.GetRequiredService<Microsoft.Extensions.Options.IOptions<PageFetcherApiSettings>>().Value;
|
||||
c.BaseAddress = new Uri(settings.BaseUrl.TrimEnd('/') + "/");
|
||||
if (!string.IsNullOrWhiteSpace(settings.InternalApiKey))
|
||||
c.DefaultRequestHeaders.Add("X-Internal-Api-Key", settings.InternalApiKey);
|
||||
});
|
||||
|
||||
builder.Services.AddRefitClient<IRefitRagApi>()
|
||||
.ConfigureHttpClient((sp, c) =>
|
||||
@@ -50,7 +61,7 @@ try
|
||||
|
||||
builder.Services.AddScoped<IRagApiClient, RagApiClient>();
|
||||
builder.Services.AddHttpClient<IMatcherAiClient, MatcherAiClient>();
|
||||
builder.Services.AddHttpClient<IJobTextExtractor, JobTextExtractor>();
|
||||
builder.Services.AddScoped<IJobTextExtractor, JobTextExtractor>();
|
||||
|
||||
builder.Services.AddDbContext<CvMatcherDbContext>(options =>
|
||||
{
|
||||
|
||||
@@ -1,26 +1,23 @@
|
||||
using System.Net;
|
||||
using System.Text.RegularExpressions;
|
||||
using CvMatcher.Models.Settings;
|
||||
using Api.Services.Contracts;
|
||||
using Microsoft.Extensions.Options;
|
||||
using PageFetcher.Models;
|
||||
|
||||
namespace Api.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Extracts normalised plain text from a job posting, either from a pasted description or by
|
||||
/// fetching and stripping the HTML of the job page URL.
|
||||
/// fetching the job page text via <c>page-fetcher-api</c> (headless Chromium rendering).
|
||||
/// </summary>
|
||||
public sealed class JobTextExtractor : IJobTextExtractor
|
||||
{
|
||||
private readonly HttpClient _http;
|
||||
private readonly IPageFetcherApiClient _pageFetcher;
|
||||
private readonly MatcherSettings _settings;
|
||||
|
||||
public JobTextExtractor(HttpClient http, IOptions<MatcherSettings> options)
|
||||
public JobTextExtractor(IPageFetcherApiClient pageFetcher, IOptions<MatcherSettings> options)
|
||||
{
|
||||
_http = http;
|
||||
_pageFetcher = pageFetcher;
|
||||
_settings = options.Value;
|
||||
_http.Timeout = TimeSpan.FromSeconds(25);
|
||||
_http.DefaultRequestHeaders.UserAgent.ParseAdd("MyAi.ro CV Matcher/1.0");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -31,15 +28,18 @@ public sealed class JobTextExtractor : IJobTextExtractor
|
||||
|
||||
if (string.IsNullOrWhiteSpace(jobUrl)) return string.Empty;
|
||||
if (!Uri.TryCreate(jobUrl, UriKind.Absolute, out var uri) || uri.Scheme is not ("http" or "https"))
|
||||
{
|
||||
throw new InvalidOperationException("Invalid job URL.");
|
||||
}
|
||||
|
||||
var html = await _http.GetStringAsync(uri, ct);
|
||||
html = Regex.Replace(html, "<script[\\s\\S]*?</script>", " ", RegexOptions.IgnoreCase);
|
||||
html = Regex.Replace(html, "<style[\\s\\S]*?</style>", " ", RegexOptions.IgnoreCase);
|
||||
html = Regex.Replace(html, "<[^>]+>", " ");
|
||||
return Limit(Normalize(WebUtility.HtmlDecode(html)));
|
||||
var response = await _pageFetcher.FetchAsync(new FetchPageRequest
|
||||
{
|
||||
Url = jobUrl,
|
||||
CallerService = "cv-matcher-api"
|
||||
}, ct);
|
||||
|
||||
if (!response.Success)
|
||||
throw new InvalidOperationException($"Failed to fetch job page: {response.Error}");
|
||||
|
||||
return Limit(Normalize(response.Text));
|
||||
}
|
||||
|
||||
/// <summary>Truncates text to the configured maximum character count.</summary>
|
||||
|
||||
@@ -82,6 +82,7 @@
|
||||
<ProjectReference Include="..\cv-search-data\cv-search-data.csproj" />
|
||||
<ProjectReference Include="..\cv-matcher-data\cv-matcher-data.csproj" />
|
||||
<ProjectReference Include="..\common\common.csproj" />
|
||||
<ProjectReference Include="..\page-fetcher-api-models\page-fetcher-api-models.csproj" />
|
||||
<ProjectReference Include="..\..\Helpers\startup-helpers\startup-helpers.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
Reference in New Issue
Block a user