feat: add page-fetcher-api — centralised Playwright page fetcher
Introduces page-fetcher-api, a new internal ASP.NET Core service that centralises all web-page fetching through a single Playwright (headless Chromium) browser instance. All fetches are persisted to the pageFetcher SQL schema for auditing. New projects: - Apis/page-fetcher-api-models: FetchPageRequest, FetchPageResponse, IPageFetcherApiClient - Apis/page-fetcher-data: PageFetchDbContext, PageFetchEntity, InitialSchema migration (schema: pageFetcher) - Apis/page-fetcher-api: PlaywrightBrowserService (singleton), PageFetcherService, PageController Changes to existing services: - cv-matcher-api: JobTextExtractor now calls IPageFetcherApiClient instead of HttpClient - cv-search-job: HtmlJobSearcher uses IPageFetcherApiClient (removes inline Playwright); CvSearchJobTask fetches individual job pages and applies keyword pre-filter before LLM call; passes pre-fetched JobDescription to cv-matcher-api to skip re-fetch - common: add PageFetcherApiSettings - docker-compose.yml, build.yml: add new service + env vars for callers Closes #43 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -14,6 +14,7 @@ using JobScheduler.Tasks;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using PageFetcher.Models;
|
||||
using Refit;
|
||||
using Serilog;
|
||||
using Common.Settings;
|
||||
@@ -81,7 +82,19 @@ try
|
||||
client.DefaultRequestHeaders.Add("X-Internal-Api-Key", key);
|
||||
});
|
||||
|
||||
builder.Services.AddHttpClient<HtmlJobSearcher>();
|
||||
builder.Services.AddRefitClient<IPageFetcherApiClient>()
|
||||
.ConfigureHttpClient((sp, client) =>
|
||||
{
|
||||
var config = sp.GetRequiredService<Microsoft.Extensions.Configuration.IConfiguration>();
|
||||
var baseUrl = config["PageFetcherApi:BaseUrl"] ?? string.Empty;
|
||||
if (!string.IsNullOrWhiteSpace(baseUrl))
|
||||
client.BaseAddress = new Uri(baseUrl.TrimEnd('/') + "/");
|
||||
var key = config["PageFetcherApi:InternalApiKey"];
|
||||
if (!string.IsNullOrWhiteSpace(key))
|
||||
client.DefaultRequestHeaders.Add("X-Internal-Api-Key", key);
|
||||
});
|
||||
|
||||
builder.Services.AddSingleton<HtmlJobSearcher>();
|
||||
builder.Services.AddSingleton<CvSearchEmailSender>();
|
||||
|
||||
builder.Services.AddSingleton<CvSearchJobTask>();
|
||||
|
||||
@@ -1,36 +1,39 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Web;
|
||||
using CvMatcher.Models.Settings;
|
||||
using Microsoft.Playwright;
|
||||
using PageFetcher.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace CvSearchJob.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Config-driven HTML scraper that fetches a provider's job listing page and extracts matching job URLs.
|
||||
/// Uses a two-stage anchor filter: href must contain the provider's link pattern, and anchor text must
|
||||
/// contain at least one CV keyword.
|
||||
/// Supports both plain HTTP GET (default) and headless Chromium rendering for JS-heavy SPAs.
|
||||
/// A URL and its anchor text as scraped from a job listing search-results page.
|
||||
/// </summary>
|
||||
public sealed record JobCandidate(string Url, string Title);
|
||||
|
||||
/// <summary>
|
||||
/// Config-driven HTML scraper that fetches a provider's job listing page via <c>page-fetcher-api</c>
|
||||
/// and extracts matching job URL candidates.
|
||||
/// Uses a two-stage anchor filter: href must contain the provider's link pattern, and (optionally)
|
||||
/// anchor text must contain at least one CV keyword.
|
||||
/// </summary>
|
||||
public sealed class HtmlJobSearcher
|
||||
{
|
||||
private readonly HttpClient _http;
|
||||
private readonly IPageFetcherApiClient _pageFetcher;
|
||||
private readonly ILogger<HtmlJobSearcher> _logger;
|
||||
|
||||
public HtmlJobSearcher(HttpClient http, ILogger<HtmlJobSearcher> logger)
|
||||
public HtmlJobSearcher(IPageFetcherApiClient pageFetcher, ILogger<HtmlJobSearcher> logger)
|
||||
{
|
||||
_http = http;
|
||||
_pageFetcher = pageFetcher;
|
||||
_logger = logger;
|
||||
_http.Timeout = TimeSpan.FromSeconds(20);
|
||||
_http.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (compatible; MyAi.ro CV-Search/1.0)");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fetches the provider's search result page for the combined initial + CV keywords, parses all anchor
|
||||
/// tags, applies the two-stage filter, and returns up to <see cref="JobProviderConfig.MaxResults"/> absolute URLs.
|
||||
/// Returns an empty list when the HTTP request fails rather than throwing.
|
||||
/// Fetches the provider's search result page, parses all anchor tags, applies the two-stage filter,
|
||||
/// and returns up to <see cref="JobProviderConfig.MaxResults"/> candidates (URL + title).
|
||||
/// Returns an empty list when the page fetch fails rather than throwing.
|
||||
/// </summary>
|
||||
public async Task<IReadOnlyList<string>> SearchJobUrlsAsync(
|
||||
public async Task<IReadOnlyList<JobCandidate>> SearchJobUrlsAsync(
|
||||
JobProviderConfig provider,
|
||||
IReadOnlyList<string> cvKeywords,
|
||||
string? location,
|
||||
@@ -61,24 +64,29 @@ public sealed class HtmlJobSearcher
|
||||
.Replace("{location-slug}", locationSlug);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Provider {Provider}: fetching {Url} [{Mode}] | CV keywords: [{Keywords}] | Location: {Location}",
|
||||
"Provider {Provider}: fetching {Url} | CV keywords: [{Keywords}] | Location: {Location}",
|
||||
provider.Name, searchUrl,
|
||||
provider.UseHeadlessBrowser ? "headless" : "http",
|
||||
string.Join(", ", cvKeywords),
|
||||
location ?? "(none)");
|
||||
|
||||
string? html;
|
||||
if (provider.UseHeadlessBrowser)
|
||||
html = await FetchWithPlaywrightAsync(provider.Name, searchUrl, ct);
|
||||
else
|
||||
html = await FetchWithHttpAsync(provider.Name, searchUrl, ct);
|
||||
var fetchResponse = await _pageFetcher.FetchAsync(new FetchPageRequest
|
||||
{
|
||||
Url = searchUrl,
|
||||
WaitFor = provider.UseHeadlessBrowser ? "networkidle" : "domcontentloaded",
|
||||
CallerService = "cv-search-job"
|
||||
}, ct);
|
||||
|
||||
if (html is null) return [];
|
||||
if (!fetchResponse.Success || string.IsNullOrWhiteSpace(fetchResponse.Html))
|
||||
{
|
||||
_logger.LogWarning("Provider {Provider}: page fetch failed — {Error}", provider.Name, fetchResponse.Error);
|
||||
return [];
|
||||
}
|
||||
|
||||
var html = fetchResponse.Html;
|
||||
_logger.LogInformation("Provider {Provider}: received {Length} chars of HTML", provider.Name, html.Length);
|
||||
|
||||
var baseUri = new Uri(searchUrl);
|
||||
var results = new List<string>();
|
||||
var results = new List<JobCandidate>();
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var anchorPattern = new Regex(@"<a[^>]+href=[""']([^""']+)[""'][^>]*>(.*?)</a>",
|
||||
@@ -123,7 +131,7 @@ public sealed class HtmlJobSearcher
|
||||
|
||||
var url = absoluteUri.GetLeftPart(UriPartial.Path);
|
||||
if (seen.Add(url))
|
||||
results.Add(url);
|
||||
results.Add(new JobCandidate(url, anchorText));
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
@@ -132,61 +140,4 @@ public sealed class HtmlJobSearcher
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private async Task<string?> FetchWithHttpAsync(string providerName, string url, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
return await _http.GetStringAsync(url, ct);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Provider {Provider}: HTTP fetch failed for {Url}", providerName, url);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<string?> FetchWithPlaywrightAsync(string providerName, string url, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var playwright = await Playwright.CreateAsync();
|
||||
await using var browser = await playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions
|
||||
{
|
||||
Headless = true,
|
||||
Args = ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"]
|
||||
});
|
||||
|
||||
var page = await browser.NewPageAsync();
|
||||
|
||||
IResponse? response;
|
||||
try
|
||||
{
|
||||
response = await page.GotoAsync(url, new PageGotoOptions
|
||||
{
|
||||
WaitUntil = WaitUntilState.NetworkIdle,
|
||||
Timeout = 30_000
|
||||
});
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
// NetworkIdle timed out — use whatever content rendered so far
|
||||
_logger.LogWarning("Provider {Provider}: Playwright NetworkIdle timeout for {Url}, using partial content", providerName, url);
|
||||
return await page.ContentAsync();
|
||||
}
|
||||
|
||||
if (response is null || response.Status >= 400)
|
||||
{
|
||||
_logger.LogWarning("Provider {Provider}: Playwright got HTTP {Status} for {Url}", providerName, response?.Status, url);
|
||||
return null;
|
||||
}
|
||||
|
||||
return await page.ContentAsync();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Provider {Provider}: Playwright fetch failed for {Url}", providerName, url);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using PageFetcher.Models;
|
||||
|
||||
namespace CvSearchJob.Tasks;
|
||||
|
||||
@@ -24,6 +25,7 @@ public sealed class CvSearchJobTask : IJobTask
|
||||
private readonly JobSearchSettings _settings;
|
||||
private readonly HtmlJobSearcher _searcher;
|
||||
private readonly ICvMatcherInternalApi _matcherApi;
|
||||
private readonly IPageFetcherApiClient _pageFetcher;
|
||||
private readonly CvSearchEmailSender _emailSender;
|
||||
private readonly ILogger<CvSearchJobTask> _logger;
|
||||
|
||||
@@ -34,6 +36,7 @@ public sealed class CvSearchJobTask : IJobTask
|
||||
IOptions<JobSearchSettings> settings,
|
||||
HtmlJobSearcher searcher,
|
||||
ICvMatcherInternalApi matcherApi,
|
||||
IPageFetcherApiClient pageFetcher,
|
||||
CvSearchEmailSender emailSender,
|
||||
ILogger<CvSearchJobTask> logger)
|
||||
{
|
||||
@@ -41,6 +44,7 @@ public sealed class CvSearchJobTask : IJobTask
|
||||
_settings = settings.Value;
|
||||
_searcher = searcher;
|
||||
_matcherApi = matcherApi;
|
||||
_pageFetcher = pageFetcher;
|
||||
_emailSender = emailSender;
|
||||
_logger = logger;
|
||||
}
|
||||
@@ -126,7 +130,8 @@ public sealed class CvSearchJobTask : IJobTask
|
||||
|
||||
/// <summary>
|
||||
/// Runs the full search pipeline for a session: scrapes all providers, deduplicates URLs,
|
||||
/// scores each candidate via the matcher API, and persists results that meet the minimum score threshold.
|
||||
/// fetches each individual job page via page-fetcher-api, applies a keyword pre-filter,
|
||||
/// scores passing candidates via the matcher API, and persists results that meet the minimum score threshold.
|
||||
/// </summary>
|
||||
private async Task<List<JobSearchResultEntity>> RunSearchAsync(
|
||||
JobSearchSessionEntity session,
|
||||
@@ -138,30 +143,59 @@ public sealed class CvSearchJobTask : IJobTask
|
||||
if (cvKeywords.Count == 0)
|
||||
_logger.LogWarning("Session {SessionId}: keyword list is empty — scraper will rely on provider InitialKeywords only", session.Id);
|
||||
|
||||
var jobUrls = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var jobCandidates = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase); // url → title
|
||||
|
||||
foreach (var provider in providers)
|
||||
{
|
||||
var urls = await _searcher.SearchJobUrlsAsync(provider, cvKeywords, session.Location, ct);
|
||||
_logger.LogInformation("Session {SessionId}: provider {Provider} returned {Count} URLs", session.Id, provider.Name, urls.Count);
|
||||
foreach (var url in urls) jobUrls.Add(url);
|
||||
var candidates = await _searcher.SearchJobUrlsAsync(provider, cvKeywords, session.Location, ct);
|
||||
_logger.LogInformation("Session {SessionId}: provider {Provider} returned {Count} candidates", session.Id, provider.Name, candidates.Count);
|
||||
foreach (var c in candidates)
|
||||
jobCandidates.TryAdd(c.Url, c.Title);
|
||||
}
|
||||
|
||||
var candidates = jobUrls.Take(_settings.MaxJobsToMatch).ToList();
|
||||
var deduped = jobCandidates.Take(_settings.MaxJobsToMatch).ToList();
|
||||
_logger.LogInformation(
|
||||
"Session {SessionId}: {Total} unique URLs across all providers, scoring {Scoring} (cap={Cap})",
|
||||
session.Id, jobUrls.Count, candidates.Count, _settings.MaxJobsToMatch);
|
||||
"Session {SessionId}: {Total} unique URLs across all providers, processing up to {Cap}",
|
||||
session.Id, jobCandidates.Count, deduped.Count);
|
||||
|
||||
var results = new List<JobSearchResultEntity>();
|
||||
|
||||
foreach (var url in candidates)
|
||||
foreach (var (url, title) in deduped)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Fetch individual job page text via page-fetcher-api
|
||||
var fetchResponse = await _pageFetcher.FetchAsync(new FetchPageRequest
|
||||
{
|
||||
Url = url,
|
||||
WaitFor = "domcontentloaded",
|
||||
CallerService = "cv-search-job"
|
||||
}, ct);
|
||||
|
||||
if (!fetchResponse.Success || string.IsNullOrWhiteSpace(fetchResponse.Text))
|
||||
{
|
||||
_logger.LogWarning("Session {SessionId}: fetch failed for {Url} — {Error}", session.Id, url, fetchResponse.Error);
|
||||
continue;
|
||||
}
|
||||
|
||||
var jobText = fetchResponse.Text;
|
||||
|
||||
// Keyword pre-filter: skip LLM call if no CV keyword appears in the job page text
|
||||
if (cvKeywords.Count > 0 &&
|
||||
!cvKeywords.Any(k => jobText.Contains(k, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Session {SessionId}: pre-filter skip | {Url} | no CV keyword found in job text",
|
||||
session.Id, url);
|
||||
continue;
|
||||
}
|
||||
|
||||
var matchRequest = new MatchJobRequest
|
||||
{
|
||||
CvDocumentId = session.CvDocumentId,
|
||||
JobUrl = url,
|
||||
// Pre-fetched text passed directly so cv-matcher-api skips re-fetching the page
|
||||
JobDescription = jobText,
|
||||
// User already gave GDPR consent when they clicked the one-time job search link
|
||||
GdprConsent = true
|
||||
};
|
||||
@@ -182,7 +216,7 @@ public sealed class CvSearchJobTask : IJobTask
|
||||
SessionId = session.Id,
|
||||
ProviderName = GuessProvider(url, providers),
|
||||
JobUrl = url,
|
||||
JobTitle = matchResult.Summary.Split('.').FirstOrDefault()?.Trim() ?? "Job",
|
||||
JobTitle = matchResult.Summary.Split('.').FirstOrDefault()?.Trim() ?? title,
|
||||
JobText = string.Empty,
|
||||
Score = matchResult.Score,
|
||||
ResultJson = JsonSerializer.Serialize(matchResult, new JsonSerializerOptions(JsonSerializerDefaults.Web)),
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" />
|
||||
<PackageReference Include="Refit.HttpClientFactory" />
|
||||
<PackageReference Include="Microsoft.Playwright" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
@@ -26,6 +25,7 @@
|
||||
<ProjectReference Include="..\..\Apis\cv-search-data\cv-search-data.csproj" />
|
||||
<ProjectReference Include="..\..\Apis\common\common.csproj" />
|
||||
<ProjectReference Include="..\..\Apis\email-data\email-data.csproj" />
|
||||
<ProjectReference Include="..\..\Apis\page-fetcher-api-models\page-fetcher-api-models.csproj" />
|
||||
<ProjectReference Include="..\..\Helpers\startup-helpers\startup-helpers.csproj" />
|
||||
<ProjectReference Include="..\job-scheduler\job-scheduler.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
Reference in New Issue
Block a user