AI/myAi
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects 1 Releases Wiki Activity

feat: add page-fetcher-api — centralised Playwright page fetcher

Browse Source 
Introduces page-fetcher-api, a new internal ASP.NET Core service that
centralises all web-page fetching through a single Playwright (headless
Chromium) browser instance. All fetches are persisted to the pageFetcher
SQL schema for auditing.

New projects:
- Apis/page-fetcher-api-models: FetchPageRequest, FetchPageResponse, IPageFetcherApiClient
- Apis/page-fetcher-data: PageFetchDbContext, PageFetchEntity, InitialSchema migration (schema: pageFetcher)
- Apis/page-fetcher-api: PlaywrightBrowserService (singleton), PageFetcherService, PageController

Changes to existing services:
- cv-matcher-api: JobTextExtractor now calls IPageFetcherApiClient instead of HttpClient
- cv-search-job: HtmlJobSearcher uses IPageFetcherApiClient (removes inline Playwright);
  CvSearchJobTask fetches individual job pages and applies keyword pre-filter before
  LLM call; passes pre-fetched JobDescription to cv-matcher-api to skip re-fetch
- common: add PageFetcherApiSettings
- docker-compose.yml, build.yml: add new service + env vars for callers

Closes #43

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
claude
2026-06-08 17:43:56 +03:00
parent 1222a86eb7
commit 898dd09d50
31 changed files with 1121 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Jobs/cv-search-job/Services/HtmlJobSearcher.cs
+32 -81
View File
@@ -1,36 +1,39 @@
using System.Text.RegularExpressions;
using System.Web;
using CvMatcher.Models.Settings;
using Microsoft.Playwright;
using PageFetcher.Models;
using Microsoft.Extensions.Logging;
namespace CvSearchJob.Services;
/// <summary>
/// Config-driven HTML scraper that fetches a provider's job listing page and extracts matching job URLs.
/// Uses a two-stage anchor filter: href must contain the provider's link pattern, and anchor text must
/// contain at least one CV keyword.
/// Supports both plain HTTP GET (default) and headless Chromium rendering for JS-heavy SPAs.
/// A URL and its anchor text as scraped from a job listing search-results page.
/// </summary>
public sealed record JobCandidate(string Url, string Title);
/// <summary>
/// Config-driven HTML scraper that fetches a provider's job listing page via <c>page-fetcher-api</c>
/// and extracts matching job URL candidates.
/// Uses a two-stage anchor filter: href must contain the provider's link pattern, and (optionally)
/// anchor text must contain at least one CV keyword.
/// </summary>
public sealed class HtmlJobSearcher
{
private readonly HttpClient _http;
private readonly IPageFetcherApiClient _pageFetcher;
private readonly ILogger<HtmlJobSearcher> _logger;
public HtmlJobSearcher(HttpClient http, ILogger<HtmlJobSearcher> logger)
public HtmlJobSearcher(IPageFetcherApiClient pageFetcher, ILogger<HtmlJobSearcher> logger)
{
_http = http;
_pageFetcher = pageFetcher;
_logger = logger;
_http.Timeout = TimeSpan.FromSeconds(20);
_http.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (compatible; MyAi.ro CV-Search/1.0)");
}
/// <summary>
/// Fetches the provider's search result page for the combined initial + CV keywords, parses all anchor
/// tags, applies the two-stage filter, and returns up to <see cref="JobProviderConfig.MaxResults"/> absolute URLs.
/// Returns an empty list when the HTTP request fails rather than throwing.
/// Fetches the provider's search result page, parses all anchor tags, applies the two-stage filter,
/// and returns up to <see cref="JobProviderConfig.MaxResults"/> candidates (URL + title).
/// Returns an empty list when the page fetch fails rather than throwing.
/// </summary>
public async Task<IReadOnlyList<string>> SearchJobUrlsAsync(
public async Task<IReadOnlyList<JobCandidate>> SearchJobUrlsAsync(
JobProviderConfig provider,
IReadOnlyList<string> cvKeywords,
string? location,
@@ -61,24 +64,29 @@ public sealed class HtmlJobSearcher
.Replace("{location-slug}", locationSlug);
_logger.LogInformation(
"Provider {Provider}: fetching {Url} [{Mode}] | CV keywords: [{Keywords}] | Location: {Location}",
"Provider {Provider}: fetching {Url} | CV keywords: [{Keywords}] | Location: {Location}",
provider.Name, searchUrl,
provider.UseHeadlessBrowser ? "headless" : "http",
string.Join(", ", cvKeywords),
location ?? "(none)");
string? html;
if (provider.UseHeadlessBrowser)
html = await FetchWithPlaywrightAsync(provider.Name, searchUrl, ct);
else
html = await FetchWithHttpAsync(provider.Name, searchUrl, ct);
var fetchResponse = await _pageFetcher.FetchAsync(new FetchPageRequest
{
Url = searchUrl,
WaitFor = provider.UseHeadlessBrowser ? "networkidle" : "domcontentloaded",
CallerService = "cv-search-job"
}, ct);
if (html is null) return [];
if (!fetchResponse.Success || string.IsNullOrWhiteSpace(fetchResponse.Html))
{
_logger.LogWarning("Provider {Provider}: page fetch failed — {Error}", provider.Name, fetchResponse.Error);
return [];
}
var html = fetchResponse.Html;
_logger.LogInformation("Provider {Provider}: received {Length} chars of HTML", provider.Name, html.Length);
var baseUri = new Uri(searchUrl);
var results = new List<string>();
var results = new List<JobCandidate>();
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var anchorPattern = new Regex(@"<a[^>]+href=[""']([^""']+)[""'][^>]*>(.*?)</a>",
@@ -123,7 +131,7 @@ public sealed class HtmlJobSearcher
var url = absoluteUri.GetLeftPart(UriPartial.Path);
if (seen.Add(url))
results.Add(url);
results.Add(new JobCandidate(url, anchorText));
}
_logger.LogInformation(
@@ -132,61 +140,4 @@ public sealed class HtmlJobSearcher
return results;
}
private async Task<string?> FetchWithHttpAsync(string providerName, string url, CancellationToken ct)
{
try
{
return await _http.GetStringAsync(url, ct);
}
catch (Exception ex)
{
_logger.LogError(ex, "Provider {Provider}: HTTP fetch failed for {Url}", providerName, url);
return null;
}
}
private async Task<string?> FetchWithPlaywrightAsync(string providerName, string url, CancellationToken ct)
{
try
{
using var playwright = await Playwright.CreateAsync();
await using var browser = await playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions
{
Headless = true,
Args = ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"]
});
var page = await browser.NewPageAsync();
IResponse? response;
try
{
response = await page.GotoAsync(url, new PageGotoOptions
{
WaitUntil = WaitUntilState.NetworkIdle,
Timeout = 30_000
});
}
catch (TimeoutException)
{
// NetworkIdle timed out — use whatever content rendered so far
_logger.LogWarning("Provider {Provider}: Playwright NetworkIdle timeout for {Url}, using partial content", providerName, url);
return await page.ContentAsync();
}
if (response is null || response.Status >= 400)
{
_logger.LogWarning("Provider {Provider}: Playwright got HTTP {Status} for {Url}", providerName, response?.Status, url);
return null;
}
return await page.ContentAsync();
}
catch (Exception ex)
{
_logger.LogError(ex, "Provider {Provider}: Playwright fetch failed for {Url}", providerName, url);
return null;
}
}
}