feat: add page-fetcher-api — centralised Playwright page fetcher

Introduces page-fetcher-api, a new internal ASP.NET Core service that centralises all web-page fetching through a single Playwright (headless Chromium) browser instance. All fetches are persisted to the pageFetcher SQL schema for auditing. New projects: - Apis/page-fetcher-api-models: FetchPageRequest, FetchPageResponse, IPageFetcherApiClient - Apis/page-fetcher-data: PageFetchDbContext, PageFetchEntity, InitialSchema migration (schema: pageFetcher) - Apis/page-fetcher-api: PlaywrightBrowserService (singleton), PageFetcherService, PageController Changes to existing services: - cv-matcher-api: JobTextExtractor now calls IPageFetcherApiClient instead of HttpClient - cv-search-job: HtmlJobSearcher uses IPageFetcherApiClient (removes inline Playwright); CvSearchJobTask fetches individual job pages and applies keyword pre-filter before LLM call; passes pre-fetched JobDescription to cv-matcher-api to skip re-fetch - common: add PageFetcherApiSettings - docker-compose.yml, build.yml: add new service + env vars for callers Closes #43 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-08 17:43:56 +03:00
parent 1222a86eb7
commit 898dd09d50
31 changed files with 1121 additions and 110 deletions
@@ -0,0 +1,143 @@
+using System.Diagnostics;
+using System.Net;
+using System.Text.RegularExpressions;
+using Microsoft.Extensions.Options;
+using Microsoft.Playwright;
+using PageFetcher.Data;
+using PageFetcher.Data.Entities;
+using PageFetcher.Models;
+
+namespace PageFetcherApi.Services;
+
+/// <summary>
+/// Fetches a web page via Playwright, extracts plain text, persists the result to the database,
+/// and returns a <see cref="FetchPageResponse"/>.
+/// </summary>
+public sealed class PageFetcherService
+{
+    private readonly PlaywrightBrowserService _browserService;
+    private readonly PageFetchDbContext _db;
+    private readonly PageFetcherSettings _settings;
+    private readonly ILogger<PageFetcherService> _logger;
+
+    public PageFetcherService(
+        PlaywrightBrowserService browserService,
+        PageFetchDbContext db,
+        IOptions<PageFetcherSettings> settings,
+        ILogger<PageFetcherService> logger)
+    {
+        _browserService = browserService;
+        _db = db;
+        _settings = settings.Value;
+        _logger = logger;
+    }
+
+    /// <summary>
+    /// Fetches the page at <paramref name="request.Url"/> using Playwright, saves the fetch record,
+    /// and returns the HTML and extracted text.
+    /// Returns a failed response (with <see cref="FetchPageResponse.Success"/> = false) rather than throwing
+    /// on network or navigation errors.
+    /// </summary>
+    public async Task<FetchPageResponse> FetchAsync(FetchPageRequest request, CancellationToken ct)
+    {
+        var sw = Stopwatch.StartNew();
+        string html = string.Empty;
+        string text = string.Empty;
+        int? statusCode = null;
+        bool success = false;
+        string? errorMessage = null;
+        string finalUrl = request.Url;
+
+        try
+        {
+            var page = await _browserService.Browser.NewPageAsync();
+            await using var _ = page.ConfigureAwait(false);
+
+            var waitUntil = request.WaitFor?.ToLowerInvariant() switch
+            {
+                "load" => WaitUntilState.Load,
+                "domcontentloaded" => WaitUntilState.DOMContentLoaded,
+                _ => WaitUntilState.NetworkIdle
+            };
+
+            IResponse? response;
+            try
+            {
+                response = await page.GotoAsync(request.Url, new PageGotoOptions
+                {
+                    WaitUntil = waitUntil,
+                    Timeout = _settings.TimeoutSeconds * 1_000
+                });
+            }
+            catch (TimeoutException)
+            {
+                _logger.LogWarning("Playwright NetworkIdle timeout for {Url}, using partial content", request.Url);
+                response = null;
+            }
+
+            statusCode = response?.Status;
+            finalUrl = page.Url;
+            html = await page.ContentAsync();
+            text = ExtractText(html);
+            success = true;
+
+            _logger.LogInformation("Fetched {Url} → HTTP {Status} | HTML {HtmlLen} chars | text {TextLen} chars | {DurationMs} ms",
+                request.Url, statusCode?.ToString() ?? "timeout", html.Length, text.Length, sw.ElapsedMilliseconds);
+        }
+        catch (Exception ex)
+        {
+            errorMessage = ex.Message;
+            _logger.LogError(ex, "Failed to fetch {Url}", request.Url);
+        }
+        finally
+        {
+            sw.Stop();
+        }
+
+        // Persist fetch record
+        var entity = new PageFetchEntity
+        {
+            Id = Guid.NewGuid().ToString("N"),
+            Url = request.Url,
+            CallerService = request.CallerService ?? string.Empty,
+            HttpStatusCode = statusCode,
+            Html = html,
+            Text = text,
+            DurationMs = sw.ElapsedMilliseconds,
+            Success = success,
+            ErrorMessage = errorMessage
+        };
+
+        _db.PageFetches.Add(entity);
+        await _db.SaveChangesAsync(ct);
+
+        return new FetchPageResponse
+        {
+            Url = finalUrl,
+            StatusCode = statusCode ?? 0,
+            Html = html,
+            Text = text,
+            Success = success,
+            Error = errorMessage
+        };
+    }
+
+    /// <summary>
+    /// Strips script/style blocks and all HTML tags from raw HTML, normalises whitespace,
+    /// and truncates to <see cref="PageFetcherSettings.MaxTextChars"/>.
+    /// </summary>
+    private string ExtractText(string html)
+    {
+        if (string.IsNullOrWhiteSpace(html)) return string.Empty;
+
+        var text = html;
+        text = Regex.Replace(text, "<script[\\s\\S]*?</script>", " ", RegexOptions.IgnoreCase);
+        text = Regex.Replace(text, "<style[\\s\\S]*?</style>", " ", RegexOptions.IgnoreCase);
+        text = Regex.Replace(text, "<[^>]+>", " ");
+        text = WebUtility.HtmlDecode(text);
+        text = string.Join(' ', text.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)).Trim();
+
+        var max = Math.Max(4_000, _settings.MaxTextChars);
+        return text.Length <= max ? text : text[..max];
+    }
+}