claude
898dd09d50
Introduces page-fetcher-api, a new internal ASP.NET Core service that centralises all web-page fetching through a single Playwright (headless Chromium) browser instance. All fetches are persisted to the pageFetcher SQL schema for auditing. New projects: - Apis/page-fetcher-api-models: FetchPageRequest, FetchPageResponse, IPageFetcherApiClient - Apis/page-fetcher-data: PageFetchDbContext, PageFetchEntity, InitialSchema migration (schema: pageFetcher) - Apis/page-fetcher-api: PlaywrightBrowserService (singleton), PageFetcherService, PageController Changes to existing services: - cv-matcher-api: JobTextExtractor now calls IPageFetcherApiClient instead of HttpClient - cv-search-job: HtmlJobSearcher uses IPageFetcherApiClient (removes inline Playwright); CvSearchJobTask fetches individual job pages and applies keyword pre-filter before LLM call; passes pre-fetched JobDescription to cv-matcher-api to skip re-fetch - common: add PageFetcherApiSettings - docker-compose.yml, build.yml: add new service + env vars for callers Closes #43 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
59 lines
2.1 KiB
C#
59 lines
2.1 KiB
C#
using CvMatcher.Models.Settings;
|
|
using Api.Services.Contracts;
|
|
using Microsoft.Extensions.Options;
|
|
using PageFetcher.Models;
|
|
|
|
namespace Api.Services;
|
|
|
|
/// <summary>
|
|
/// Extracts normalised plain text from a job posting, either from a pasted description or by
|
|
/// fetching the job page text via <c>page-fetcher-api</c> (headless Chromium rendering).
|
|
/// </summary>
|
|
public sealed class JobTextExtractor : IJobTextExtractor
|
|
{
|
|
private readonly IPageFetcherApiClient _pageFetcher;
|
|
private readonly MatcherSettings _settings;
|
|
|
|
public JobTextExtractor(IPageFetcherApiClient pageFetcher, IOptions<MatcherSettings> options)
|
|
{
|
|
_pageFetcher = pageFetcher;
|
|
_settings = options.Value;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<string> ExtractAsync(string? jobUrl, string? jobDescription, CancellationToken ct)
|
|
{
|
|
var pasted = Normalize(jobDescription ?? string.Empty);
|
|
if (!string.IsNullOrWhiteSpace(pasted)) return Limit(pasted);
|
|
|
|
if (string.IsNullOrWhiteSpace(jobUrl)) return string.Empty;
|
|
if (!Uri.TryCreate(jobUrl, UriKind.Absolute, out var uri) || uri.Scheme is not ("http" or "https"))
|
|
throw new InvalidOperationException("Invalid job URL.");
|
|
|
|
var response = await _pageFetcher.FetchAsync(new FetchPageRequest
|
|
{
|
|
Url = jobUrl,
|
|
CallerService = "cv-matcher-api"
|
|
}, ct);
|
|
|
|
if (!response.Success)
|
|
throw new InvalidOperationException($"Failed to fetch job page: {response.Error}");
|
|
|
|
return Limit(Normalize(response.Text));
|
|
}
|
|
|
|
/// <summary>Truncates text to the configured maximum character count.</summary>
|
|
private string Limit(string value)
|
|
{
|
|
var max = Math.Max(4000, _settings.MaxJobTextChars);
|
|
return value.Length <= max ? value : value[..max];
|
|
}
|
|
|
|
/// <summary>Collapses all whitespace runs to single spaces and trims the result.</summary>
|
|
private static string Normalize(string value)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(value)) return string.Empty;
|
|
return string.Join(' ', value.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)).Trim();
|
|
}
|
|
}
|