feat: add page-fetcher-api — centralised Playwright page fetcher
Introduces page-fetcher-api, a new internal ASP.NET Core service that centralises all web-page fetching through a single Playwright (headless Chromium) browser instance. All fetches are persisted to the pageFetcher SQL schema for auditing. New projects: - Apis/page-fetcher-api-models: FetchPageRequest, FetchPageResponse, IPageFetcherApiClient - Apis/page-fetcher-data: PageFetchDbContext, PageFetchEntity, InitialSchema migration (schema: pageFetcher) - Apis/page-fetcher-api: PlaywrightBrowserService (singleton), PageFetcherService, PageController Changes to existing services: - cv-matcher-api: JobTextExtractor now calls IPageFetcherApiClient instead of HttpClient - cv-search-job: HtmlJobSearcher uses IPageFetcherApiClient (removes inline Playwright); CvSearchJobTask fetches individual job pages and applies keyword pre-filter before LLM call; passes pre-fetched JobDescription to cv-matcher-api to skip re-fetch - common: add PageFetcherApiSettings - docker-compose.yml, build.yml: add new service + env vars for callers Closes #43 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,47 @@
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using PageFetcher.Models;
|
||||
using PageFetcherApi.Services;
|
||||
using Swashbuckle.AspNetCore.Annotations;
|
||||
|
||||
namespace PageFetcherApi.Controllers;
|
||||
|
||||
/// <summary>
|
||||
/// Handles page-fetch requests: navigates to the URL via Playwright and returns rendered HTML and extracted text.
|
||||
/// </summary>
|
||||
[ApiController]
|
||||
[Route("api/page")]
|
||||
public sealed class PageController : ControllerBase
|
||||
{
|
||||
private readonly PageFetcherService _service;
|
||||
private readonly ILogger<PageController> _logger;
|
||||
|
||||
public PageController(PageFetcherService service, ILogger<PageController> logger)
|
||||
{
|
||||
_service = service;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fetches a web page via headless Chromium.
|
||||
/// Returns rendered HTML and extracted plain text.
|
||||
/// </summary>
|
||||
[HttpPost("fetch")]
|
||||
[SwaggerOperation(Summary = "Fetch a web page", Description = "Navigates to the given URL using Playwright, returns rendered HTML and stripped plain text.")]
|
||||
[SwaggerResponse(StatusCodes.Status200OK, "Page fetched successfully", typeof(FetchPageResponse))]
|
||||
[SwaggerResponse(StatusCodes.Status400BadRequest, "Invalid or non-HTTP(S) URL")]
|
||||
public async Task<ActionResult<FetchPageResponse>> Fetch([FromBody] FetchPageRequest request, CancellationToken ct)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(request.Url))
|
||||
return BadRequest(new { Error = "Url is required." });
|
||||
|
||||
if (!Uri.TryCreate(request.Url, UriKind.Absolute, out var uri) ||
|
||||
(uri.Scheme != Uri.UriSchemeHttp && uri.Scheme != Uri.UriSchemeHttps))
|
||||
return BadRequest(new { Error = "Url must be an absolute HTTP or HTTPS URL." });
|
||||
|
||||
_logger.LogInformation("Fetch request: {Url} | caller={Caller} | waitFor={WaitFor}",
|
||||
request.Url, request.CallerService, request.WaitFor);
|
||||
|
||||
var result = await _service.FetchAsync(request, ct);
|
||||
return Ok(result);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user