myAi/Jobs/cv-search-job/Services/HtmlJobSearcher.cs

using System.Text.RegularExpressions;
using System.Web;
using CvMatcher.Models.Settings;
using PageFetcher.Models;
using Microsoft.Extensions.Logging;

namespace CvSearchJob.Services;

/// <summary>
/// A URL and its anchor text as scraped from a job listing search-results page.
/// </summary>
public sealed record JobCandidate(string Url, string Title);

/// <summary>
/// Config-driven HTML scraper that fetches a provider's job listing page via <c>page-fetcher-api</c>
/// and extracts matching job URL candidates.
/// Uses a two-stage anchor filter: href must contain the provider's link pattern, and (optionally)
/// anchor text must contain at least one CV keyword.
/// </summary>
public sealed class HtmlJobSearcher
{
    private readonly IPageFetcherApiClient _pageFetcher;
    private readonly ILogger<HtmlJobSearcher> _logger;

    public HtmlJobSearcher(IPageFetcherApiClient pageFetcher, ILogger<HtmlJobSearcher> logger)
    {
        _pageFetcher = pageFetcher;
        _logger = logger;
    }

    /// <summary>
    /// Fetches the provider's search result page, parses all anchor tags, applies the two-stage filter,
    /// and returns up to <see cref="JobProviderConfig.MaxResults"/> candidates (URL + title).
    /// Returns an empty list when the page fetch fails rather than throwing.
    /// </summary>
    public async Task<IReadOnlyList<JobCandidate>> SearchJobUrlsAsync(
        JobProviderConfig provider,
        IReadOnlyList<string> cvKeywords,
        string? location,
        CancellationToken ct)
    {
        var allKeywords = provider.InitialKeywords
            .Concat(cvKeywords)
            .Where(k => !string.IsNullOrWhiteSpace(k))
            .Distinct(StringComparer.OrdinalIgnoreCase)
            .ToList();

        if (allKeywords.Count == 0)
        {
            _logger.LogWarning("Provider {Provider}: no keywords available (CV keywords empty, InitialKeywords empty), skipping", provider.Name);
            return [];
        }

        var keywordsEncoded = HttpUtility.UrlEncode(string.Join(" ", allKeywords));
        var locationEncoded = HttpUtility.UrlEncode(location ?? string.Empty);
        var locationSlug = (location ?? string.Empty)
            .ToLowerInvariant()
            .Replace(",", "")
            .Replace(" ", "-")
            .Trim('-');
        var searchUrl = provider.SearchUrlTemplate
            .Replace("{keywords}", keywordsEncoded)
            .Replace("{location}", locationEncoded)
            .Replace("{location-slug}", locationSlug);

        _logger.LogInformation(
            "Provider {Provider}: fetching {Url} | CV keywords: [{Keywords}] | Location: {Location}",
            provider.Name, searchUrl,
            string.Join(", ", cvKeywords),
            location ?? "(none)");

        var fetchResponse = await _pageFetcher.FetchAsync(new FetchPageRequest
        {
            Url = searchUrl,
            WaitFor = provider.UseHeadlessBrowser ? "networkidle" : "domcontentloaded",
            CallerService = "cv-search-job"
        }, ct);

        if (!fetchResponse.Success || string.IsNullOrWhiteSpace(fetchResponse.Html))
        {
            _logger.LogWarning("Provider {Provider}: page fetch failed — {Error}", provider.Name, fetchResponse.Error);
            return [];
        }

        var html = fetchResponse.Html;
        _logger.LogInformation("Provider {Provider}: received {Length} chars of HTML", provider.Name, html.Length);

        var baseUri = new Uri(searchUrl);
        var results = new List<JobCandidate>();
        var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);

        var anchorPattern = new Regex(@"<a[^>]+href=[""']([^""']+)[""'][^>]*>(.*?)</a>",
            RegexOptions.IgnoreCase | RegexOptions.Singleline);

        var allAnchors = anchorPattern.Matches(html);
        var stage1Pass = 0;
        var stage2Pass = 0;

        foreach (Match match in allAnchors)
        {
            if (results.Count >= provider.MaxResults) break;

            var href = match.Groups[1].Value.Trim();
            var anchorText = Regex.Replace(match.Groups[2].Value, "<[^>]+>", " ").Trim();

            if (!href.Contains(provider.JobLinkContains, StringComparison.OrdinalIgnoreCase))
                continue;

            stage1Pass++;

            if (provider.RequireKeywordInAnchor &&
                !cvKeywords.Any(k => anchorText.Contains(k, StringComparison.OrdinalIgnoreCase)))
            {
                _logger.LogDebug(
                    "Provider {Provider}: stage-2 reject | href={Href} | text={Text}",
                    provider.Name, href, anchorText.Length > 100 ? anchorText[..100] : anchorText);
                continue;
            }

            stage2Pass++;

            if (!Uri.TryCreate(href, UriKind.Absolute, out var absoluteUri))
            {
                if (!Uri.TryCreate(baseUri, href, out absoluteUri))
                    continue;
            }

            // Skip non-HTTP(S) URLs (e.g. file:// or javascript: that can appear in scraped HTML)
            if (absoluteUri.Scheme != Uri.UriSchemeHttp && absoluteUri.Scheme != Uri.UriSchemeHttps)
                continue;

            var url = absoluteUri.GetLeftPart(UriPartial.Path);
            if (seen.Add(url))
                results.Add(new JobCandidate(url, anchorText));
        }

        _logger.LogInformation(
            "Provider {Provider}: {TotalAnchors} anchors found | {Stage1} passed href filter ('{LinkPattern}') | {Stage2} passed keyword filter | {Unique} unique URLs returned",
            provider.Name, allAnchors.Count, stage1Pass, provider.JobLinkContains, stage2Pass, results.Count);

        return results;
    }
}