myAi/Jobs/cv-search-job/Services/HtmlJobSearcher.cs

using System.Text.RegularExpressions;
using System.Web;
using CvMatcher.Models.Settings;
using Microsoft.Extensions.Logging;

namespace CvSearchJob.Services;

/// <summary>
/// Config-driven HTML scraper that fetches a provider's job listing page and extracts matching job URLs.
/// Uses a two-stage anchor filter: href must contain the provider's link pattern, and anchor text must
/// contain at least one CV keyword.
/// </summary>
public sealed class HtmlJobSearcher
{
    private readonly HttpClient _http;
    private readonly ILogger<HtmlJobSearcher> _logger;

    public HtmlJobSearcher(HttpClient http, ILogger<HtmlJobSearcher> logger)
    {
        _http = http;
        _logger = logger;
        _http.Timeout = TimeSpan.FromSeconds(20);
        _http.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (compatible; MyAi.ro CV-Search/1.0)");
    }

    /// <summary>
    /// Fetches the provider's search result page for the combined initial + CV keywords, parses all anchor
    /// tags, applies the two-stage filter, and returns up to <see cref="JobProviderConfig.MaxResults"/> absolute URLs.
    /// Returns an empty list when the HTTP request fails rather than throwing.
    /// </summary>
    /// <param name="provider">Provider configuration including search URL template, link filter, and result cap.</param>
    /// <param name="cvKeywords">Keywords extracted from the user's CV to inject into the search query.</param>
    /// <param name="ct">Cancellation token.</param>
    /// <returns>Deduplicated list of absolute job page URLs (query string stripped).</returns>
    public async Task<IReadOnlyList<string>> SearchJobUrlsAsync(
        JobProviderConfig provider,
        IReadOnlyList<string> cvKeywords,
        CancellationToken ct)
    {
        var allKeywords = provider.InitialKeywords
            .Concat(cvKeywords)
            .Where(k => !string.IsNullOrWhiteSpace(k))
            .Distinct(StringComparer.OrdinalIgnoreCase)
            .ToList();

        if (allKeywords.Count == 0)
        {
            _logger.LogWarning("Provider {Provider}: no keywords available (CV keywords empty, InitialKeywords empty), skipping", provider.Name);
            return [];
        }

        var keywordsEncoded = HttpUtility.UrlEncode(string.Join(" ", allKeywords));
        var searchUrl = provider.SearchUrlTemplate.Replace("{keywords}", keywordsEncoded);

        _logger.LogInformation(
            "Provider {Provider}: fetching {Url} | CV keywords: [{Keywords}]",
            provider.Name, searchUrl, string.Join(", ", cvKeywords));

        string html;
        try
        {
            html = await _http.GetStringAsync(searchUrl, ct);
            _logger.LogInformation("Provider {Provider}: received {Length} chars of HTML", provider.Name, html.Length);
        }
        catch (Exception ex)
        {
            _logger.LogWarning(ex, "Provider {Provider}: HTTP fetch failed for {Url}", provider.Name, searchUrl);
            return [];
        }

        var baseUri = new Uri(searchUrl);
        var results = new List<string>();
        var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);

        // Match all anchor tags capturing href and inner text
        var anchorPattern = new Regex(@"<a[^>]+href=[""']([^""']+)[""'][^>]*>(.*?)</a>",
            RegexOptions.IgnoreCase | RegexOptions.Singleline);

        var allAnchors = anchorPattern.Matches(html);
        var stage1Pass = 0;
        var stage2Pass = 0;

        foreach (Match match in allAnchors)
        {
            if (results.Count >= provider.MaxResults) break;

            var href = match.Groups[1].Value.Trim();
            var anchorText = Regex.Replace(match.Groups[2].Value, "<[^>]+>", " ").Trim();

            if (!href.Contains(provider.JobLinkContains, StringComparison.OrdinalIgnoreCase))
                continue;

            stage1Pass++;

            // Stage 2: anchor text must contain at least one CV keyword
            if (!cvKeywords.Any(k => anchorText.Contains(k, StringComparison.OrdinalIgnoreCase)))
            {
                _logger.LogDebug(
                    "Provider {Provider}: stage-2 reject | href={Href} | text={Text}",
                    provider.Name, href, anchorText.Length > 100 ? anchorText[..100] : anchorText);
                continue;
            }

            stage2Pass++;

            // Make absolute URL
            if (!Uri.TryCreate(href, UriKind.Absolute, out var absoluteUri))
            {
                if (!Uri.TryCreate(baseUri, href, out absoluteUri))
                    continue;
            }

            // Strip query string and fragment so different tracking variants of the same URL collapse to one.
            var url = absoluteUri.GetLeftPart(UriPartial.Path);
            if (seen.Add(url))
                results.Add(url);
        }

        _logger.LogInformation(
            "Provider {Provider}: {TotalAnchors} anchors found | {Stage1} passed href filter ('{LinkPattern}') | {Stage2} passed keyword filter | {Unique} unique URLs returned",
            provider.Name, allAnchors.Count, stage1Pass, provider.JobLinkContains, stage2Pass, results.Count);

        return results;
    }
}