Add internet job search feature (cv-search-job)

- New cv-search-models shared library: EF entities + CvSearchDbContext for cvSearch schema (JobSearchTokens, JobSearchSessions, JobSearchResults tables) - New cv-search-job worker service: polls DB for pending sessions, scrapes job boards via configurable HTML scraping, runs LLM scoring via cv-matcher-api, emails ranked results - cv-matcher-api: JobTokenService creates one-time tokens; JobSearchController handles link clicks and creates sessions - api: proxies job-search start endpoint, appends job search link to match result email - CI workflow updated to build and push myai-cv-search-job:staging image - CLAUDE.md documentation added for all affected services Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-05-22 17:56:23 +03:00
parent a0ae262afc
commit 6293fa89e3
38 changed files with 2074 additions and 18 deletions
@@ -0,0 +1,86 @@
+using System.Text.RegularExpressions;
+using System.Web;
+using CvSearch.Models.Settings;
+using Microsoft.Extensions.Logging;
+
+namespace CvSearchJob.Services;
+
+public sealed class HtmlJobSearcher
+{
+    private readonly HttpClient _http;
+    private readonly ILogger<HtmlJobSearcher> _logger;
+
+    public HtmlJobSearcher(HttpClient http, ILogger<HtmlJobSearcher> logger)
+    {
+        _http = http;
+        _logger = logger;
+        _http.Timeout = TimeSpan.FromSeconds(20);
+        _http.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (compatible; MyAi.ro CV-Search/1.0)");
+    }
+
+    public async Task<IReadOnlyList<string>> SearchJobUrlsAsync(
+        JobProviderConfig provider,
+        IReadOnlyList<string> cvKeywords,
+        CancellationToken ct)
+    {
+        var allKeywords = provider.InitialKeywords
+            .Concat(cvKeywords)
+            .Where(k => !string.IsNullOrWhiteSpace(k))
+            .Distinct(StringComparer.OrdinalIgnoreCase)
+            .ToList();
+
+        if (allKeywords.Count == 0)
+            return [];
+
+        var keywordsEncoded = HttpUtility.UrlEncode(string.Join(" ", allKeywords));
+        var searchUrl = provider.SearchUrlTemplate.Replace("{keywords}", keywordsEncoded);
+
+        string html;
+        try
+        {
+            html = await _http.GetStringAsync(searchUrl, ct);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Failed to fetch search results from {Provider} at {Url}", provider.Name, searchUrl);
+            return [];
+        }
+
+        var baseUri = new Uri(searchUrl);
+        var results = new List<string>();
+        var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+
+        // Match all anchor tags capturing href and inner text
+        var anchorPattern = new Regex(@"<a[^>]+href=[""']([^""']+)[""'][^>]*>(.*?)</a>",
+            RegexOptions.IgnoreCase | RegexOptions.Singleline);
+
+        foreach (Match match in anchorPattern.Matches(html))
+        {
+            if (results.Count >= provider.MaxResults) break;
+
+            var href = match.Groups[1].Value.Trim();
+            var anchorText = Regex.Replace(match.Groups[2].Value, "<[^>]+>", " ").Trim();
+
+            if (!href.Contains(provider.JobLinkContains, StringComparison.OrdinalIgnoreCase))
+                continue;
+
+            // Stage 2: anchor text must contain at least one CV keyword
+            if (!cvKeywords.Any(k => anchorText.Contains(k, StringComparison.OrdinalIgnoreCase)))
+                continue;
+
+            // Make absolute URL
+            if (!Uri.TryCreate(href, UriKind.Absolute, out var absoluteUri))
+            {
+                if (!Uri.TryCreate(baseUri, href, out absoluteUri))
+                    continue;
+            }
+
+            var url = absoluteUri.GetLeftPart(UriPartial.Path);
+            if (seen.Add(url))
+                results.Add(url);
+        }
+
+        _logger.LogInformation("Provider {Provider}: found {Count} job URLs", provider.Name, results.Count);
+        return results;
+    }
+}