claude
e95ed36647
Phases 1-10 of the planned refactoring:
Phase 1: rename shared-models -> common
- namespace Shared.Models -> Common throughout
- remove stale AspNetCore.Http.Features 5.0 reference
Phase 2: create shared-data with abstract BaseEntity
- BaseEntity: required string Id { get; init; } + DateTime CreatedAt { get; init; }
Phase 3: rename myai-models -> myai-data
- namespace MyAi.Models -> MyAi.Data
- MigrationsAssembly("myai-data")
Phase 4: rename cv-search-models -> cv-search-data
- namespace CvSearch.Models -> CvSearch.Data
- move JobSearchSettings to cv-matcher-api-models
- JobSearch*Entity now inherits BaseEntity
Phase 5: extract rag-data from rag-api
- new project: Apis/rag-data with RagDbContext + entities + migrations
- RagDocumentEntity inherits BaseEntity; cache entities use CacheKey PK
- fix duplicate AddHttpClient<RagAiClient>/AddScoped registrations in rag-api
- MigrationsAssembly("rag-data")
Phase 6: extract cv-matcher-data from cv-matcher-api
- new project: Apis/cv-matcher-data with CvMatcherDbContext + entities + migrations
- CvMatchResultEntity inherits BaseEntity; CvMatcherChatCacheEntity uses CacheKey PK
- MigrationsAssembly("cv-matcher-data")
Phase 7: create empty cv-cleanup-job-models and cv-search-job-models
Phase 8: update all 5 Dockerfiles for renamed/new projects
Phase 9: reorganise .sln virtual folders (Apis/Jobs/Models/Data/Helpers)
- update root CLAUDE.md with new project taxonomy and migration commands
- update cv-matcher-api/CLAUDE.md and cv-search-job/CLAUDE.md
Phase 10: add Directory.Packages.props for centralised NuGet versions
- remove Version= from all PackageReference elements in active .csproj files
No database changes. No runtime behaviour changes.
All MigrationId strings in __EFMigrationsHistory are unaffected.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
87 lines
2.9 KiB
C#
87 lines
2.9 KiB
C#
using System.Text.RegularExpressions;
|
|
using System.Web;
|
|
using CvMatcher.Models.Settings;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace CvSearchJob.Services;
|
|
|
|
public sealed class HtmlJobSearcher
|
|
{
|
|
private readonly HttpClient _http;
|
|
private readonly ILogger<HtmlJobSearcher> _logger;
|
|
|
|
public HtmlJobSearcher(HttpClient http, ILogger<HtmlJobSearcher> logger)
|
|
{
|
|
_http = http;
|
|
_logger = logger;
|
|
_http.Timeout = TimeSpan.FromSeconds(20);
|
|
_http.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (compatible; MyAi.ro CV-Search/1.0)");
|
|
}
|
|
|
|
public async Task<IReadOnlyList<string>> SearchJobUrlsAsync(
|
|
JobProviderConfig provider,
|
|
IReadOnlyList<string> cvKeywords,
|
|
CancellationToken ct)
|
|
{
|
|
var allKeywords = provider.InitialKeywords
|
|
.Concat(cvKeywords)
|
|
.Where(k => !string.IsNullOrWhiteSpace(k))
|
|
.Distinct(StringComparer.OrdinalIgnoreCase)
|
|
.ToList();
|
|
|
|
if (allKeywords.Count == 0)
|
|
return [];
|
|
|
|
var keywordsEncoded = HttpUtility.UrlEncode(string.Join(" ", allKeywords));
|
|
var searchUrl = provider.SearchUrlTemplate.Replace("{keywords}", keywordsEncoded);
|
|
|
|
string html;
|
|
try
|
|
{
|
|
html = await _http.GetStringAsync(searchUrl, ct);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogWarning(ex, "Failed to fetch search results from {Provider} at {Url}", provider.Name, searchUrl);
|
|
return [];
|
|
}
|
|
|
|
var baseUri = new Uri(searchUrl);
|
|
var results = new List<string>();
|
|
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
|
|
|
// Match all anchor tags capturing href and inner text
|
|
var anchorPattern = new Regex(@"<a[^>]+href=[""']([^""']+)[""'][^>]*>(.*?)</a>",
|
|
RegexOptions.IgnoreCase | RegexOptions.Singleline);
|
|
|
|
foreach (Match match in anchorPattern.Matches(html))
|
|
{
|
|
if (results.Count >= provider.MaxResults) break;
|
|
|
|
var href = match.Groups[1].Value.Trim();
|
|
var anchorText = Regex.Replace(match.Groups[2].Value, "<[^>]+>", " ").Trim();
|
|
|
|
if (!href.Contains(provider.JobLinkContains, StringComparison.OrdinalIgnoreCase))
|
|
continue;
|
|
|
|
// Stage 2: anchor text must contain at least one CV keyword
|
|
if (!cvKeywords.Any(k => anchorText.Contains(k, StringComparison.OrdinalIgnoreCase)))
|
|
continue;
|
|
|
|
// Make absolute URL
|
|
if (!Uri.TryCreate(href, UriKind.Absolute, out var absoluteUri))
|
|
{
|
|
if (!Uri.TryCreate(baseUri, href, out absoluteUri))
|
|
continue;
|
|
}
|
|
|
|
var url = absoluteUri.GetLeftPart(UriPartial.Path);
|
|
if (seen.Add(url))
|
|
results.Add(url);
|
|
}
|
|
|
|
_logger.LogInformation("Provider {Provider}: found {Count} job URLs", provider.Name, results.Count);
|
|
return results;
|
|
}
|
|
}
|