66 lines
3.1 KiB
C#
66 lines
3.1 KiB
C#
using System.Text.RegularExpressions;
|
|
using Api.Services.Contracts;
|
|
using Rag.Models;
|
|
|
|
namespace Api.Services;
|
|
|
|
public sealed class DocumentClassifier : IDocumentClassifier
|
|
{
|
|
private static readonly HashSet<string> KnownTypes = new(StringComparer.OrdinalIgnoreCase)
|
|
{
|
|
"cv", "job", "article", "contract", "invoice", "product", "documentation", "unknown"
|
|
};
|
|
|
|
public Task<DocumentClassification> ClassifyAsync(string text, string? providedType, string? providedTitle, CancellationToken ct)
|
|
{
|
|
if (!string.IsNullOrWhiteSpace(providedType))
|
|
{
|
|
var normalized = NormalizeType(providedType);
|
|
return Task.FromResult(new DocumentClassification
|
|
{
|
|
DocumentType = normalized,
|
|
Confidence = KnownTypes.Contains(normalized) && normalized != "unknown" ? 1.0 : 0.6,
|
|
Title = BuildTitle(providedTitle, text, normalized)
|
|
});
|
|
}
|
|
|
|
var lower = text.ToLowerInvariant();
|
|
var scores = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase)
|
|
{
|
|
["cv"] = Count(lower, "curriculum vitae", "resume", "work experience", "professional experience", "education", "skills", "technologies", "linkedin", "github"),
|
|
["job"] = Count(lower, "job description", "requirements", "responsibilities", "qualifications", "apply", "we are looking", "salary", "benefits", "remote", "hybrid"),
|
|
["contract"] = Count(lower, "agreement", "contract", "party", "parties", "liability", "termination", "confidentiality", "governing law"),
|
|
["invoice"] = Count(lower, "invoice", "vat", "subtotal", "total", "amount due", "due date", "billing"),
|
|
["documentation"] = Count(lower, "api", "endpoint", "configuration", "install", "usage", "parameters", "response", "request"),
|
|
["product"] = Count(lower, "features", "pricing", "sku", "product", "specification", "warranty")
|
|
};
|
|
|
|
var best = scores.OrderByDescending(x => x.Value).First();
|
|
var type = best.Value <= 0 ? "unknown" : best.Key;
|
|
var confidence = best.Value <= 0 ? 0.25 : Math.Min(0.95, 0.45 + best.Value * 0.08);
|
|
|
|
return Task.FromResult(new DocumentClassification
|
|
{
|
|
DocumentType = type,
|
|
Confidence = confidence,
|
|
Title = BuildTitle(providedTitle, text, type)
|
|
});
|
|
}
|
|
|
|
private static int Count(string lower, params string[] terms) => terms.Count(term => lower.Contains(term));
|
|
|
|
private static string NormalizeType(string value)
|
|
{
|
|
var cleaned = Regex.Replace(value.Trim().ToLowerInvariant(), "[^a-z0-9_-]", "-");
|
|
return string.IsNullOrWhiteSpace(cleaned) ? "unknown" : cleaned;
|
|
}
|
|
|
|
private static string BuildTitle(string? providedTitle, string text, string documentType)
|
|
{
|
|
if (!string.IsNullOrWhiteSpace(providedTitle)) return providedTitle.Trim();
|
|
var firstLine = text.Split('.', '\n', '\r').Select(x => x.Trim()).FirstOrDefault(x => x.Length > 20);
|
|
if (!string.IsNullOrWhiteSpace(firstLine)) return firstLine.Length <= 120 ? firstLine : firstLine[..120];
|
|
return $"{documentType} document";
|
|
}
|
|
}
|