AI/myAi
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects 1 Releases Wiki Activity

feat: add page-fetcher-api — centralised Playwright page fetcher

Browse Source 
Introduces page-fetcher-api, a new internal ASP.NET Core service that
centralises all web-page fetching through a single Playwright (headless
Chromium) browser instance. All fetches are persisted to the pageFetcher
SQL schema for auditing.

New projects:
- Apis/page-fetcher-api-models: FetchPageRequest, FetchPageResponse, IPageFetcherApiClient
- Apis/page-fetcher-data: PageFetchDbContext, PageFetchEntity, InitialSchema migration (schema: pageFetcher)
- Apis/page-fetcher-api: PlaywrightBrowserService (singleton), PageFetcherService, PageController

Changes to existing services:
- cv-matcher-api: JobTextExtractor now calls IPageFetcherApiClient instead of HttpClient
- cv-search-job: HtmlJobSearcher uses IPageFetcherApiClient (removes inline Playwright);
  CvSearchJobTask fetches individual job pages and applies keyword pre-filter before
  LLM call; passes pre-fetched JobDescription to cv-matcher-api to skip re-fetch
- common: add PageFetcherApiSettings
- docker-compose.yml, build.yml: add new service + env vars for callers

Closes #43

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
claude
2026-06-08 17:43:56 +03:00
parent 1222a86eb7
commit 898dd09d50
31 changed files with 1121 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Apis/page-fetcher-data/Data/Entities/PageFetchEntity.cs
+34
View File
@@ -0,0 +1,34 @@
using Shared.Data.Entities;
namespace PageFetcher.Data.Entities;
/// <summary>
/// Audit record of a single page-fetch operation performed by the page-fetcher-api.
/// Stores the full rendered HTML and extracted plain text for every URL fetched.
/// </summary>
public sealed class PageFetchEntity : BaseEntity
{
/// <summary>The URL that was requested.</summary>
public string Url { get; set; } = string.Empty;
/// <summary>Name of the service that requested the fetch (e.g. <c>cv-matcher-api</c>, <c>cv-search-job</c>).</summary>
public string CallerService { get; set; } = string.Empty;
/// <summary>HTTP status code returned by the remote server. <c>null</c> on network failure.</summary>
public int? HttpStatusCode { get; set; }
/// <summary>Full rendered HTML as returned by Playwright.</summary>
public string Html { get; set; } = string.Empty;
/// <summary>Plain text extracted from the HTML (script/style stripped, whitespace normalised).</summary>
public string Text { get; set; } = string.Empty;
/// <summary>Playwright round-trip time in milliseconds.</summary>
public long DurationMs { get; set; }
/// <summary><c>true</c> when the page was fetched successfully; <c>false</c> on timeout or network error.</summary>
public bool Success { get; set; }
/// <summary>Exception message when <see cref="Success"/> is <c>false</c>.</summary>
public string? ErrorMessage { get; set; }
}
Apis/page-fetcher-data/MigrationConstants.cs
+8
View File
@@ -0,0 +1,8 @@
namespace PageFetcher.Data;
/// <summary>Schema and migration-history table name constants for the pageFetcher EF schema.</summary>
public static class MigrationConstants
{
public const string SchemaName = "pageFetcher";
public const string MigrationTableName = "_Migrations";
}
Apis/page-fetcher-data/Migrations/20260608143523_InitialSchema.Designer.cs
Generated
+82
View File
@@ -0,0 +1,82 @@
// <auto-generated />
using System;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Metadata;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using PageFetcher.Data;
#nullable disable
namespace PageFetcher.Data.Migrations
{
[DbContext(typeof(PageFetchDbContext))]
[Migration("20260608143523_InitialSchema")]
partial class InitialSchema
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasDefaultSchema("pageFetcher")
.HasAnnotation("ProductVersion", "10.0.7")
.HasAnnotation("Relational:MaxIdentifierLength", 128);
SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder);
modelBuilder.Entity("PageFetcher.Data.Entities.PageFetchEntity", b =>
{
b.Property<string>("Id")
.HasMaxLength(64)
.HasColumnType("nvarchar(64)");
b.Property<string>("CallerService")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("nvarchar(64)");
b.Property<DateTime>("CreatedAt")
.ValueGeneratedOnAdd()
.HasColumnType("datetime2")
.HasDefaultValueSql("SYSUTCDATETIME()");
b.Property<long>("DurationMs")
.HasColumnType("bigint");
b.Property<string>("ErrorMessage")
.HasMaxLength(2000)
.HasColumnType("nvarchar(2000)");
b.Property<string>("Html")
.IsRequired()
.HasColumnType("nvarchar(max)");
b.Property<int?>("HttpStatusCode")
.HasColumnType("int");
b.Property<bool>("Success")
.HasColumnType("bit");
b.Property<string>("Text")
.IsRequired()
.HasColumnType("nvarchar(max)");
b.Property<string>("Url")
.IsRequired()
.HasMaxLength(2000)
.HasColumnType("nvarchar(2000)");
b.HasKey("Id");
b.HasIndex("CreatedAt");
b.HasIndex("Url");
b.ToTable("PageFetches", "pageFetcher");
});
#pragma warning restore 612, 618
}
}
}
Apis/page-fetcher-data/Migrations/20260608143523_InitialSchema.cs
+59
View File
@@ -0,0 +1,59 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace PageFetcher.Data.Migrations
{
/// <inheritdoc />
public partial class InitialSchema : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.EnsureSchema(
name: MigrationConstants.SchemaName);
migrationBuilder.CreateTable(
name: "PageFetches",
schema: MigrationConstants.SchemaName,
columns: table => new
{
Id = table.Column<string>(type: "nvarchar(64)", maxLength: 64, nullable: false),
Url = table.Column<string>(type: "nvarchar(2000)", maxLength: 2000, nullable: false),
CallerService = table.Column<string>(type: "nvarchar(64)", maxLength: 64, nullable: false),
HttpStatusCode = table.Column<int>(type: "int", nullable: true),
Html = table.Column<string>(type: "nvarchar(max)", nullable: false),
Text = table.Column<string>(type: "nvarchar(max)", nullable: false),
DurationMs = table.Column<long>(type: "bigint", nullable: false),
Success = table.Column<bool>(type: "bit", nullable: false),
ErrorMessage = table.Column<string>(type: "nvarchar(2000)", maxLength: 2000, nullable: true),
CreatedAt = table.Column<DateTime>(type: "datetime2", nullable: false, defaultValueSql: "SYSUTCDATETIME()")
},
constraints: table =>
{
table.PrimaryKey("PK_PageFetches", x => x.Id);
});
migrationBuilder.CreateIndex(
name: "IX_PageFetches_CreatedAt",
schema: MigrationConstants.SchemaName,
table: "PageFetches",
column: "CreatedAt");
migrationBuilder.CreateIndex(
name: "IX_PageFetches_Url",
schema: MigrationConstants.SchemaName,
table: "PageFetches",
column: "Url");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "PageFetches",
schema: MigrationConstants.SchemaName);
}
}
}
Apis/page-fetcher-data/Migrations/PageFetchDbContextModelSnapshot.cs
+79
View File
@@ -0,0 +1,79 @@
// <auto-generated />
using System;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Metadata;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using PageFetcher.Data;
#nullable disable
namespace PageFetcher.Data.Migrations
{
[DbContext(typeof(PageFetchDbContext))]
partial class PageFetchDbContextModelSnapshot : ModelSnapshot
{
protected override void BuildModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasDefaultSchema("pageFetcher")
.HasAnnotation("ProductVersion", "10.0.7")
.HasAnnotation("Relational:MaxIdentifierLength", 128);
SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder);
modelBuilder.Entity("PageFetcher.Data.Entities.PageFetchEntity", b =>
{
b.Property<string>("Id")
.HasMaxLength(64)
.HasColumnType("nvarchar(64)");
b.Property<string>("CallerService")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("nvarchar(64)");
b.Property<DateTime>("CreatedAt")
.ValueGeneratedOnAdd()
.HasColumnType("datetime2")
.HasDefaultValueSql("SYSUTCDATETIME()");
b.Property<long>("DurationMs")
.HasColumnType("bigint");
b.Property<string>("ErrorMessage")
.HasMaxLength(2000)
.HasColumnType("nvarchar(2000)");
b.Property<string>("Html")
.IsRequired()
.HasColumnType("nvarchar(max)");
b.Property<int?>("HttpStatusCode")
.HasColumnType("int");
b.Property<bool>("Success")
.HasColumnType("bit");
b.Property<string>("Text")
.IsRequired()
.HasColumnType("nvarchar(max)");
b.Property<string>("Url")
.IsRequired()
.HasMaxLength(2000)
.HasColumnType("nvarchar(2000)");
b.HasKey("Id");
b.HasIndex("CreatedAt");
b.HasIndex("Url");
b.ToTable("PageFetches", "pageFetcher");
});
#pragma warning restore 612, 618
}
}
}
Apis/page-fetcher-data/PageFetchDbContext.cs
+45
View File
@@ -0,0 +1,45 @@
using Microsoft.EntityFrameworkCore;
using PageFetcher.Data.Entities;
namespace PageFetcher.Data;
/// <summary>
/// EF Core DbContext for the <c>pageFetcher</c> schema.
/// Owns the <c>PageFetches</c> audit table.
/// </summary>
public sealed class PageFetchDbContext : DbContext
{
public const string SchemaName = MigrationConstants.SchemaName;
public const string MigrationTableName = MigrationConstants.MigrationTableName;
public PageFetchDbContext(DbContextOptions<PageFetchDbContext> options) : base(options) { }
public DbSet<PageFetchEntity> PageFetches => Set<PageFetchEntity>();
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
{
base.OnConfiguring(optionsBuilder);
optionsBuilder.UseSqlServer(x => x.MigrationsHistoryTable(MigrationTableName, SchemaName));
}
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
modelBuilder.HasDefaultSchema(SchemaName);
modelBuilder.Entity<PageFetchEntity>(entity =>
{
entity.ToTable("PageFetches");
entity.HasKey(x => x.Id);
entity.Property(x => x.Id).HasMaxLength(64);
entity.Property(x => x.Url).HasMaxLength(2000).IsRequired();
entity.Property(x => x.CallerService).HasMaxLength(64).IsRequired();
entity.Property(x => x.Html).IsRequired();
entity.Property(x => x.Text).IsRequired();
entity.Property(x => x.ErrorMessage).HasMaxLength(2000);
entity.Property(x => x.CreatedAt).HasDefaultValueSql("SYSUTCDATETIME()");
entity.HasIndex(x => x.Url);
entity.HasIndex(x => x.CreatedAt);
});
}
}
Apis/page-fetcher-data/page-fetcher-data.csproj
+19
View File
@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<AssemblyName>page-fetcher-data</AssemblyName>
<RootNamespace>PageFetcher.Data</RootNamespace>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\shared-data\shared-data.csproj" />
</ItemGroup>
</Project>