From a83f6f705f308b390267f6b017c3fe0a3f5f9a29 Mon Sep 17 00:00:00 2001 From: claude Date: Mon, 8 Jun 2026 18:43:42 +0300 Subject: [PATCH] =?UTF-8?q?Remove=20UseHeadlessBrowser=20from=20JobProvide?= =?UTF-8?q?r=20=E2=80=94=20all=20fetches=20now=20go=20via=20page-fetcher-a?= =?UTF-8?q?pi?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit page-fetcher-api always uses Playwright (networkidle by default), so the per-provider flag that chose between headless and plain HTTP is obsolete. - Removed from JobProviderEntity, CvSearchDbContext, JobProviderConfig, JobTokenService - HtmlJobSearcher no longer passes WaitFor (uses page-fetcher-api default) - EF migration drops the column from cvSearch.JobProviders Co-Authored-By: Claude Sonnet 4.6 --- .../Settings/JobSearchSettings.cs | 2 - .../Services/JobTokenService.cs | 1 - Apis/cv-search-data/Data/CvSearchDbContext.cs | 1 - .../Data/Entities/JobProviderEntity.cs | 3 - ...54221_RemoveUseHeadlessBrowser.Designer.cs | 238 ++++++++++++++++++ ...20260608154221_RemoveUseHeadlessBrowser.cs | 32 +++ .../CvSearchDbContextModelSnapshot.cs | 5 - .../cv-search-job/Services/HtmlJobSearcher.cs | 1 - 8 files changed, 270 insertions(+), 13 deletions(-) create mode 100644 Apis/cv-search-data/Migrations/20260608154221_RemoveUseHeadlessBrowser.Designer.cs create mode 100644 Apis/cv-search-data/Migrations/20260608154221_RemoveUseHeadlessBrowser.cs diff --git a/Apis/cv-matcher-api-models/Settings/JobSearchSettings.cs b/Apis/cv-matcher-api-models/Settings/JobSearchSettings.cs index 63db298..9a2b907 100644 --- a/Apis/cv-matcher-api-models/Settings/JobSearchSettings.cs +++ b/Apis/cv-matcher-api-models/Settings/JobSearchSettings.cs @@ -21,8 +21,6 @@ public sealed class JobProviderConfig public string JobLinkContains { get; set; } = string.Empty; public List InitialKeywords { get; set; } = []; public int MaxResults { get; set; } = 20; - /// When true the scraper uses a headless Chromium browser to render JS-heavy pages. - public bool UseHeadlessBrowser { get; set; } /// /// When false, the Stage 2 anchor-text keyword filter is skipped. /// Set to false for providers whose search URL already filters by relevance server-side. diff --git a/Apis/cv-matcher-api/Services/JobTokenService.cs b/Apis/cv-matcher-api/Services/JobTokenService.cs index 5658b8f..c77d298 100644 --- a/Apis/cv-matcher-api/Services/JobTokenService.cs +++ b/Apis/cv-matcher-api/Services/JobTokenService.cs @@ -128,7 +128,6 @@ public sealed class JobTokenService : IJobTokenService JobLinkContains = entity.JobLinkContains, InitialKeywords = keywords, MaxResults = entity.MaxResults, - UseHeadlessBrowser = entity.UseHeadlessBrowser, RequireKeywordInAnchor = entity.RequireKeywordInAnchor }; } diff --git a/Apis/cv-search-data/Data/CvSearchDbContext.cs b/Apis/cv-search-data/Data/CvSearchDbContext.cs index 172d22d..6bc1133 100644 --- a/Apis/cv-search-data/Data/CvSearchDbContext.cs +++ b/Apis/cv-search-data/Data/CvSearchDbContext.cs @@ -79,7 +79,6 @@ public sealed class CvSearchDbContext : DbContext entity.Property(x => x.InitialKeywordsJson).HasMaxLength(2000).HasDefaultValue("[]").IsRequired(); entity.Property(x => x.MaxResults).HasDefaultValue(20); entity.Property(x => x.DisplayOrder).HasDefaultValue(0); - entity.Property(x => x.UseHeadlessBrowser).HasDefaultValue(false); }); } } diff --git a/Apis/cv-search-data/Data/Entities/JobProviderEntity.cs b/Apis/cv-search-data/Data/Entities/JobProviderEntity.cs index 1bdc641..cdc7c66 100644 --- a/Apis/cv-search-data/Data/Entities/JobProviderEntity.cs +++ b/Apis/cv-search-data/Data/Entities/JobProviderEntity.cs @@ -31,9 +31,6 @@ public sealed class JobProviderEntity /// Controls display ordering in future admin UIs. public int DisplayOrder { get; set; } - /// When true, the scraper renders the page with headless Chromium instead of a plain HTTP GET. - public bool UseHeadlessBrowser { get; set; } - /// /// When false, the Stage 2 anchor-text keyword filter is skipped. /// Set to false for providers whose search URL already filters by relevance server-side (ejobs.ro, bestjobs.eu). diff --git a/Apis/cv-search-data/Migrations/20260608154221_RemoveUseHeadlessBrowser.Designer.cs b/Apis/cv-search-data/Migrations/20260608154221_RemoveUseHeadlessBrowser.Designer.cs new file mode 100644 index 0000000..12e4cf4 --- /dev/null +++ b/Apis/cv-search-data/Migrations/20260608154221_RemoveUseHeadlessBrowser.Designer.cs @@ -0,0 +1,238 @@ +// +using System; +using CvSearch.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace CvSearch.Data.Migrations +{ + [DbContext(typeof(CvSearchDbContext))] + [Migration("20260608154221_RemoveUseHeadlessBrowser")] + partial class RemoveUseHeadlessBrowser + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasDefaultSchema("cvSearch") + .HasAnnotation("ProductVersion", "10.0.7") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("CvSearch.Data.Entities.JobProviderEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("DisplayOrder") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasDefaultValue(0); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("InitialKeywordsJson") + .IsRequired() + .ValueGeneratedOnAdd() + .HasMaxLength(2000) + .HasColumnType("nvarchar(2000)") + .HasDefaultValue("[]"); + + b.Property("JobLinkContains") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("MaxResults") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasDefaultValue(20); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("RequireKeywordInAnchor") + .HasColumnType("bit"); + + b.Property("SearchUrlTemplate") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.HasKey("Id"); + + b.ToTable("JobProviders", "cvSearch"); + }); + + modelBuilder.Entity("CvSearch.Data.Entities.JobSearchResultEntity", b => + { + b.Property("Id") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("JobText") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("JobTitle") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("JobUrl") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("nvarchar(2048)"); + + b.Property("ProviderName") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("ResultJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Score") + .HasColumnType("int"); + + b.Property("SessionId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("Id"); + + b.HasIndex("SessionId"); + + b.ToTable("JobSearchResults", "cvSearch"); + }); + + modelBuilder.Entity("CvSearch.Data.Entities.JobSearchSessionEntity", b => + { + b.Property("Id") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("CvDocumentId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Email") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("Keywords") + .IsRequired() + .HasMaxLength(1000) + .HasColumnType("nvarchar(1000)"); + + b.Property("Language") + .IsRequired() + .ValueGeneratedOnAdd() + .HasMaxLength(8) + .HasColumnType("nvarchar(8)") + .HasDefaultValue("en"); + + b.Property("Location") + .HasColumnType("nvarchar(max)"); + + b.Property("ProviderConfigJson") + .HasColumnType("nvarchar(max)"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("TokenId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("Id"); + + b.HasIndex("Status"); + + b.ToTable("JobSearchSessions", "cvSearch"); + }); + + modelBuilder.Entity("CvSearch.Data.Entities.JobSearchTokenEntity", b => + { + b.Property("Id") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("CvDocumentId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Email") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("ExpiresAt") + .HasColumnType("datetime2"); + + b.Property("Keywords") + .IsRequired() + .ValueGeneratedOnAdd() + .HasMaxLength(1000) + .HasColumnType("nvarchar(1000)") + .HasDefaultValue(""); + + b.Property("Language") + .IsRequired() + .ValueGeneratedOnAdd() + .HasMaxLength(8) + .HasColumnType("nvarchar(8)") + .HasDefaultValue("en"); + + b.Property("Location") + .HasColumnType("nvarchar(max)"); + + b.Property("Used") + .ValueGeneratedOnAdd() + .HasColumnType("bit") + .HasDefaultValue(false); + + b.HasKey("Id"); + + b.ToTable("JobSearchTokens", "cvSearch"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Apis/cv-search-data/Migrations/20260608154221_RemoveUseHeadlessBrowser.cs b/Apis/cv-search-data/Migrations/20260608154221_RemoveUseHeadlessBrowser.cs new file mode 100644 index 0000000..0fb30bf --- /dev/null +++ b/Apis/cv-search-data/Migrations/20260608154221_RemoveUseHeadlessBrowser.cs @@ -0,0 +1,32 @@ +using CvSearch.Data; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace CvSearch.Data.Migrations +{ + /// + public partial class RemoveUseHeadlessBrowser : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "UseHeadlessBrowser", + schema: MigrationConstants.SchemaName, + table: "JobProviders"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "UseHeadlessBrowser", + schema: MigrationConstants.SchemaName, + table: "JobProviders", + type: "bit", + nullable: false, + defaultValue: false); + } + } +} diff --git a/Apis/cv-search-data/Migrations/CvSearchDbContextModelSnapshot.cs b/Apis/cv-search-data/Migrations/CvSearchDbContextModelSnapshot.cs index 389ecd0..fdc51e0 100644 --- a/Apis/cv-search-data/Migrations/CvSearchDbContextModelSnapshot.cs +++ b/Apis/cv-search-data/Migrations/CvSearchDbContextModelSnapshot.cs @@ -69,11 +69,6 @@ namespace CvSearch.Data.Migrations .HasMaxLength(1024) .HasColumnType("nvarchar(1024)"); - b.Property("UseHeadlessBrowser") - .ValueGeneratedOnAdd() - .HasColumnType("bit") - .HasDefaultValue(false); - b.HasKey("Id"); b.ToTable("JobProviders", "cvSearch"); diff --git a/Jobs/cv-search-job/Services/HtmlJobSearcher.cs b/Jobs/cv-search-job/Services/HtmlJobSearcher.cs index b4f7e01..5c5b268 100644 --- a/Jobs/cv-search-job/Services/HtmlJobSearcher.cs +++ b/Jobs/cv-search-job/Services/HtmlJobSearcher.cs @@ -72,7 +72,6 @@ public sealed class HtmlJobSearcher var fetchResponse = await _pageFetcher.FetchAsync(new FetchPageRequest { Url = searchUrl, - WaitFor = provider.UseHeadlessBrowser ? "networkidle" : "domcontentloaded", CallerService = "cv-search-job" }, ct);