feat: add page-fetcher-api — centralised Playwright page fetcher
Introduces page-fetcher-api, a new internal ASP.NET Core service that centralises all web-page fetching through a single Playwright (headless Chromium) browser instance. All fetches are persisted to the pageFetcher SQL schema for auditing. New projects: - Apis/page-fetcher-api-models: FetchPageRequest, FetchPageResponse, IPageFetcherApiClient - Apis/page-fetcher-data: PageFetchDbContext, PageFetchEntity, InitialSchema migration (schema: pageFetcher) - Apis/page-fetcher-api: PlaywrightBrowserService (singleton), PageFetcherService, PageController Changes to existing services: - cv-matcher-api: JobTextExtractor now calls IPageFetcherApiClient instead of HttpClient - cv-search-job: HtmlJobSearcher uses IPageFetcherApiClient (removes inline Playwright); CvSearchJobTask fetches individual job pages and applies keyword pre-filter before LLM call; passes pre-fetched JobDescription to cv-matcher-api to skip re-fetch - common: add PageFetcherApiSettings - docker-compose.yml, build.yml: add new service + env vars for callers Closes #43 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,73 @@
|
||||
{
|
||||
"Serilog": {
|
||||
"Using": [
|
||||
"Serilog.Sinks.Console",
|
||||
"Serilog.Sinks.File"
|
||||
],
|
||||
"MinimumLevel": {
|
||||
"Default": "Information",
|
||||
"Override": {
|
||||
"Microsoft.AspNetCore": "Warning",
|
||||
"Microsoft.AspNetCore.Hosting": "Information",
|
||||
"Microsoft.AspNetCore.Routing": "Warning",
|
||||
"System.Net.Http.HttpClient": "Warning",
|
||||
"PageFetcherApi": "Information"
|
||||
}
|
||||
},
|
||||
"WriteTo": [
|
||||
{
|
||||
"Name": "Console",
|
||||
"Args": {
|
||||
"outputTemplate": "[{Timestamp:HH:mm:ss} {Level:u3}] {SourceContext}: {Message:lj}{NewLine}{Exception}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"Name": "File",
|
||||
"Args": {
|
||||
"path": "logs/page-fetcher-api-.log",
|
||||
"rollingInterval": "Day",
|
||||
"retainedFileCountLimit": 30,
|
||||
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] {SourceContext}: {Message:lj}{NewLine}{Exception}"
|
||||
}
|
||||
}
|
||||
],
|
||||
"Enrich": [
|
||||
"FromLogContext",
|
||||
"WithMachineName",
|
||||
"WithEnvironmentName"
|
||||
]
|
||||
},
|
||||
"Logging": {
|
||||
"LogLevel": {
|
||||
"Default": "Information",
|
||||
"Microsoft.AspNetCore": "Warning",
|
||||
"Microsoft.AspNetCore.Hosting": "Information",
|
||||
"Microsoft.AspNetCore.Routing": "Warning",
|
||||
"System.Net.Http.HttpClient": "Warning",
|
||||
"PageFetcherApi": "Information"
|
||||
}
|
||||
},
|
||||
"LogEnvironmentOnStartup": true,
|
||||
"AllowedHosts": "*",
|
||||
"KeyVault": {
|
||||
"VaultUri": "",
|
||||
"Enabled": false
|
||||
},
|
||||
"Database": {
|
||||
"Host": "localhost",
|
||||
"Port": 1433,
|
||||
"Name": "MyAiDb",
|
||||
"User": "sa",
|
||||
"Password": "",
|
||||
"TrustServerCertificate": true
|
||||
},
|
||||
"InternalApi": {
|
||||
"ApiKey": "",
|
||||
"RequireApiKey": true
|
||||
},
|
||||
"PageFetcher": {
|
||||
"DefaultWaitFor": "networkidle",
|
||||
"TimeoutSeconds": 30,
|
||||
"MaxTextChars": 60000
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user