Merge staging into production: drop the throwaway rebuild comment

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WpTYCBLH58XzrM7n3xPJ5N

.gitea/workflows/build.yml

@@ -1,13 +1,19 @@
-name: Build and Push Docker Images Staging
+name: Build and Push Docker Images
 
+# Branch-driven deploys — no yaml edits to switch environment:
+#   merge into `staging`    -> tag :staging    (staging Watchtower deploys)
+#   merge into `production` -> tag :production (production Watchtower deploys)
+# `main` is the day-to-day work branch and deploys nothing.
 on:
   push:
     branches:
       - staging
+      - production
 
 env:
-  GIT_HOST: git.easysoft.ro
+  GIT_HOST: docker-git.easysoft.ro
   REGISTRY_HOST: registry.easysoft.ro
+  DOCKER_BUILDKIT: "1"
   API_IMAGE: apps/myai-api
   CV_MATCHER_API_IMAGE: apps/myai-cv-matcher-api
   RAG_API_IMAGE: apps/myai-rag-api
@@ -16,18 +22,20 @@ env:
   CV_CLEANUP_JOB_IMAGE: apps/myai-cv-cleanup-job
   CV_SEARCH_JOB_IMAGE: apps/myai-cv-search-job
   PAGE_FETCHER_API_IMAGE: apps/myai-page-fetcher-api
-  IMAGE_TAG: staging
+  IMAGE_TAG: ${{ github.ref_name }}        # branch name == image tag (staging | production)
+  WEB_PORT: "5140"                         # host port the web container is published on
 
 jobs:
   build:
     runs-on: host
 
     steps:
-      - name: Checkout repository
+      - name: Checkout the pushed commit
         env:
           TOKEN: ${{ secrets.REPO_TOKEN }}
         run: |
           git clone "http://gelu:${TOKEN}@${GIT_HOST}:3000/${GITHUB_REPOSITORY}.git" .
+          git checkout "${{ github.sha }}"
 
       - name: Login to registry
         run: |
@@ -53,7 +61,8 @@ jobs:
 
       - name: Build Web image
         run: |
-          docker build -f web/Dockerfile -t "${REGISTRY_HOST}/${WEB_IMAGE}:${IMAGE_TAG}" .
+          docker build --build-arg GIT_SHA="${{ github.sha }}" \
+            -f web/Dockerfile -t "${REGISTRY_HOST}/${WEB_IMAGE}:${IMAGE_TAG}" .
 
       - name: Build CV cleanup job image
         run: |
@@ -99,8 +108,97 @@ jobs:
         run: |
           docker push "${REGISTRY_HOST}/${PAGE_FETCHER_API_IMAGE}:${IMAGE_TAG}"
 
-      - name: Reclaim disk space
+      # Watchtower's poll is the fallback, not the mechanism: 30s on staging but 300s on
+      # production, so without this a green run can sit five minutes ahead of the deploy it
+      # claims to have made. Copied from easyDent, including the soft failure -- an unset
+      # secret or an unreachable API must degrade to the poll, never fail the build.
+      - name: Trigger Watchtower redeploy
+        env:
+          URL_STAGING: ${{ secrets.WATCHTOWER_URL_STAGING }}
+          URL_PRODUCTION: ${{ secrets.WATCHTOWER_URL_PRODUCTION }}
+          TOKEN: ${{ secrets.WATCHTOWER_TOKEN }}
+        run: |
+          if [ "${IMAGE_TAG}" = "production" ]; then URL="${URL_PRODUCTION}"; else URL="${URL_STAGING}"; fi
+          if [ -n "${URL}" ] && [ -n "${TOKEN}" ]; then
+            echo "Triggering Watchtower at ${URL}"
+            curl -sf -m 30 -H "Authorization: Bearer ${TOKEN}" "${URL}" && echo "  -> redeploy triggered" \
+              || echo "  -> trigger failed; Watchtower will still pick it up on the next poll"
+          else
+            echo "Watchtower push-trigger not configured (WATCHTOWER_* secrets unset); relying on the poll interval."
+          fi
+
+      - name: Reclaim disk space (keep recent build cache)
         if: always()
         run: |
-          docker image prune -af
+          docker image prune -f   # dangling only (keep base images)
-          docker builder prune -af
+  # Building and pushing an image proves nothing about what the host is running.
+  # Watchtower pulls asynchronously, and for a month it was pulling a tag nobody
+  # intended -- with every run green, because no step ever asked the deployed site
+  # what it was serving. This job asks.
+  #
+  # It polls the deploy host directly on the LAN rather than the public hostname:
+  # the runner sits inside the network, only easysoft.ro has a staging equivalent in
+  # public DNS, and going direct also takes Caddy and any CDN out of the answer.
+  smoke:
+    runs-on: host
+    needs: build
+    steps:
+      - name: Wait for the deploy host to serve this commit
+        run: |
+          case "${{ github.ref_name }}" in
+            staging)    HOST=192.168.1.111 ;;
+            production) HOST=192.168.1.101 ;;
+            *) echo "::error::No deploy host mapped for '${{ github.ref_name }}'."; exit 1 ;;
+          esac
+
+          URL="http://${HOST}:${WEB_PORT}/version.json"
+          echo "Polling ${URL} for ${{ github.sha }}"
+
+          # 10 minutes: Watchtower's poke is fire-and-forget with a 30s fallback poll,
+          # and the container still has to start.
+          # ⚠️ Steps run under `bash -e -o pipefail`, so a polling loop has to be written
+          # defensively: the FIRST miss is the normal case, not an error.
+          #   - `curl -sf | sed` fails the whole pipeline under pipefail while the old
+          #     container is still up (404/connection refused), so `|| GOT=""` is required
+          #   - `[ test ] && { ... }` returns non-zero when the test fails, which under -e
+          #     aborts the step. Use `if`.
+          # Getting both wrong made the first run of this job fail in 20 seconds.
+          DEADLINE=$(( $(date +%s) + 600 ))
+          while :; do
+            GOT=$(curl -sf -m 15 "${URL}" 2>/dev/null | sed -n 's/.*"version":"\([^"]*\)".*/\1/p') || GOT=""
+
+            if [ "${GOT}" = "${{ github.sha }}" ]; then
+              echo "Serving ${GOT}."
+              break
+            fi
+
+            if [ "$(date +%s)" -ge "${DEADLINE}" ]; then
+              echo "::error::Timed out after 10m. ${HOST} is serving '${GOT:-nothing}', wanted ${{ github.sha }}."
+              echo "Either Watchtower never pulled the new image, the container failed to"
+              echo "start, or the stack's IMAGE_TAG does not match this branch."
+              exit 1
+            fi
+            echo "  still serving '${GOT:-nothing}' ..."
+            sleep 15
+          done
+
+      - name: Check the site actually answers
+        run: |
+          case "${{ github.ref_name }}" in
+            staging)    HOST=192.168.1.111 ;;
+            production) HOST=192.168.1.101 ;;
+          esac
+          # `-L` follows redirects and we assert on the FINAL code, because a 302 from `/`
+          # is a healthy answer for a site running in UnderConstruction mode -- it means the
+          # app is up and routing. Asserting a bare 200 failed jecreativ.ro's first
+          # production deploy for doing exactly what it was configured to do.
+          #
+          # `|| CODE=000` for the same reason as above: curl exiting non-zero on a
+          # connection failure must produce a reportable code, not kill the step before
+          # it can say what went wrong. (`-s` without `-f` already tolerates 4xx/5xx.)
+          CODE=$(curl -sL -o /dev/null -w '%{http_code}' -m 20 "http://${HOST}:${WEB_PORT}/") || CODE=000
+          if [ "${CODE}" != "200" ]; then
+            echo "::error::Home page returned ${CODE}."
+            exit 1
+          fi
+          echo "Home page 200."

.gitignore

@@ -376,3 +376,6 @@ files/
 
 /docker-compose/.env.production
 /docker-compose/.env.staging
+
+# local infra access notes (secrets) — never commit
+ACCESS.md

CLAUDE.md

@@ -34,11 +34,15 @@ This applies to both the staging and production repos as appropriate.
 - .NET 10, ASP.NET Core, Worker Service
 - Entity Framework Core + SQL Server (multi-schema)
 - Refit for typed HTTP clients between services
-- Serilog (JSON structured logging, Console + File + Email sinks)
+- Serilog — Compact JSON logs to stdout (+ optional email sink); see Observability
 - MailKit for SMTP (used exclusively in `email-api`)
 - Docker Compose for local and production deployment
 - Watchtower for automatic container updates in production
 
+## Observability (central stack on monitoring host 10.0.0.156)
+- **Logs**: every service uses `ConfigureJsonSerilog(ServiceName, appVersion)` (startup-helpers) → Serilog **Compact JSON** to stdout, enriched `Application`/`Environment`/`AppVersion`. The host's Grafana **Alloy** agent ships stdout → **Loki**; view/query in Grafana. No file sink; optional email sink only if `SerilogEmail:*` is configured.
+- **No app metrics/traces** — these are simple/minimal services, so (unlike easyDent) they don't expose Prometheus metrics or OTLP traces. Container/host metrics still come from the host's cAdvisor/node_exporter.
+
 ## Project taxonomy
 
 | Category | Naming | Contains | EF dependency |

Directory.Packages.props

@@ -23,6 +23,7 @@
     <PackageVersion Include="Serilog.AspNetCore" Version="10.0.0" />
     <PackageVersion Include="Serilog.Enrichers.Environment" Version="3.0.1" />
     <PackageVersion Include="Serilog.Sinks.Console" Version="6.1.1" />
+    <PackageVersion Include="Serilog.Formatting.Compact" Version="3.0.0" />
     <PackageVersion Include="Serilog.Sinks.Email" Version="4.2.1" />
     <PackageVersion Include="Serilog.Sinks.File" Version="7.0.0" />
     <!-- Swagger -->

Helpers/startup-helpers/StartupExtensions.cs

@@ -39,11 +39,10 @@ public static class StartupExtensions
                 .ReadFrom.Configuration(context.Configuration)
                 .ReadFrom.Services(services)
                 .Enrich.FromLogContext()
-                .Enrich.WithMachineName()
+                .Enrich.WithProperty("Application", serviceName)
-                .Enrich.WithEnvironmentName()
+                .Enrich.WithProperty("Environment", context.HostingEnvironment.EnvironmentName)
-                .Enrich.WithProperty("Service", serviceName)
                 .Enrich.WithProperty("AppVersion", appVersion)
-                .WriteTo.Console(new Serilog.Formatting.Json.JsonFormatter());
+                .WriteTo.Console(new Serilog.Formatting.Compact.CompactJsonFormatter());
 
             AddEmailSinkIfConfigured(configuration, context.Configuration, serviceName);
         });
@@ -57,11 +56,10 @@ public static class StartupExtensions
                 .ReadFrom.Configuration(builder.Configuration)
                 .ReadFrom.Services(services)
                 .Enrich.FromLogContext()
-                .Enrich.WithMachineName()
+                .Enrich.WithProperty("Application", serviceName)
-                .Enrich.WithEnvironmentName()
+                .Enrich.WithProperty("Environment", builder.Environment.EnvironmentName)
-                .Enrich.WithProperty("Service", serviceName)
                 .Enrich.WithProperty("AppVersion", appVersion)
-                .WriteTo.Console(new Serilog.Formatting.Json.JsonFormatter());
+                .WriteTo.Console(new Serilog.Formatting.Compact.CompactJsonFormatter());
 
             AddEmailSinkIfConfigured(configuration, builder.Configuration, serviceName);
         });

Helpers/startup-helpers/startup-helpers.csproj

@@ -17,6 +17,7 @@
     <PackageReference Include="DotNetEnv" />
     <PackageReference Include="Serilog.AspNetCore" />
     <PackageReference Include="Serilog.Enrichers.Environment" />
+    <PackageReference Include="Serilog.Formatting.Compact" />
     <PackageReference Include="Serilog.Sinks.Email" />
     <PackageReference Include="Serilog.Sinks.File" />
     <PackageReference Include="Swashbuckle.AspNetCore" />

README.md

@@ -1,20 +1,27 @@
-# Introduction 
+# myAi
-TODO: Give a short introduction of your project. Let this section explain the objectives or the motivation behind this project. 
 
-# Getting Started
+The **myai.ro** platform — a set of .NET microservices (CV matching, RAG, email, CV search, page
-TODO: Guide users through getting your code up and running on their own system. In this section you can talk about:
+fetching, …) behind a web frontend + API. Part of the easySoft platform.
-1.	Installation process
-2.	Software dependencies
-3.	Latest releases
-4.	API references
 
-# Build and Test
+## Layout
-TODO: Describe and show how to build your code and run the tests. 
+Multiple services (`*-api`, `*-job`) + `web`, sharing a common bootstrap in
+`startup-helpers/` (Serilog, Swagger, `.env`/Key Vault loading, middleware). See **CLAUDE.md**
+for the full service map, dependency chain, and conventions.
 
-# Contribute
+## Run locally
-TODO: Explain how other users and developers can contribute to make your code better. 
+```bash
+docker compose up --build      # or run individual services with: dotnet run --project <svc>
+```
 
-If you want to learn more about creating good readme files then refer the following [guidelines](https://docs.microsoft.com/en-us/azure/devops/repos/git/create-a-readme?view=azure-devops). You can also seek inspiration from the below readme files:
+## Deploy
-- [ASP.NET Core](https://github.com/aspnet/Home)
+CI builds `registry.easysoft.ro/apps/myai-*:{staging,production}`; Watchtower rolls them out to
-- [Visual Studio Code](https://github.com/Microsoft/vscode)
+the **staging (`10.0.0.183`)** + **production (`10.0.0.248`)** Portainer stacks. Edge Caddy serves
-- [Chakra Core](https://github.com/Microsoft/ChakraCore)
+**myai.ro** (prod) / **myai.easysoft.ro** (staging).
+
+## Logging
+Every service: `ConfigureJsonSerilog(name, version)` → Serilog **Compact JSON** to stdout → Grafana
+**Alloy** → **Loki**. No app metrics/traces (simple services).
+
+---
+See **CLAUDE.md** for the detailed solution guide and **ACCESS.md** (local, gitignored) for
+infrastructure access.

docker-compose/.env.template

@@ -49,8 +49,10 @@ Ai__Ollama__ChatModel=llama2
 Ai__Ollama__EmbeddingModel=embedding-model
 Ai__Ollama__TimeoutSeconds=30
 
-# Database (shared) - maps to Database:Host etc. used by apps
+# Database (shared) - maps to Database:Host etc. used by apps.
-Database__Host=sqlserver
+# Deployed (staging/prod) uses the LAN DNS name (resolves to the MSSQL VM 10.0.0.240);
+# for local dev leave it unset to use the docker-compose 'sqlserver' service default.
+Database__Host=mssql.easysoft.ro
 Database__Port=1433
 Database__Name=MyAiDb
 Database__User=sa

docker-compose/docker-compose.yml

@@ -1,6 +1,14 @@
+# ⚠️ The IMAGE_TAG fallback is a DELIBERATELY INVALID tag, not `staging`.
+# On 2026-07-26 these stacks were recreated by hand and lost their environment
+# variables. The old `${IMAGE_TAG:-staging}` then quietly resolved to `staging`, so the
+# production host pulled staging images -- with no mail credentials and no recipient
+# addresses -- and served them for a month. Nothing failed, because falling back to a
+# real tag is indistinguishable from being configured. Now an unset IMAGE_TAG yields
+# `IMAGE_TAG-NOT-SET`, the pull fails with "manifest not found", the running container
+# is left untouched and the deploy goes red. Loud beats plausible.
 services:
   rag-api:
-    image: registry.easysoft.ro/apps/myai-rag-api:${IMAGE_TAG:-staging}
+    image: registry.easysoft.ro/apps/myai-rag-api:${IMAGE_TAG:-IMAGE_TAG-NOT-SET}
     container_name: myai-rag-api
     environment:
       - ASPNETCORE_ENVIRONMENT=${ASPNETCORE_ENVIRONMENT:-Staging}
@@ -50,7 +58,7 @@ services:
       - "com.centurylinklabs.watchtower.enable=true"
 
   cv-matcher-api:
-    image: registry.easysoft.ro/apps/myai-cv-matcher-api:${IMAGE_TAG:-staging}
+    image: registry.easysoft.ro/apps/myai-cv-matcher-api:${IMAGE_TAG:-IMAGE_TAG-NOT-SET}
     container_name: myai-cv-matcher-api
     depends_on:
       - rag-api
@@ -102,7 +110,7 @@ services:
       - "com.centurylinklabs.watchtower.enable=true"
 
   email-api:
-    image: registry.easysoft.ro/apps/myai-email-api:${IMAGE_TAG:-staging}
+    image: registry.easysoft.ro/apps/myai-email-api:${IMAGE_TAG:-IMAGE_TAG-NOT-SET}
     container_name: myai-email-api
     environment:
       - ASPNETCORE_ENVIRONMENT=${ASPNETCORE_ENVIRONMENT:-Staging}
@@ -143,7 +151,7 @@ services:
       - "com.centurylinklabs.watchtower.enable=true"
 
   api:
-    image: registry.easysoft.ro/apps/myai-api:${IMAGE_TAG:-staging}
+    image: registry.easysoft.ro/apps/myai-api:${IMAGE_TAG:-IMAGE_TAG-NOT-SET}
     container_name: myai-api
     depends_on:
       - cv-matcher-api
@@ -217,7 +225,7 @@ services:
       - "com.centurylinklabs.watchtower.enable=true"
 
   cv-cleanup-job:
-    image: registry.easysoft.ro/apps/myai-cv-cleanup-job:${IMAGE_TAG:-staging}
+    image: registry.easysoft.ro/apps/myai-cv-cleanup-job:${IMAGE_TAG:-IMAGE_TAG-NOT-SET}
     container_name: myai-cv-cleanup-job
     depends_on:
       - api
@@ -247,7 +255,7 @@ services:
       - "com.centurylinklabs.watchtower.enable=true"
 
   cv-search-job:
-    image: registry.easysoft.ro/apps/myai-cv-search-job:${IMAGE_TAG:-staging}
+    image: registry.easysoft.ro/apps/myai-cv-search-job:${IMAGE_TAG:-IMAGE_TAG-NOT-SET}
     container_name: myai-cv-search-job
     depends_on:
       - cv-matcher-api
@@ -300,7 +308,7 @@ services:
       - "com.centurylinklabs.watchtower.enable=true"
 
   page-fetcher-api:
-    image: registry.easysoft.ro/apps/myai-page-fetcher-api:${IMAGE_TAG:-staging}
+    image: registry.easysoft.ro/apps/myai-page-fetcher-api:${IMAGE_TAG:-IMAGE_TAG-NOT-SET}
     container_name: myai-page-fetcher-api
     environment:
       - ASPNETCORE_ENVIRONMENT=${ASPNETCORE_ENVIRONMENT:-Staging}
@@ -332,7 +340,7 @@ services:
       - "com.centurylinklabs.watchtower.enable=true"
 
   web:
-    image: registry.easysoft.ro/apps/myai-web:${IMAGE_TAG:-staging}
+    image: registry.easysoft.ro/apps/myai-web:${IMAGE_TAG:-IMAGE_TAG-NOT-SET}
     container_name: myai-web
     depends_on:
       - api

web/Dockerfile

@@ -16,4 +16,13 @@ EXPOSE 8080
 ENV ASPNETCORE_URLS=http://0.0.0.0:8080
 
 COPY --from=build /app/publish .
+
+# Stamp the commit into the image so a deploy can be verified from the outside.
+#
+# Without this a smoke test can only ask "does the site return 200?" -- which it did
+# throughout the month production was quietly serving staging images. A status check
+# cannot tell one build from another; /version.json can, so the smoke job refuses to
+# pass until the host is actually serving THIS commit.
+ARG GIT_SHA=unknown
+RUN mkdir -p wwwroot && printf '{"version":"%s"}' "$GIT_SHA" > wwwroot/version.json
 ENTRYPOINT ["dotnet", "web.dll"]

web/wwwroot/cv-matcher/index.html

@@ -139,13 +139,13 @@
                             <div>
                                 <span data-i18n="contact.phone">Phone</span>
                                 <strong>
-                                    <a href="tel:+40722523764">+40 722-523-764</a>
+                                    <a href="tel:+40774885011">+40 774-885-011</a>
                                 </strong>
                             </div>
                             <div>
                                 <span>WhatsApp</span>
                                 <strong>
-                                    <a href="https://wa.me/40722523764" target="_blank" rel="noreferrer">+40 722-523-764</a>
+                                    <a href="https://wa.me/40774885011" target="_blank" rel="noreferrer">+40 774-885-011</a>
                                 </strong>
                             </div>
 

web/wwwroot/index.html

@@ -126,13 +126,13 @@
                             <div>
                                 <span data-i18n="contact.phone">Phone</span>
                                 <strong>
-                                    <a href="tel:+40722523764">+40 722-523-764</a>
+                                    <a href="tel:+40774885011">+40 774-885-011</a>
                                 </strong>
                             </div>
                             <div>
                                 <span>WhatsApp</span>
                                 <strong>
-                                    <a href="https://wa.me/40722523764" target="_blank" rel="noreferrer">+40 722-523-764</a>
+                                    <a href="https://wa.me/40774885011" target="_blank" rel="noreferrer">+40 774-885-011</a>
                                 </strong>
                             </div>
                         </div>