Merge staging into production: deploy guard (invalid IMAGE_TAG fallback + version-stamped smoke)

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WpTYCBLH58XzrM7n3xPJ5N

.gitea/workflows/build.yml

@@ -23,6 +23,7 @@ env:
   CV_SEARCH_JOB_IMAGE: apps/myai-cv-search-job
   PAGE_FETCHER_API_IMAGE: apps/myai-page-fetcher-api
   IMAGE_TAG: ${{ github.ref_name }}        # branch name == image tag (staging | production)
+  WEB_PORT: "5140"                         # host port the web container is published on
 
 jobs:
   build:
@@ -60,7 +61,8 @@ jobs:
 
       - name: Build Web image
         run: |
-          docker build -f web/Dockerfile -t "${REGISTRY_HOST}/${WEB_IMAGE}:${IMAGE_TAG}" .
+          docker build --build-arg GIT_SHA="${{ github.sha }}" \
+            -f web/Dockerfile -t "${REGISTRY_HOST}/${WEB_IMAGE}:${IMAGE_TAG}" .
 
       - name: Build CV cleanup job image
         run: |
@@ -110,3 +112,69 @@ jobs:
         if: always()
         run: |
           docker image prune -f   # dangling only (keep base images)
+  # Building and pushing an image proves nothing about what the host is running.
+  # Watchtower pulls asynchronously, and for a month it was pulling a tag nobody
+  # intended -- with every run green, because no step ever asked the deployed site
+  # what it was serving. This job asks.
+  #
+  # It polls the deploy host directly on the LAN rather than the public hostname:
+  # the runner sits inside the network, only easysoft.ro has a staging equivalent in
+  # public DNS, and going direct also takes Caddy and any CDN out of the answer.
+  smoke:
+    runs-on: host
+    needs: build
+    steps:
+      - name: Wait for the deploy host to serve this commit
+        run: |
+          case "${{ github.ref_name }}" in
+            staging)    HOST=192.168.1.111 ;;
+            production) HOST=192.168.1.101 ;;
+            *) echo "::error::No deploy host mapped for '${{ github.ref_name }}'."; exit 1 ;;
+          esac
+
+          URL="http://${HOST}:${WEB_PORT}/version.json"
+          echo "Polling ${URL} for ${{ github.sha }}"
+
+          # 10 minutes: Watchtower's poke is fire-and-forget with a 30s fallback poll,
+          # and the container still has to start.
+          # ⚠️ Steps run under `bash -e -o pipefail`, so a polling loop has to be written
+          # defensively: the FIRST miss is the normal case, not an error.
+          #   - `curl -sf | sed` fails the whole pipeline under pipefail while the old
+          #     container is still up (404/connection refused), so `|| GOT=""` is required
+          #   - `[ test ] && { ... }` returns non-zero when the test fails, which under -e
+          #     aborts the step. Use `if`.
+          # Getting both wrong made the first run of this job fail in 20 seconds.
+          DEADLINE=$(( $(date +%s) + 600 ))
+          while :; do
+            GOT=$(curl -sf -m 15 "${URL}" 2>/dev/null | sed -n 's/.*"version":"\([^"]*\)".*/\1/p') || GOT=""
+
+            if [ "${GOT}" = "${{ github.sha }}" ]; then
+              echo "Serving ${GOT}."
+              break
+            fi
+
+            if [ "$(date +%s)" -ge "${DEADLINE}" ]; then
+              echo "::error::Timed out after 10m. ${HOST} is serving '${GOT:-nothing}', wanted ${{ github.sha }}."
+              echo "Either Watchtower never pulled the new image, the container failed to"
+              echo "start, or the stack's IMAGE_TAG does not match this branch."
+              exit 1
+            fi
+            echo "  still serving '${GOT:-nothing}' ..."
+            sleep 15
+          done
+
+      - name: Check the site actually answers
+        run: |
+          case "${{ github.ref_name }}" in
+            staging)    HOST=192.168.1.111 ;;
+            production) HOST=192.168.1.101 ;;
+          esac
+          # `|| CODE=000` for the same reason as above: curl exiting non-zero on a
+          # connection failure must produce a reportable code, not kill the step before
+          # it can say what went wrong. (`-s` without `-f` already tolerates 4xx/5xx.)
+          CODE=$(curl -s -o /dev/null -w '%{http_code}' -m 20 "http://${HOST}:${WEB_PORT}/") || CODE=000
+          if [ "${CODE}" != "200" ]; then
+            echo "::error::Home page returned ${CODE}."
+            exit 1
+          fi
+          echo "Home page 200."

docker-compose/docker-compose.yml

@@ -1,6 +1,14 @@
+# ⚠️ The IMAGE_TAG fallback is a DELIBERATELY INVALID tag, not `staging`.
+# On 2026-07-26 these stacks were recreated by hand and lost their environment
+# variables. The old `${IMAGE_TAG:-staging}` then quietly resolved to `staging`, so the
+# production host pulled staging images -- with no mail credentials and no recipient
+# addresses -- and served them for a month. Nothing failed, because falling back to a
+# real tag is indistinguishable from being configured. Now an unset IMAGE_TAG yields
+# `IMAGE_TAG-NOT-SET`, the pull fails with "manifest not found", the running container
+# is left untouched and the deploy goes red. Loud beats plausible.
 services:
   rag-api:
-    image: registry.easysoft.ro/apps/myai-rag-api:${IMAGE_TAG:-staging}
+    image: registry.easysoft.ro/apps/myai-rag-api:${IMAGE_TAG:-IMAGE_TAG-NOT-SET}
     container_name: myai-rag-api
     environment:
       - ASPNETCORE_ENVIRONMENT=${ASPNETCORE_ENVIRONMENT:-Staging}
@@ -50,7 +58,7 @@ services:
       - "com.centurylinklabs.watchtower.enable=true"
 
   cv-matcher-api:
-    image: registry.easysoft.ro/apps/myai-cv-matcher-api:${IMAGE_TAG:-staging}
+    image: registry.easysoft.ro/apps/myai-cv-matcher-api:${IMAGE_TAG:-IMAGE_TAG-NOT-SET}
     container_name: myai-cv-matcher-api
     depends_on:
       - rag-api
@@ -102,7 +110,7 @@ services:
       - "com.centurylinklabs.watchtower.enable=true"
 
   email-api:
-    image: registry.easysoft.ro/apps/myai-email-api:${IMAGE_TAG:-staging}
+    image: registry.easysoft.ro/apps/myai-email-api:${IMAGE_TAG:-IMAGE_TAG-NOT-SET}
     container_name: myai-email-api
     environment:
       - ASPNETCORE_ENVIRONMENT=${ASPNETCORE_ENVIRONMENT:-Staging}
@@ -143,7 +151,7 @@ services:
       - "com.centurylinklabs.watchtower.enable=true"
 
   api:
-    image: registry.easysoft.ro/apps/myai-api:${IMAGE_TAG:-staging}
+    image: registry.easysoft.ro/apps/myai-api:${IMAGE_TAG:-IMAGE_TAG-NOT-SET}
     container_name: myai-api
     depends_on:
       - cv-matcher-api
@@ -217,7 +225,7 @@ services:
       - "com.centurylinklabs.watchtower.enable=true"
 
   cv-cleanup-job:
-    image: registry.easysoft.ro/apps/myai-cv-cleanup-job:${IMAGE_TAG:-staging}
+    image: registry.easysoft.ro/apps/myai-cv-cleanup-job:${IMAGE_TAG:-IMAGE_TAG-NOT-SET}
     container_name: myai-cv-cleanup-job
     depends_on:
       - api
@@ -247,7 +255,7 @@ services:
       - "com.centurylinklabs.watchtower.enable=true"
 
   cv-search-job:
-    image: registry.easysoft.ro/apps/myai-cv-search-job:${IMAGE_TAG:-staging}
+    image: registry.easysoft.ro/apps/myai-cv-search-job:${IMAGE_TAG:-IMAGE_TAG-NOT-SET}
     container_name: myai-cv-search-job
     depends_on:
       - cv-matcher-api
@@ -300,7 +308,7 @@ services:
       - "com.centurylinklabs.watchtower.enable=true"
 
   page-fetcher-api:
-    image: registry.easysoft.ro/apps/myai-page-fetcher-api:${IMAGE_TAG:-staging}
+    image: registry.easysoft.ro/apps/myai-page-fetcher-api:${IMAGE_TAG:-IMAGE_TAG-NOT-SET}
     container_name: myai-page-fetcher-api
     environment:
       - ASPNETCORE_ENVIRONMENT=${ASPNETCORE_ENVIRONMENT:-Staging}
@@ -332,7 +340,7 @@ services:
       - "com.centurylinklabs.watchtower.enable=true"
 
   web:
-    image: registry.easysoft.ro/apps/myai-web:${IMAGE_TAG:-staging}
+    image: registry.easysoft.ro/apps/myai-web:${IMAGE_TAG:-IMAGE_TAG-NOT-SET}
     container_name: myai-web
     depends_on:
       - api

web/Dockerfile

@@ -16,4 +16,13 @@ EXPOSE 8080
 ENV ASPNETCORE_URLS=http://0.0.0.0:8080
 
 COPY --from=build /app/publish .
+
+# Stamp the commit into the image so a deploy can be verified from the outside.
+#
+# Without this a smoke test can only ask "does the site return 200?" -- which it did
+# throughout the month production was quietly serving staging images. A status check
+# cannot tell one build from another; /version.json can, so the smoke job refuses to
+# pass until the host is actually serving THIS commit.
+ARG GIT_SHA=unknown
+RUN mkdir -p wwwroot && printf '{"version":"%s"}' "$GIT_SHA" > wwwroot/version.json
 ENTRYPOINT ["dotnet", "web.dll"]