From dee6c5a20515f9d9350741ab377d523da1c3878b Mon Sep 17 00:00:00 2001 From: Gelu Date: Mon, 24 Aug 2026 10:21:18 +0000 Subject: [PATCH] ci: make the branch strategy enforced instead of merely intended Same guard as the four site repos. myAi was the ONLY stack that kept its environment variables through the 26 July migration, so it alone deployed correctly all along -- but it shares the ${IMAGE_TAG:-staging} default that made the others fail silently, so it gets the same treatment. 1. ${IMAGE_TAG:-staging} -> ${IMAGE_TAG:-IMAGE_TAG-NOT-SET} on all eight images: an unconfigured stack fails the pull rather than quietly becoming staging. 2. A smoke job that asks the deploy host what commit it serves, via a /version.json stamped into the web image at build time. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WpTYCBLH58XzrM7n3xPJ5N --- .gitea/workflows/build.yml | 70 ++++++++++++++++++++++++++++++- docker-compose/docker-compose.yml | 24 +++++++---- web/Dockerfile | 9 ++++ 3 files changed, 94 insertions(+), 9 deletions(-) diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index 558ddaf..98540c5 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -23,6 +23,7 @@ env: CV_SEARCH_JOB_IMAGE: apps/myai-cv-search-job PAGE_FETCHER_API_IMAGE: apps/myai-page-fetcher-api IMAGE_TAG: ${{ github.ref_name }} # branch name == image tag (staging | production) + WEB_PORT: "5140" # host port the web container is published on jobs: build: @@ -60,7 +61,8 @@ jobs: - name: Build Web image run: | - docker build -f web/Dockerfile -t "${REGISTRY_HOST}/${WEB_IMAGE}:${IMAGE_TAG}" . + docker build --build-arg GIT_SHA="${{ github.sha }}" \ + -f web/Dockerfile -t "${REGISTRY_HOST}/${WEB_IMAGE}:${IMAGE_TAG}" . - name: Build CV cleanup job image run: | @@ -110,3 +112,69 @@ jobs: if: always() run: | docker image prune -f # dangling only (keep base images) + # Building and pushing an image proves nothing about what the host is running. + # Watchtower pulls asynchronously, and for a month it was pulling a tag nobody + # intended -- with every run green, because no step ever asked the deployed site + # what it was serving. This job asks. + # + # It polls the deploy host directly on the LAN rather than the public hostname: + # the runner sits inside the network, only easysoft.ro has a staging equivalent in + # public DNS, and going direct also takes Caddy and any CDN out of the answer. + smoke: + runs-on: host + needs: build + steps: + - name: Wait for the deploy host to serve this commit + run: | + case "${{ github.ref_name }}" in + staging) HOST=192.168.1.111 ;; + production) HOST=192.168.1.101 ;; + *) echo "::error::No deploy host mapped for '${{ github.ref_name }}'."; exit 1 ;; + esac + + URL="http://${HOST}:${WEB_PORT}/version.json" + echo "Polling ${URL} for ${{ github.sha }}" + + # 10 minutes: Watchtower's poke is fire-and-forget with a 30s fallback poll, + # and the container still has to start. + # ⚠️ Steps run under `bash -e -o pipefail`, so a polling loop has to be written + # defensively: the FIRST miss is the normal case, not an error. + # - `curl -sf | sed` fails the whole pipeline under pipefail while the old + # container is still up (404/connection refused), so `|| GOT=""` is required + # - `[ test ] && { ... }` returns non-zero when the test fails, which under -e + # aborts the step. Use `if`. + # Getting both wrong made the first run of this job fail in 20 seconds. + DEADLINE=$(( $(date +%s) + 600 )) + while :; do + GOT=$(curl -sf -m 15 "${URL}" 2>/dev/null | sed -n 's/.*"version":"\([^"]*\)".*/\1/p') || GOT="" + + if [ "${GOT}" = "${{ github.sha }}" ]; then + echo "Serving ${GOT}." + break + fi + + if [ "$(date +%s)" -ge "${DEADLINE}" ]; then + echo "::error::Timed out after 10m. ${HOST} is serving '${GOT:-nothing}', wanted ${{ github.sha }}." + echo "Either Watchtower never pulled the new image, the container failed to" + echo "start, or the stack's IMAGE_TAG does not match this branch." + exit 1 + fi + echo " still serving '${GOT:-nothing}' ..." + sleep 15 + done + + - name: Check the site actually answers + run: | + case "${{ github.ref_name }}" in + staging) HOST=192.168.1.111 ;; + production) HOST=192.168.1.101 ;; + esac + # `|| CODE=000` for the same reason as above: curl exiting non-zero on a + # connection failure must produce a reportable code, not kill the step before + # it can say what went wrong. (`-s` without `-f` already tolerates 4xx/5xx.) + CODE=$(curl -s -o /dev/null -w '%{http_code}' -m 20 "http://${HOST}:${WEB_PORT}/") || CODE=000 + if [ "${CODE}" != "200" ]; then + echo "::error::Home page returned ${CODE}." + exit 1 + fi + echo "Home page 200." diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index f2d7274..42c1fcb 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -1,6 +1,14 @@ +# ⚠️ The IMAGE_TAG fallback is a DELIBERATELY INVALID tag, not `staging`. +# On 2026-07-26 these stacks were recreated by hand and lost their environment +# variables. The old `${IMAGE_TAG:-staging}` then quietly resolved to `staging`, so the +# production host pulled staging images -- with no mail credentials and no recipient +# addresses -- and served them for a month. Nothing failed, because falling back to a +# real tag is indistinguishable from being configured. Now an unset IMAGE_TAG yields +# `IMAGE_TAG-NOT-SET`, the pull fails with "manifest not found", the running container +# is left untouched and the deploy goes red. Loud beats plausible. services: rag-api: - image: registry.easysoft.ro/apps/myai-rag-api:${IMAGE_TAG:-staging} + image: registry.easysoft.ro/apps/myai-rag-api:${IMAGE_TAG:-IMAGE_TAG-NOT-SET} container_name: myai-rag-api environment: - ASPNETCORE_ENVIRONMENT=${ASPNETCORE_ENVIRONMENT:-Staging} @@ -50,7 +58,7 @@ services: - "com.centurylinklabs.watchtower.enable=true" cv-matcher-api: - image: registry.easysoft.ro/apps/myai-cv-matcher-api:${IMAGE_TAG:-staging} + image: registry.easysoft.ro/apps/myai-cv-matcher-api:${IMAGE_TAG:-IMAGE_TAG-NOT-SET} container_name: myai-cv-matcher-api depends_on: - rag-api @@ -102,7 +110,7 @@ services: - "com.centurylinklabs.watchtower.enable=true" email-api: - image: registry.easysoft.ro/apps/myai-email-api:${IMAGE_TAG:-staging} + image: registry.easysoft.ro/apps/myai-email-api:${IMAGE_TAG:-IMAGE_TAG-NOT-SET} container_name: myai-email-api environment: - ASPNETCORE_ENVIRONMENT=${ASPNETCORE_ENVIRONMENT:-Staging} @@ -143,7 +151,7 @@ services: - "com.centurylinklabs.watchtower.enable=true" api: - image: registry.easysoft.ro/apps/myai-api:${IMAGE_TAG:-staging} + image: registry.easysoft.ro/apps/myai-api:${IMAGE_TAG:-IMAGE_TAG-NOT-SET} container_name: myai-api depends_on: - cv-matcher-api @@ -217,7 +225,7 @@ services: - "com.centurylinklabs.watchtower.enable=true" cv-cleanup-job: - image: registry.easysoft.ro/apps/myai-cv-cleanup-job:${IMAGE_TAG:-staging} + image: registry.easysoft.ro/apps/myai-cv-cleanup-job:${IMAGE_TAG:-IMAGE_TAG-NOT-SET} container_name: myai-cv-cleanup-job depends_on: - api @@ -247,7 +255,7 @@ services: - "com.centurylinklabs.watchtower.enable=true" cv-search-job: - image: registry.easysoft.ro/apps/myai-cv-search-job:${IMAGE_TAG:-staging} + image: registry.easysoft.ro/apps/myai-cv-search-job:${IMAGE_TAG:-IMAGE_TAG-NOT-SET} container_name: myai-cv-search-job depends_on: - cv-matcher-api @@ -300,7 +308,7 @@ services: - "com.centurylinklabs.watchtower.enable=true" page-fetcher-api: - image: registry.easysoft.ro/apps/myai-page-fetcher-api:${IMAGE_TAG:-staging} + image: registry.easysoft.ro/apps/myai-page-fetcher-api:${IMAGE_TAG:-IMAGE_TAG-NOT-SET} container_name: myai-page-fetcher-api environment: - ASPNETCORE_ENVIRONMENT=${ASPNETCORE_ENVIRONMENT:-Staging} @@ -332,7 +340,7 @@ services: - "com.centurylinklabs.watchtower.enable=true" web: - image: registry.easysoft.ro/apps/myai-web:${IMAGE_TAG:-staging} + image: registry.easysoft.ro/apps/myai-web:${IMAGE_TAG:-IMAGE_TAG-NOT-SET} container_name: myai-web depends_on: - api diff --git a/web/Dockerfile b/web/Dockerfile index f38dff5..b2cb257 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -16,4 +16,13 @@ EXPOSE 8080 ENV ASPNETCORE_URLS=http://0.0.0.0:8080 COPY --from=build /app/publish . + +# Stamp the commit into the image so a deploy can be verified from the outside. +# +# Without this a smoke test can only ask "does the site return 200?" -- which it did +# throughout the month production was quietly serving staging images. A status check +# cannot tell one build from another; /version.json can, so the smoke job refuses to +# pass until the host is actually serving THIS commit. +ARG GIT_SHA=unknown +RUN mkdir -p wwwroot && printf '{"version":"%s"}' "$GIT_SHA" > wwwroot/version.json ENTRYPOINT ["dotnet", "web.dll"] \ No newline at end of file