diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index 558ddaf..98540c5 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -23,6 +23,7 @@ env: CV_SEARCH_JOB_IMAGE: apps/myai-cv-search-job PAGE_FETCHER_API_IMAGE: apps/myai-page-fetcher-api IMAGE_TAG: ${{ github.ref_name }} # branch name == image tag (staging | production) + WEB_PORT: "5140" # host port the web container is published on jobs: build: @@ -60,7 +61,8 @@ jobs: - name: Build Web image run: | - docker build -f web/Dockerfile -t "${REGISTRY_HOST}/${WEB_IMAGE}:${IMAGE_TAG}" . + docker build --build-arg GIT_SHA="${{ github.sha }}" \ + -f web/Dockerfile -t "${REGISTRY_HOST}/${WEB_IMAGE}:${IMAGE_TAG}" . - name: Build CV cleanup job image run: | @@ -110,3 +112,69 @@ jobs: if: always() run: | docker image prune -f # dangling only (keep base images) + # Building and pushing an image proves nothing about what the host is running. + # Watchtower pulls asynchronously, and for a month it was pulling a tag nobody + # intended -- with every run green, because no step ever asked the deployed site + # what it was serving. This job asks. + # + # It polls the deploy host directly on the LAN rather than the public hostname: + # the runner sits inside the network, only easysoft.ro has a staging equivalent in + # public DNS, and going direct also takes Caddy and any CDN out of the answer. + smoke: + runs-on: host + needs: build + steps: + - name: Wait for the deploy host to serve this commit + run: | + case "${{ github.ref_name }}" in + staging) HOST=192.168.1.111 ;; + production) HOST=192.168.1.101 ;; + *) echo "::error::No deploy host mapped for '${{ github.ref_name }}'."; exit 1 ;; + esac + + URL="http://${HOST}:${WEB_PORT}/version.json" + echo "Polling ${URL} for ${{ github.sha }}" + + # 10 minutes: Watchtower's poke is fire-and-forget with a 30s fallback poll, + # and the container still has to start. + # ⚠️ Steps run under `bash -e -o pipefail`, so a polling loop has to be written + # defensively: the FIRST miss is the normal case, not an error. + # - `curl -sf | sed` fails the whole pipeline under pipefail while the old + # container is still up (404/connection refused), so `|| GOT=""` is required + # - `[ test ] && { ... }` returns non-zero when the test fails, which under -e + # aborts the step. Use `if`. + # Getting both wrong made the first run of this job fail in 20 seconds. + DEADLINE=$(( $(date +%s) + 600 )) + while :; do + GOT=$(curl -sf -m 15 "${URL}" 2>/dev/null | sed -n 's/.*"version":"\([^"]*\)".*/\1/p') || GOT="" + + if [ "${GOT}" = "${{ github.sha }}" ]; then + echo "Serving ${GOT}." + break + fi + + if [ "$(date +%s)" -ge "${DEADLINE}" ]; then + echo "::error::Timed out after 10m. ${HOST} is serving '${GOT:-nothing}', wanted ${{ github.sha }}." + echo "Either Watchtower never pulled the new image, the container failed to" + echo "start, or the stack's IMAGE_TAG does not match this branch." + exit 1 + fi + echo " still serving '${GOT:-nothing}' ..." + sleep 15 + done + + - name: Check the site actually answers + run: | + case "${{ github.ref_name }}" in + staging) HOST=192.168.1.111 ;; + production) HOST=192.168.1.101 ;; + esac + # `|| CODE=000` for the same reason as above: curl exiting non-zero on a + # connection failure must produce a reportable code, not kill the step before + # it can say what went wrong. (`-s` without `-f` already tolerates 4xx/5xx.) + CODE=$(curl -s -o /dev/null -w '%{http_code}' -m 20 "http://${HOST}:${WEB_PORT}/") || CODE=000 + if [ "${CODE}" != "200" ]; then + echo "::error::Home page returned ${CODE}." + exit 1 + fi + echo "Home page 200." diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index f2d7274..42c1fcb 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -1,6 +1,14 @@ +# ⚠️ The IMAGE_TAG fallback is a DELIBERATELY INVALID tag, not `staging`. +# On 2026-07-26 these stacks were recreated by hand and lost their environment +# variables. The old `${IMAGE_TAG:-staging}` then quietly resolved to `staging`, so the +# production host pulled staging images -- with no mail credentials and no recipient +# addresses -- and served them for a month. Nothing failed, because falling back to a +# real tag is indistinguishable from being configured. Now an unset IMAGE_TAG yields +# `IMAGE_TAG-NOT-SET`, the pull fails with "manifest not found", the running container +# is left untouched and the deploy goes red. Loud beats plausible. services: rag-api: - image: registry.easysoft.ro/apps/myai-rag-api:${IMAGE_TAG:-staging} + image: registry.easysoft.ro/apps/myai-rag-api:${IMAGE_TAG:-IMAGE_TAG-NOT-SET} container_name: myai-rag-api environment: - ASPNETCORE_ENVIRONMENT=${ASPNETCORE_ENVIRONMENT:-Staging} @@ -50,7 +58,7 @@ services: - "com.centurylinklabs.watchtower.enable=true" cv-matcher-api: - image: registry.easysoft.ro/apps/myai-cv-matcher-api:${IMAGE_TAG:-staging} + image: registry.easysoft.ro/apps/myai-cv-matcher-api:${IMAGE_TAG:-IMAGE_TAG-NOT-SET} container_name: myai-cv-matcher-api depends_on: - rag-api @@ -102,7 +110,7 @@ services: - "com.centurylinklabs.watchtower.enable=true" email-api: - image: registry.easysoft.ro/apps/myai-email-api:${IMAGE_TAG:-staging} + image: registry.easysoft.ro/apps/myai-email-api:${IMAGE_TAG:-IMAGE_TAG-NOT-SET} container_name: myai-email-api environment: - ASPNETCORE_ENVIRONMENT=${ASPNETCORE_ENVIRONMENT:-Staging} @@ -143,7 +151,7 @@ services: - "com.centurylinklabs.watchtower.enable=true" api: - image: registry.easysoft.ro/apps/myai-api:${IMAGE_TAG:-staging} + image: registry.easysoft.ro/apps/myai-api:${IMAGE_TAG:-IMAGE_TAG-NOT-SET} container_name: myai-api depends_on: - cv-matcher-api @@ -217,7 +225,7 @@ services: - "com.centurylinklabs.watchtower.enable=true" cv-cleanup-job: - image: registry.easysoft.ro/apps/myai-cv-cleanup-job:${IMAGE_TAG:-staging} + image: registry.easysoft.ro/apps/myai-cv-cleanup-job:${IMAGE_TAG:-IMAGE_TAG-NOT-SET} container_name: myai-cv-cleanup-job depends_on: - api @@ -247,7 +255,7 @@ services: - "com.centurylinklabs.watchtower.enable=true" cv-search-job: - image: registry.easysoft.ro/apps/myai-cv-search-job:${IMAGE_TAG:-staging} + image: registry.easysoft.ro/apps/myai-cv-search-job:${IMAGE_TAG:-IMAGE_TAG-NOT-SET} container_name: myai-cv-search-job depends_on: - cv-matcher-api @@ -300,7 +308,7 @@ services: - "com.centurylinklabs.watchtower.enable=true" page-fetcher-api: - image: registry.easysoft.ro/apps/myai-page-fetcher-api:${IMAGE_TAG:-staging} + image: registry.easysoft.ro/apps/myai-page-fetcher-api:${IMAGE_TAG:-IMAGE_TAG-NOT-SET} container_name: myai-page-fetcher-api environment: - ASPNETCORE_ENVIRONMENT=${ASPNETCORE_ENVIRONMENT:-Staging} @@ -332,7 +340,7 @@ services: - "com.centurylinklabs.watchtower.enable=true" web: - image: registry.easysoft.ro/apps/myai-web:${IMAGE_TAG:-staging} + image: registry.easysoft.ro/apps/myai-web:${IMAGE_TAG:-IMAGE_TAG-NOT-SET} container_name: myai-web depends_on: - api diff --git a/web/Dockerfile b/web/Dockerfile index f38dff5..b2cb257 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -16,4 +16,13 @@ EXPOSE 8080 ENV ASPNETCORE_URLS=http://0.0.0.0:8080 COPY --from=build /app/publish . + +# Stamp the commit into the image so a deploy can be verified from the outside. +# +# Without this a smoke test can only ask "does the site return 200?" -- which it did +# throughout the month production was quietly serving staging images. A status check +# cannot tell one build from another; /version.json can, so the smoke job refuses to +# pass until the host is actually serving THIS commit. +ARG GIT_SHA=unknown +RUN mkdir -p wwwroot && printf '{"version":"%s"}' "$GIT_SHA" > wwwroot/version.json ENTRYPOINT ["dotnet", "web.dll"] \ No newline at end of file