fix(ci): Add retry logic for package linking with delay

Addresses timing issue where packages aren't immediately queryable via
API after being pushed to the registry.

Changes:
- Initial 10-second delay for package indexing
- Retry logic: 3 attempts with 5-second delays
- Only retries on 404 (not found) errors
- Returns success on 201/204 (linked) or 400 (already linked)
- Better logging shows attempt progress

This fixes the race condition where link-packages ran before packages
were indexed in Gitea's registry API.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

.woodpecker.yml

@@ -1,48 +1,323 @@
-# Temporary minimal pipeline for testing package linking
+# Woodpecker CI Quality Enforcement Pipeline - Monorepo
 when:
   - event: [push, pull_request, manual]
 
+variables:
+  - &node_image "node:20-alpine"
+  - &install_deps |
+    corepack enable
+    pnpm install --frozen-lockfile
+  - &use_deps |
+    corepack enable
+  # Kaniko base command setup
+  - &kaniko_setup |
+    mkdir -p /kaniko/.docker
+    echo "{\"auths\":{\"git.mosaicstack.dev\":{\"username\":\"$GITEA_USER\",\"password\":\"$GITEA_TOKEN\"}}}" > /kaniko/.docker/config.json
+
+services:
+  postgres:
+    image: postgres:17-alpine
+    environment:
+      POSTGRES_DB: test_db
+      POSTGRES_USER: test_user
+      POSTGRES_PASSWORD: test_password
+
 steps:
-  # Test package linking with proper variable escaping
+  install:
-  link-packages-test:
+    image: *node_image
+    commands:
+      - *install_deps
+
+  security-audit:
+    image: *node_image
+    commands:
+      - *use_deps
+      - pnpm audit --audit-level=high
+    depends_on:
+      - install
+
+  lint:
+    image: *node_image
+    environment:
+      SKIP_ENV_VALIDATION: "true"
+    commands:
+      - *use_deps
+      - pnpm lint
+    depends_on:
+      - install
+    when:
+      - evaluate: 'CI_PIPELINE_EVENT != "pull_request" || CI_COMMIT_BRANCH != "main"'
+
+  prisma-generate:
+    image: *node_image
+    environment:
+      SKIP_ENV_VALIDATION: "true"
+    commands:
+      - *use_deps
+      - pnpm --filter "@mosaic/api" prisma:generate
+    depends_on:
+      - install
+
+  prisma-migrate:
+    image: *node_image
+    environment:
+      SKIP_ENV_VALIDATION: "true"
+      DATABASE_URL: "postgresql://test_user:test_password@postgres:5432/test_db?schema=public"
+    commands:
+      - *use_deps
+      - pnpm --filter "@mosaic/api" prisma migrate deploy
+    depends_on:
+      - prisma-generate
+
+  typecheck:
+    image: *node_image
+    environment:
+      SKIP_ENV_VALIDATION: "true"
+    commands:
+      - *use_deps
+      - pnpm typecheck
+    depends_on:
+      - prisma-generate
+
+  test:
+    image: *node_image
+    environment:
+      SKIP_ENV_VALIDATION: "true"
+      DATABASE_URL: "postgresql://test_user:test_password@postgres:5432/test_db?schema=public"
+      ENCRYPTION_KEY: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
+    commands:
+      - *use_deps
+      - pnpm test
+    depends_on:
+      - prisma-migrate
+
+  build:
+    image: *node_image
+    environment:
+      SKIP_ENV_VALIDATION: "true"
+      NODE_ENV: "production"
+    commands:
+      - *use_deps
+      - pnpm build
+    depends_on:
+      - typecheck # Only block on critical checks
+      - security-audit
+      - prisma-generate
+
+  # ======================
+  # Docker Build & Push (main/develop only)
+  # ======================
+  # Requires secrets: gitea_username, gitea_token
+  #
+  # Tagging Strategy:
+  #   - Always: commit SHA (e.g., 658ec077)
+  #   - main branch: 'latest'
+  #   - develop branch: 'dev'
+  #   - git tags: version tag (e.g., v1.0.0)
+
+  # Build and push API image using Kaniko
+  docker-build-api:
+    image: gcr.io/kaniko-project/executor:debug
+    environment:
+      GITEA_USER:
+        from_secret: gitea_username
+      GITEA_TOKEN:
+        from_secret: gitea_token
+      CI_COMMIT_BRANCH: ${CI_COMMIT_BRANCH}
+      CI_COMMIT_TAG: ${CI_COMMIT_TAG}
+      CI_COMMIT_SHA: ${CI_COMMIT_SHA}
+    commands:
+      - *kaniko_setup
+      - |
+        DESTINATIONS="--destination git.mosaicstack.dev/mosaic/stack-api:${CI_COMMIT_SHA:0:8}"
+        if [ "$CI_COMMIT_BRANCH" = "main" ]; then
+          DESTINATIONS="$DESTINATIONS --destination git.mosaicstack.dev/mosaic/stack-api:latest"
+        elif [ "$CI_COMMIT_BRANCH" = "develop" ]; then
+          DESTINATIONS="$DESTINATIONS --destination git.mosaicstack.dev/mosaic/stack-api:dev"
+        fi
+        if [ -n "$CI_COMMIT_TAG" ]; then
+          DESTINATIONS="$DESTINATIONS --destination git.mosaicstack.dev/mosaic/stack-api:$CI_COMMIT_TAG"
+        fi
+        /kaniko/executor --context . --dockerfile apps/api/Dockerfile $DESTINATIONS
+    when:
+      - branch: [main, develop]
+        event: [push, manual, tag]
+    depends_on:
+      - build
+
+  # Build and push Web image using Kaniko
+  docker-build-web:
+    image: gcr.io/kaniko-project/executor:debug
+    environment:
+      GITEA_USER:
+        from_secret: gitea_username
+      GITEA_TOKEN:
+        from_secret: gitea_token
+      CI_COMMIT_BRANCH: ${CI_COMMIT_BRANCH}
+      CI_COMMIT_TAG: ${CI_COMMIT_TAG}
+      CI_COMMIT_SHA: ${CI_COMMIT_SHA}
+    commands:
+      - *kaniko_setup
+      - |
+        DESTINATIONS="--destination git.mosaicstack.dev/mosaic/stack-web:${CI_COMMIT_SHA:0:8}"
+        if [ "$CI_COMMIT_BRANCH" = "main" ]; then
+          DESTINATIONS="$DESTINATIONS --destination git.mosaicstack.dev/mosaic/stack-web:latest"
+        elif [ "$CI_COMMIT_BRANCH" = "develop" ]; then
+          DESTINATIONS="$DESTINATIONS --destination git.mosaicstack.dev/mosaic/stack-web:dev"
+        fi
+        if [ -n "$CI_COMMIT_TAG" ]; then
+          DESTINATIONS="$DESTINATIONS --destination git.mosaicstack.dev/mosaic/stack-web:$CI_COMMIT_TAG"
+        fi
+        /kaniko/executor --context . --dockerfile apps/web/Dockerfile --build-arg NEXT_PUBLIC_API_URL=https://api.mosaicstack.dev $DESTINATIONS
+    when:
+      - branch: [main, develop]
+        event: [push, manual, tag]
+    depends_on:
+      - build
+
+  # Build and push Postgres image using Kaniko
+  docker-build-postgres:
+    image: gcr.io/kaniko-project/executor:debug
+    environment:
+      GITEA_USER:
+        from_secret: gitea_username
+      GITEA_TOKEN:
+        from_secret: gitea_token
+      CI_COMMIT_BRANCH: ${CI_COMMIT_BRANCH}
+      CI_COMMIT_TAG: ${CI_COMMIT_TAG}
+      CI_COMMIT_SHA: ${CI_COMMIT_SHA}
+    commands:
+      - *kaniko_setup
+      - |
+        DESTINATIONS="--destination git.mosaicstack.dev/mosaic/stack-postgres:${CI_COMMIT_SHA:0:8}"
+        if [ "$CI_COMMIT_BRANCH" = "main" ]; then
+          DESTINATIONS="$DESTINATIONS --destination git.mosaicstack.dev/mosaic/stack-postgres:latest"
+        elif [ "$CI_COMMIT_BRANCH" = "develop" ]; then
+          DESTINATIONS="$DESTINATIONS --destination git.mosaicstack.dev/mosaic/stack-postgres:dev"
+        fi
+        if [ -n "$CI_COMMIT_TAG" ]; then
+          DESTINATIONS="$DESTINATIONS --destination git.mosaicstack.dev/mosaic/stack-postgres:$CI_COMMIT_TAG"
+        fi
+        /kaniko/executor --context docker/postgres --dockerfile docker/postgres/Dockerfile $DESTINATIONS
+    when:
+      - branch: [main, develop]
+        event: [push, manual, tag]
+    depends_on:
+      - build
+
+  # Build and push OpenBao image using Kaniko
+  docker-build-openbao:
+    image: gcr.io/kaniko-project/executor:debug
+    environment:
+      GITEA_USER:
+        from_secret: gitea_username
+      GITEA_TOKEN:
+        from_secret: gitea_token
+      CI_COMMIT_BRANCH: ${CI_COMMIT_BRANCH}
+      CI_COMMIT_TAG: ${CI_COMMIT_TAG}
+      CI_COMMIT_SHA: ${CI_COMMIT_SHA}
+    commands:
+      - *kaniko_setup
+      - |
+        DESTINATIONS="--destination git.mosaicstack.dev/mosaic/stack-openbao:${CI_COMMIT_SHA:0:8}"
+        if [ "$CI_COMMIT_BRANCH" = "main" ]; then
+          DESTINATIONS="$DESTINATIONS --destination git.mosaicstack.dev/mosaic/stack-openbao:latest"
+        elif [ "$CI_COMMIT_BRANCH" = "develop" ]; then
+          DESTINATIONS="$DESTINATIONS --destination git.mosaicstack.dev/mosaic/stack-openbao:dev"
+        fi
+        if [ -n "$CI_COMMIT_TAG" ]; then
+          DESTINATIONS="$DESTINATIONS --destination git.mosaicstack.dev/mosaic/stack-openbao:$CI_COMMIT_TAG"
+        fi
+        /kaniko/executor --context docker/openbao --dockerfile docker/openbao/Dockerfile $DESTINATIONS
+    when:
+      - branch: [main, develop]
+        event: [push, manual, tag]
+    depends_on:
+      - build
+
+  # Build and push Orchestrator image using Kaniko
+  docker-build-orchestrator:
+    image: gcr.io/kaniko-project/executor:debug
+    environment:
+      GITEA_USER:
+        from_secret: gitea_username
+      GITEA_TOKEN:
+        from_secret: gitea_token
+      CI_COMMIT_BRANCH: ${CI_COMMIT_BRANCH}
+      CI_COMMIT_TAG: ${CI_COMMIT_TAG}
+      CI_COMMIT_SHA: ${CI_COMMIT_SHA}
+    commands:
+      - *kaniko_setup
+      - |
+        DESTINATIONS="--destination git.mosaicstack.dev/mosaic/stack-orchestrator:${CI_COMMIT_SHA:0:8}"
+        if [ "$CI_COMMIT_BRANCH" = "main" ]; then
+          DESTINATIONS="$DESTINATIONS --destination git.mosaicstack.dev/mosaic/stack-orchestrator:latest"
+        elif [ "$CI_COMMIT_BRANCH" = "develop" ]; then
+          DESTINATIONS="$DESTINATIONS --destination git.mosaicstack.dev/mosaic/stack-orchestrator:dev"
+        fi
+        if [ -n "$CI_COMMIT_TAG" ]; then
+          DESTINATIONS="$DESTINATIONS --destination git.mosaicstack.dev/mosaic/stack-orchestrator:$CI_COMMIT_TAG"
+        fi
+        /kaniko/executor --context . --dockerfile apps/orchestrator/Dockerfile $DESTINATIONS
+    when:
+      - branch: [main, develop]
+        event: [push, manual, tag]
+    depends_on:
+      - build
+
+  # ======================
+  # Link Packages to Repository
+  # ======================
+  # Links all Docker packages to the mosaic/stack repository
+  # This makes packages visible on the repository page in Gitea
+  link-packages:
     image: alpine:3
     environment:
       GITEA_TOKEN:
         from_secret: gitea_token
     commands:
       - apk add --no-cache curl
-      - echo "Testing package linking with variable expansion..."
+      - echo "Waiting 10 seconds for packages to be indexed in registry..."
+      - sleep 10
       - |
         link_package() {
           PKG="$$1"
-          echo ""
+          echo "Linking $$PKG..."
-          echo "Testing package: $$PKG"
-          STATUS=$$(curl -s -o /tmp/link-response.txt -w "%{http_code}" -X POST \
-            -H "Authorization: token $$GITEA_TOKEN" \
-            "https://git.mosaicstack.dev/api/v1/packages/mosaic/container/$$PKG/-/link/stack")
-          echo "  URL: https://git.mosaicstack.dev/api/v1/packages/mosaic/container/$$PKG/-/link/stack"
-          echo "  Status: $$STATUS"
 
-          if [ "$$STATUS" = "201" ] || [ "$$STATUS" = "204" ]; then
+          # Retry up to 3 times with 5 second delays
-            echo "  ✅ Successfully linked $$PKG to stack"
+          for attempt in 1 2 3; do
-          elif [ "$$STATUS" = "400" ]; then
+            STATUS=$$(curl -s -o /tmp/link-response.txt -w "%{http_code}" -X POST \
-            echo "  ✅ $$PKG already linked (OK)"
+              -H "Authorization: token $$GITEA_TOKEN" \
-            cat /tmp/link-response.txt
+              "https://git.mosaicstack.dev/api/v1/packages/mosaic/container/$$PKG/-/link/stack")
-          else
+
-            echo "  ❌ $$PKG link failed"
+            if [ "$$STATUS" = "201" ] || [ "$$STATUS" = "204" ]; then
-            echo "  Response:"
+              echo "  ✅ Linked $$PKG to stack"
-            cat /tmp/link-response.txt
+              return 0
-          fi
+            elif [ "$$STATUS" = "400" ]; then
+              echo "  ✅ $$PKG already linked (OK)"
+              return 0
+            elif [ "$$STATUS" = "404" ] && [ $$attempt -lt 3 ]; then
+              echo "  ⏳ $$PKG not found yet, waiting 5s (attempt $$attempt/3)..."
+              sleep 5
+            else
+              echo "  ❌ $$PKG link failed with status $$STATUS"
+              cat /tmp/link-response.txt
+              return 1
+            fi
+          done
         }
 
-        echo "=== Linking all stack packages ==="
         link_package "stack-api"
         link_package "stack-web"
         link_package "stack-postgres"
         link_package "stack-openbao"
         link_package "stack-orchestrator"
-        echo ""
-        echo "=== Test complete ==="
     when:
       - branch: [main, develop]
-        event: [push, manual]
+        event: [push, manual, tag]
+    depends_on:
+      - docker-build-api
+      - docker-build-web
+      - docker-build-postgres
+      - docker-build-openbao
+      - docker-build-orchestrator