chore: dev environment on @data-fair/lib-dev and the shared service conventions

.dockerignore

@@ -1,4 +1,9 @@
-node_modules
-.nuxt
-.output
-data
+**/node_modules
+**/dist
+ui/dts
+.env
+data
+dev/logs
+test-results
+playwright-report
+.git

.github/workflows/pull-requests.yml

@@ -4,6 +4,12 @@ name: Check pull requests
 
 on: pull_request
 
+# a new push to the pull request cancels the checks of the previous one; master and tags
+# never cancel (no concurrency group there)
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
 jobs:
   quality:
-    uses: ./.github/workflows/reuse-quality.yml
+    uses: ./.github/workflows/reuse-quality.yml

.github/workflows/reuse-quality.yml

@@ -1,18 +1,24 @@
 name: Quality checks
 
+# The full gate: static checks and builds in one job, then the whole test suite (unit, api,
+# e2e) in a second job that reuses the generated types and the UI build as an artifact.
+# Used by pull requests and releases; the pre-push hook only
+# runs a light subset (dev/pre-push.sh). No shards: the suite takes a few minutes on one
+# worker, less than the setup a second runner would pay again.
+
 on:
   workflow_call:
 
 jobs:
-  quality:
+  static:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout git repository
       uses: actions/checkout@v5
 
     - uses: actions/setup-node@v5
       with:
-        node-version: 24
+        node-version-file: .nvmrc
         cache: 'npm'
 
     - name: Install dependencies
@@ -24,47 +30,119 @@ jobs:
     - name: Build types
       run: npm run build-types
 
+    - name: Build UI (generates the ui/dts files vue-tsc needs)
+      run: npm -w ui run build
+
     - name: Type check
       run: npm run check-types
 
-    - name: Build UI
-      run: npm -w ui run build
+    - name: Audit
+      run: npm audit --omit=dev --audit-level=critical
+
+    # the API imports the generated .type modules and serves ui/dist/index.html at startup
+    - name: Upload generated types and UI build
+      uses: actions/upload-artifact@v4
+      with:
+        name: build
+        include-hidden-files: true
+        path: |
+          api/**/.type/
+          worker/**/.type/
+          ui/dist/
+          ui/dts/
+        retention-days: 1
+
+  tests:
+    needs: static
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout git repository
+      uses: actions/checkout@v5
+
+    - uses: actions/setup-node@v5
+      with:
+        node-version-file: .nvmrc
+        cache: 'npm'
+
+    - name: Install dependencies
+      run: npm ci
+
+    - name: Download generated types and UI build
+      uses: actions/download-artifact@v4
+      with:
+        name: build
+
+    # only the browser binaries are cached, keyed on the playwright version; never test state
+    - name: Read the Playwright version
+      id: playwright
+      run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
+
+    - name: Cache the Playwright browser
+      id: playwright-cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/ms-playwright
+        key: playwright-${{ runner.os }}-${{ steps.playwright.outputs.version }}
+
+    - name: Install the Playwright browser
+      if: steps.playwright-cache.outputs.cache-hit != 'true'
+      run: npx playwright install --with-deps chromium
+
+    - name: Install the Playwright browser system dependencies
+      if: steps.playwright-cache.outputs.cache-hit == 'true'
+      run: npx playwright install-deps chromium
 
     - name: Init .env
-      run: ./dev/init-env.sh
+      run: ./node_modules/.bin/df-dev init-env # local bin only: a bare npx would fetch an unrelated "df-dev" from the registry if the dependency were missing
 
     - name: Use localhost as DEV_HOST in CI
       run: sed -i 's/^DEV_HOST=.*/DEV_HOST=localhost/' .env
 
-    - name: Start docker compose services
-      run: docker compose --profile dev up -d --wait
+    - name: Start the core docker compose services (optional ones stay off, like a default dev env)
+      run: docker compose up -d --wait
 
+    # logs in dev/logs, uploaded on failure
     - name: Start dev API
-      run: npx dotenv -- npm run dev-api &
+      run: mkdir -p dev/logs && NODE_CONFIG="$(./node_modules/.bin/df-dev absent-config)" ./node_modules/.bin/dotenv -- npm -w api run dev > dev/logs/api.log 2>&1 &
 
     - name: Start dev worker
-      run: npx dotenv -- npm run dev-worker &
+      run: NODE_CONFIG="$(./node_modules/.bin/df-dev absent-config)" ./node_modules/.bin/dotenv -- npm -w worker run dev > dev/logs/worker.log 2>&1 &
 
-    - name: Wait for dev API to be ready
+    - name: Start dev UI
+      run: ./node_modules/.bin/dotenv -- npm -w ui run dev > dev/logs/ui.log 2>&1 &
+
+    - name: Wait for the dev API, worker and UI to be ready
       run: |
         set -a; source .env; set +a
         for i in {1..30}; do
-          curl -sf "http://localhost:${DEV_API_PORT}/api/v1/_ping" && exit 0
+          if curl -sf "http://localhost:${DEV_API_PORT}/api/v1/_ping" >/dev/null \
+            && curl -sf "http://localhost:${DEV_WORKER_OBSERVER_PORT}/metrics" >/dev/null \
+            && curl -sf "http://localhost:${DEV_UI_PORT}/processings/" >/dev/null; then
+            exit 0
+          fi
           sleep 2
         done
-        echo "dev API did not become ready" >&2
+        echo "dev API, worker or UI did not become ready" >&2
+        tail -n 50 dev/logs/*.log
         exit 1
 
     - name: Run tests
-      run: npm run test-unit && npm run test-api
+      run: npm test
 
-    - name: Audit
-      run: npm audit --omit=dev --audit-level=critical
+    - name: Upload test results on failure
+      if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: test-results
+        path: |
+          test-results/
+          dev/logs/
+        retention-days: 7
 
     - name: Docker compose logs on failure
       if: failure()
-      run: docker compose --profile dev logs
+      run: docker compose --profile '*' logs
 
     - name: Stop docker compose services
       if: always()
-      run: docker compose --profile dev down
+      run: docker compose --profile '*' down

.gitignore

@@ -1,9 +1,18 @@
-data/
-dev/logs/
+# dependencies and generated code
 node_modules/
-**/config/local-*
-.config/
 .type/
+
+# local environment and data
 .env
-playwright-report/
+**/config/local-*
+data/
+dev/logs/
+
+# test output
 test-results/
+playwright-report/
+
+# agent working files
+docs/superpowers/
+.superpowers/
+.playwright-mcp/

.husky/commit-msg

@@ -1 +1 @@
-npx --no-install commitlint --edit ""
+npx --no -- commitlint --edit "$1"

.husky/pre-push

@@ -1 +1 @@
-npm run quality
+bash dev/pre-push.sh

.zellij.kdl

@@ -1,32 +1,23 @@
+// Dev environment of processings, see @data-fair/lib-dev. Agents: npx --no df-dev status | logs <pane> | restart <pane>
 layout {
-    pane {
-      split_direction "vertical"
-      pane name="processings" borderless=true {
-        command "${DEV_SHELL}"
-        args "-ic" "nvm use > /dev/null 2>&1 && bash"
-      }
-      pane name="deps" {
-        command "${DEV_SHELL}"
-        args "-ic" "npm run dev-deps"
-      }
+  pane split_direction="vertical" {
+    pane name="processings" borderless=true
+    pane name="status" command="./node_modules/.bin/df-status" {
+      args "--start-deps"
     }
-    pane {
-      split_direction "vertical"
-      pane name="ui" {
-        command "${DEV_SHELL}"
-        args "-ic" "nvm use > /dev/null 2>&1 && npm run dev-ui"
-      }
-      pane name="api" {
-        command "${DEV_SHELL}"
-        args "-ic" "nvm use > /dev/null 2>&1 && npm run dev-api"
-      }
-      pane name="worker" {
-        command "${DEV_SHELL}"
-        args "-ic" "nvm use > /dev/null 2>&1 && npm run dev-worker"
-      }
+  }
+  pane split_direction="vertical" {
+    pane name="ui" command="bash" {
+      args "-c" "cd ui && FORCE_COLOR=1 NODE_CONFIG_DIR=../api/config/ exec ../node_modules/.bin/df-mem-cap vite --port $DEV_UI_PORT"
     }
-    pane size=1 borderless=true {
-        command "bash"
-        args "-ic" "echo -n -e \"Dev server available at \\e[1;96mhttp://${DEV_HOST}:${NGINX_PORT1}/processings\\033[0m\""
+    pane name="api" command="bash" {
+      args "-c" "./node_modules/.bin/df-dev wait-deps && export NODE_CONFIG=$(./node_modules/.bin/df-dev absent-config) && cd api && FORCE_COLOR=1 NODE_ENV=development DEBUG=upgrade exec ../node_modules/.bin/nodemon -e js,ts,json index.ts"
     }
+    pane name="worker" command="bash" {
+      args "-c" "./node_modules/.bin/df-dev wait-deps && export NODE_CONFIG=$(./node_modules/.bin/df-dev absent-config) && cd worker && FORCE_COLOR=1 NODE_ENV=development DEBUG='worker,upgrade*' exec ../node_modules/.bin/nodemon -e js,ts,json index.ts"
+    }
+  }
+  pane name="url" size=1 borderless=true command="bash" {
+    args "-c" "echo -n -e \"Dev server available at \\e[1;96mhttp://$DEV_HOST:$NGINX_PORT/processings/dev\\e[0m\"; exec sleep infinity"
+  }
 }