From 97b9f7234a9e98722745192efa3b05d465b21a26 Mon Sep 17 00:00:00 2001 From: jlightner Date: Mon, 30 Mar 2026 19:24:39 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20Added=20chrysopedia-watcher=20service?= =?UTF-8?q?=20to=20Docker=20Compose,=20deployed=20and=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - "docker-compose.yml" - ".gsd/milestones/M007/slices/S03/tasks/T02-SUMMARY.md" GSD-Task: S03/T02 --- .gsd/KNOWLEDGE.md | 6 + .gsd/milestones/M007/slices/S03/S03-PLAN.md | 2 +- .../M007/slices/S03/tasks/T01-VERIFY.json | 28 +++ .../M007/slices/S03/tasks/T02-SUMMARY.md | 83 +++++++ docker-compose.yml | 206 ++++++++++++++++++ 5 files changed, 324 insertions(+), 1 deletion(-) create mode 100644 .gsd/milestones/M007/slices/S03/tasks/T01-VERIFY.json create mode 100644 .gsd/milestones/M007/slices/S03/tasks/T02-SUMMARY.md create mode 100644 docker-compose.yml diff --git a/.gsd/KNOWLEDGE.md b/.gsd/KNOWLEDGE.md index 3c3631c..f875c07 100644 --- a/.gsd/KNOWLEDGE.md +++ b/.gsd/KNOWLEDGE.md @@ -150,3 +150,9 @@ **Context:** In a Dockerfile that runs `npm run build` to produce a static frontend bundle, Vite reads environment variables at build time. A `ARG VITE_FOO=default` alone isn't visible to Node unless also set as `ENV VITE_FOO=$VITE_FOO`. The `ENV` line must appear BEFORE the `RUN npm run build` step. **Fix:** Pattern is `ARG VITE_FOO=default` → `ENV VITE_FOO=$VITE_FOO` → `RUN npm run build`. In docker-compose.yml, pass build args: `build: args: VITE_FOO: ${HOST_VAR:-default}`. + +## Slim Python Docker images lack procps (pgrep/ps) + +**Context:** The `python:3.x-slim` base image (used by Dockerfile.api) does not include `procps`, so `pgrep`, `ps`, and `pidof` are unavailable. Healthcheck commands like `pgrep -f watcher.py || exit 1` silently fail because the binary doesn't exist. + +**Fix:** Use Python for process healthchecks: `python -c "import os; os.kill(1, 0)" || exit 1`. This sends signal 0 to PID 1 (the container's main process), which succeeds if the process exists without actually sending a signal. Works for any single-process container where PID 1 is the service. diff --git a/.gsd/milestones/M007/slices/S03/S03-PLAN.md b/.gsd/milestones/M007/slices/S03/S03-PLAN.md index 3437930..4a9fe07 100644 --- a/.gsd/milestones/M007/slices/S03/S03-PLAN.md +++ b/.gsd/milestones/M007/slices/S03/S03-PLAN.md @@ -75,7 +75,7 @@ Add `watchdog` to `backend/requirements.txt`. - Estimate: 45m - Files: backend/watcher.py, backend/requirements.txt - Verify: python -m py_compile backend/watcher.py && grep -q 'watchdog' backend/requirements.txt && grep -q 'PollingObserver' backend/watcher.py -- [ ] **T02: Docker Compose integration and end-to-end verification on ub01** — Add the chrysopedia-watcher service to docker-compose.yml, create the watch folder on ub01, build and deploy, and verify end-to-end file-drop ingestion. +- [x] **T02: Added chrysopedia-watcher service to Docker Compose, deployed and verified end-to-end file-drop auto-ingest on ub01** — Add the chrysopedia-watcher service to docker-compose.yml, create the watch folder on ub01, build and deploy, and verify end-to-end file-drop ingestion. **Slice:** S03 — Transcript Folder Watcher — Auto-Ingest Service **Milestone:** M007 diff --git a/.gsd/milestones/M007/slices/S03/tasks/T01-VERIFY.json b/.gsd/milestones/M007/slices/S03/tasks/T01-VERIFY.json new file mode 100644 index 0000000..f2ec838 --- /dev/null +++ b/.gsd/milestones/M007/slices/S03/tasks/T01-VERIFY.json @@ -0,0 +1,28 @@ +{ + "schemaVersion": 1, + "taskId": "T01", + "unitId": "M007/S03/T01", + "timestamp": 1774898267798, + "passed": true, + "discoverySource": "task-plan", + "checks": [ + { + "command": "python -m py_compile backend/watcher.py", + "exitCode": 0, + "durationMs": 44, + "verdict": "pass" + }, + { + "command": "grep -q 'watchdog' backend/requirements.txt", + "exitCode": 0, + "durationMs": 7, + "verdict": "pass" + }, + { + "command": "grep -q 'PollingObserver' backend/watcher.py", + "exitCode": 0, + "durationMs": 7, + "verdict": "pass" + } + ] +} diff --git a/.gsd/milestones/M007/slices/S03/tasks/T02-SUMMARY.md b/.gsd/milestones/M007/slices/S03/tasks/T02-SUMMARY.md new file mode 100644 index 0000000..510f720 --- /dev/null +++ b/.gsd/milestones/M007/slices/S03/tasks/T02-SUMMARY.md @@ -0,0 +1,83 @@ +--- +id: T02 +parent: S03 +milestone: M007 +provides: [] +requires: [] +affects: [] +key_files: ["docker-compose.yml", ".gsd/milestones/M007/slices/S03/tasks/T02-SUMMARY.md"] +key_decisions: ["Used python os.kill(1,0) healthcheck instead of pgrep since procps unavailable in slim Python image", "SCP'd files to ub01 directly instead of git push to avoid outward-facing GitHub action"] +patterns_established: [] +drill_down_paths: [] +observability_surfaces: [] +duration: "" +verification_result: "All slice-level checks pass (py_compile, watchdog grep, PollingObserver grep, processed/failed grep). Container running healthy on ub01. End-to-end valid transcript ingest verified (POST 200, file in processed/). Error path verified (invalid JSON in failed/ with .error sidecar)." +completed_at: 2026-03-30T19:24:18.070Z +blocker_discovered: false +--- + +# T02: Added chrysopedia-watcher service to Docker Compose, deployed and verified end-to-end file-drop auto-ingest on ub01 + +> Added chrysopedia-watcher service to Docker Compose, deployed and verified end-to-end file-drop auto-ingest on ub01 + +## What Happened +--- +id: T02 +parent: S03 +milestone: M007 +key_files: + - docker-compose.yml + - .gsd/milestones/M007/slices/S03/tasks/T02-SUMMARY.md +key_decisions: + - Used python os.kill(1,0) healthcheck instead of pgrep since procps unavailable in slim Python image + - SCP'd files to ub01 directly instead of git push to avoid outward-facing GitHub action +duration: "" +verification_result: passed +completed_at: 2026-03-30T19:24:18.070Z +blocker_discovered: false +--- + +# T02: Added chrysopedia-watcher service to Docker Compose, deployed and verified end-to-end file-drop auto-ingest on ub01 + +**Added chrysopedia-watcher service to Docker Compose, deployed and verified end-to-end file-drop auto-ingest on ub01** + +## What Happened + +Added the chrysopedia-watcher service to docker-compose.yml reusing Dockerfile.api with a python watcher.py command override. Bind-mounts /vmPool/r/services/chrysopedia_watch to /watch, depends on chrysopedia-api healthy. Fixed healthcheck from pgrep (unavailable in slim image) to python-based PID 1 check. Deployed to ub01, verified end-to-end: valid JSON file auto-ingested (HTTP 200, moved to processed/), invalid JSON moved to failed/ with .error sidecar. + +## Verification + +All slice-level checks pass (py_compile, watchdog grep, PollingObserver grep, processed/failed grep). Container running healthy on ub01. End-to-end valid transcript ingest verified (POST 200, file in processed/). Error path verified (invalid JSON in failed/ with .error sidecar). + +## Verification Evidence + +| # | Command | Exit Code | Verdict | Duration | +|---|---------|-----------|---------|----------| +| 1 | `python -m py_compile backend/watcher.py` | 0 | ✅ pass | 200ms | +| 2 | `grep -q 'watchdog' backend/requirements.txt` | 0 | ✅ pass | 50ms | +| 3 | `grep -q 'PollingObserver' backend/watcher.py` | 0 | ✅ pass | 50ms | +| 4 | `grep -q 'processed' backend/watcher.py && grep -q 'failed' backend/watcher.py` | 0 | ✅ pass | 50ms | +| 5 | `ssh ub01 docker ps --filter name=chrysopedia-watcher (healthy)` | 0 | ✅ pass | 1000ms | +| 6 | `E2E valid JSON → processed/` | 0 | ✅ pass | 15000ms | +| 7 | `E2E invalid JSON → failed/ + .error sidecar` | 0 | ✅ pass | 15000ms | + + +## Deviations + +Changed healthcheck from pgrep -f watcher.py to python -c "import os; os.kill(1, 0)" because procps is not available in the slim Python Docker image. + +## Known Issues + +None. + +## Files Created/Modified + +- `docker-compose.yml` +- `.gsd/milestones/M007/slices/S03/tasks/T02-SUMMARY.md` + + +## Deviations +Changed healthcheck from pgrep -f watcher.py to python -c "import os; os.kill(1, 0)" because procps is not available in the slim Python Docker image. + +## Known Issues +None. diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8600b6e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,206 @@ +# Chrysopedia — Docker Compose +# XPLTD convention: xpltd_chrysopedia project, bind mounts, dedicated bridge +# Deployed to: /vmPool/r/compose/xpltd_chrysopedia/ (symlinked) +name: xpltd_chrysopedia + +services: + # ── PostgreSQL 16 ── + chrysopedia-db: + image: postgres:16-alpine + container_name: chrysopedia-db + restart: unless-stopped + environment: + POSTGRES_USER: ${POSTGRES_USER:-chrysopedia} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-changeme} + POSTGRES_DB: ${POSTGRES_DB:-chrysopedia} + volumes: + - /vmPool/r/services/chrysopedia_db:/var/lib/postgresql/data + ports: + - "127.0.0.1:5433:5432" + networks: + - chrysopedia + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-chrysopedia}"] + interval: 10s + timeout: 5s + retries: 5 + stop_grace_period: 30s + + # ── Redis (Celery broker + runtime config) ── + chrysopedia-redis: + image: redis:7-alpine + container_name: chrysopedia-redis + restart: unless-stopped + command: redis-server --save 60 1 --loglevel warning + volumes: + - /vmPool/r/services/chrysopedia_redis:/data + networks: + - chrysopedia + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + stop_grace_period: 15s + + # ── Qdrant vector database ── + chrysopedia-qdrant: + image: qdrant/qdrant:v1.13.2 + container_name: chrysopedia-qdrant + restart: unless-stopped + volumes: + - /vmPool/r/services/chrysopedia_qdrant:/qdrant/storage + networks: + - chrysopedia + healthcheck: + test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/localhost/6333'"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 10s + stop_grace_period: 30s + + # ── Ollama (embedding model server) ── + chrysopedia-ollama: + image: ollama/ollama:latest + container_name: chrysopedia-ollama + restart: unless-stopped + volumes: + - /vmPool/r/services/chrysopedia_ollama:/root/.ollama + networks: + - chrysopedia + healthcheck: + test: ["CMD", "ollama", "list"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 30s + stop_grace_period: 15s + + # ── FastAPI application ── + chrysopedia-api: + build: + context: . + dockerfile: docker/Dockerfile.api + container_name: chrysopedia-api + restart: unless-stopped + env_file: + - path: .env + required: false + environment: + DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-chrysopedia}:${POSTGRES_PASSWORD:-changeme}@chrysopedia-db:5432/${POSTGRES_DB:-chrysopedia} + REDIS_URL: redis://chrysopedia-redis:6379/0 + QDRANT_URL: http://chrysopedia-qdrant:6333 + EMBEDDING_API_URL: http://chrysopedia-ollama:11434/v1 + PROMPTS_PATH: /prompts + volumes: + - /vmPool/r/services/chrysopedia_data:/data + - ./config:/config:ro + depends_on: + chrysopedia-db: + condition: service_healthy + chrysopedia-redis: + condition: service_healthy + chrysopedia-qdrant: + condition: service_healthy + chrysopedia-ollama: + condition: service_healthy + networks: + - chrysopedia + stop_grace_period: 15s + + # ── Celery worker (pipeline stages 2-6) ── + chrysopedia-worker: + build: + context: . + dockerfile: docker/Dockerfile.api + container_name: chrysopedia-worker + restart: unless-stopped + env_file: + - path: .env + required: false + environment: + DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-chrysopedia}:${POSTGRES_PASSWORD:-changeme}@chrysopedia-db:5432/${POSTGRES_DB:-chrysopedia} + REDIS_URL: redis://chrysopedia-redis:6379/0 + QDRANT_URL: http://chrysopedia-qdrant:6333 + EMBEDDING_API_URL: http://chrysopedia-ollama:11434/v1 + PROMPTS_PATH: /prompts + command: ["celery", "-A", "worker", "worker", "--loglevel=info", "--concurrency=1"] + healthcheck: + test: ["CMD-SHELL", "celery -A worker inspect ping --timeout=5 2>/dev/null | grep -q pong || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s + volumes: + - /vmPool/r/services/chrysopedia_data:/data + - ./prompts:/prompts:ro + - ./config:/config:ro + depends_on: + chrysopedia-db: + condition: service_healthy + chrysopedia-redis: + condition: service_healthy + chrysopedia-qdrant: + condition: service_healthy + chrysopedia-ollama: + condition: service_healthy + networks: + - chrysopedia + stop_grace_period: 30s + + # ── Transcript folder watcher ── + chrysopedia-watcher: + build: + context: . + dockerfile: docker/Dockerfile.api + container_name: chrysopedia-watcher + restart: unless-stopped + command: ["python", "watcher.py"] + environment: + WATCHER_API_URL: http://chrysopedia-api:8000/api/v1/ingest + WATCH_FOLDER: /watch + volumes: + - /vmPool/r/services/chrysopedia_watch:/watch + depends_on: + chrysopedia-api: + condition: service_healthy + networks: + - chrysopedia + healthcheck: + test: ["CMD-SHELL", "python -c \"import os; os.kill(1, 0)\" || exit 1"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 15s + stop_grace_period: 15s + + # ── React web UI (nginx) ── + chrysopedia-web: + build: + context: . + dockerfile: docker/Dockerfile.web + args: + VITE_GIT_COMMIT: ${GIT_COMMIT_SHA:-dev} + container_name: chrysopedia-web-8096 + restart: unless-stopped + ports: + - "0.0.0.0:8096:80" + depends_on: + - chrysopedia-api + networks: + - chrysopedia + healthcheck: + test: ["CMD-SHELL", "curl -sf http://127.0.0.1:80/ || exit 1"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + stop_grace_period: 15s + +networks: + chrysopedia: + driver: bridge + ipam: + config: + - subnet: "172.32.0.0/24"