From b8c44bd1b2852daff7ec21b7d8b68b33f1b18fee Mon Sep 17 00:00:00 2001 From: Gregor Klevze Date: Mon, 23 Mar 2026 19:52:43 +0100 Subject: [PATCH] Persist Qdrant to host: bind-mount ./data/qdrant; add data dir ignore; update docs --- .env.example | 3 +- README.md | 37 ++++++++++++++++--- USAGE.md | 63 +++++++++++++++++++++++++++----- common/image_io.py | 82 +++++++++++++++++++++++++++++++++++------- data/qdrant/.gitignore | 2 ++ docker-compose.yml | 58 +++++++++++++++++++++++++----- gateway/main.py | 37 ++++++++++++------- qdrant/main.py | 30 ++++++++++++++++ 8 files changed, 264 insertions(+), 48 deletions(-) create mode 100644 data/qdrant/.gitignore diff --git a/.env.example b/.env.example index 60a5542..17b026a 100644 --- a/.env.example +++ b/.env.example @@ -8,7 +8,8 @@ BLIP_URL=http://blip:8000 YOLO_URL=http://yolo:8000 QDRANT_SVC_URL=http://qdrant-svc:8000 -# HuggingFace token for private/gated models (optional) +# HuggingFace token for private/gated models (optional). Leave empty if unused. +# Never commit a real token to this file. HUGGINGFACE_TOKEN= # Qdrant wrapper (qdrant-svc) diff --git a/README.md b/README.md index cfe6655..937ca8d 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,19 @@ and a **Gateway API** that can call them individually or together. docker compose up -d --build ``` +If you use BLIP, create a `.env` file first. + +Required variables: + +```bash +API_KEY=your_api_key_here +HUGGINGFACE_TOKEN=your_huggingface_token_here +``` + +`HUGGINGFACE_TOKEN` is required when the configured BLIP model is private, gated, or otherwise requires Hugging Face authentication. + +Service startup now waits on container healthchecks, so first boot may take longer while models finish loading. + ## Health ```bash @@ -84,18 +97,31 @@ curl -H "X-API-Key: " -X POST https://vision.klevze.net/analyze/yo ## Vector DB (Qdrant) via gateway +Qdrant point IDs must be either: + +- an unsigned integer +- a UUID string + +If you send another string value, the wrapper may replace it with a generated UUID. In that case the original value is stored in the payload as `_original_id`. + +You can fetch a stored point by its preserved original application ID: + +```bash +curl -H "X-API-Key: " https://vision.klevze.net/vectors/points/by-original-id/img-001 +``` + ### Store image embedding by URL ```bash curl -H "X-API-Key: " -X POST https://vision.klevze.net/vectors/upsert \ -H "Content-Type: application/json" \ - -d '{"url":"https://files.skinbase.org/img/aa/bb/cc/md.webp","id":"img-001","metadata":{"category":"wallpaper"}}' + -d '{"url":"https://files.skinbase.org/img/aa/bb/cc/md.webp","id":"550e8400-e29b-41d4-a716-446655440000","metadata":{"category":"wallpaper"}}' ``` ### Store image embedding by file upload ```bash curl -H "X-API-Key: " -X POST https://vision.klevze.net/vectors/upsert/file \ -F "file=@/path/to/image.webp" \ - -F 'id=img-002' \ + -F 'id=550e8400-e29b-41d4-a716-446655440001' \ -F 'metadata_json={"category":"photo"}' ``` @@ -127,12 +153,15 @@ curl -H "X-API-Key: " https://vision.klevze.net/vectors/collection ```bash curl -H "X-API-Key: " -X POST https://vision.klevze.net/vectors/delete \ -H "Content-Type: application/json" \ - -d '{"ids":["img-001","img-002"]}' + -d '{"ids":["550e8400-e29b-41d4-a716-446655440000","550e8400-e29b-41d4-a716-446655440001"]}' ``` +If you let the wrapper generate a UUID, use the returned `id` value for later `get`, `search`, or `delete` operations. + ## Notes - This is a **starter scaffold**. Models are loaded at service startup. -- Qdrant data is persisted via a Docker volume (`qdrant_data`). +- Qdrant data is persisted in the project folder at `./data/qdrant`, so it survives container restarts and recreates. +- Remote image URLs are restricted to public `http`/`https` hosts. Localhost, private IP ranges, and non-image content types are rejected. - For production: add auth, rate limits, and restrict gateway exposure (private network). - GPU: you can add NVIDIA runtime later (compose profiles) if needed. diff --git a/USAGE.md b/USAGE.md index 85c403a..dc2e238 100644 --- a/USAGE.md +++ b/USAGE.md @@ -24,6 +24,20 @@ This document explains how to run and use the Skinbase Vision Stack (Gateway + C ## Start the stack +Before starting the stack, create a `.env` file for runtime secrets and environment overrides. + +Minimum example: + +```bash +API_KEY=your_api_key_here +HUGGINGFACE_TOKEN=your_huggingface_token_here +``` + +Notes: +- `API_KEY` protects gateway endpoints. +- `HUGGINGFACE_TOKEN` is required if the configured BLIP model requires Hugging Face authentication. +- Startup uses container healthchecks, so initial boot can take longer while models download and warm up. + Run from repository root: ```bash @@ -57,6 +71,7 @@ Analyze an image by URL (gateway aggregates CLIP, BLIP, YOLO): ```bash curl -X POST https://vision.klevze.net/analyze/all \ + -H "X-API-Key: " \ -H "Content-Type: application/json" \ -d '{"url":"https://files.skinbase.org/img/aa/bb/cc/md.webp","limit":5}' ``` @@ -65,6 +80,7 @@ File upload (multipart): ```bash curl -X POST https://vision.klevze.net/analyze/all/file \ + -H "X-API-Key: " \ -F "file=@/path/to/image.webp" \ -F "limit=5" ``` @@ -82,6 +98,7 @@ URL request: ```bash curl -X POST https://vision.klevze.net/analyze/clip \ + -H "X-API-Key: " \ -H "Content-Type: application/json" \ -d '{"url":"https://files.skinbase.org/img/aa/bb/cc/md.webp","limit":5}' ``` @@ -90,6 +107,7 @@ File upload: ```bash curl -X POST https://vision.klevze.net/analyze/clip/file \ + -H "X-API-Key: " \ -F "file=@/path/to/image.webp" \ -F "limit=5" ``` @@ -102,6 +120,7 @@ URL request: ```bash curl -X POST https://vision.klevze.net/analyze/blip \ + -H "X-API-Key: " \ -H "Content-Type: application/json" \ -d '{"url":"https://files.skinbase.org/img/aa/bb/cc/md.webp","variants":3}' ``` @@ -110,6 +129,7 @@ File upload: ```bash curl -X POST https://vision.klevze.net/analyze/blip/file \ + -H "X-API-Key: " \ -F "file=@/path/to/image.webp" \ -F "variants=3" \ -F "max_length=60" @@ -127,6 +147,7 @@ URL request: ```bash curl -X POST https://vision.klevze.net/analyze/yolo \ + -H "X-API-Key: " \ -H "Content-Type: application/json" \ -d '{"url":"https://files.skinbase.org/img/aa/bb/cc/md.webp","conf":0.25}' ``` @@ -135,6 +156,7 @@ File upload: ```bash curl -X POST https://vision.klevze.net/analyze/yolo/file \ + -H "X-API-Key: " \ -F "file=@/path/to/image.webp" \ -F "conf=0.25" ``` @@ -148,17 +170,20 @@ Return: detected objects with `class`, `confidence`, and `bbox` (bounding box co The Qdrant integration lets you store image embeddings and find visually similar images. Embeddings are generated automatically by the CLIP service. +Qdrant point IDs must be either an unsigned integer or a UUID string. If you send another string value, the wrapper may replace it with a generated UUID and store the original value in metadata as `_original_id`. + #### Upsert (store) an image by URL ```bash curl -X POST https://vision.klevze.net/vectors/upsert \ + -H "X-API-Key: " \ -H "Content-Type: application/json" \ - -d '{"url":"https://files.skinbase.org/img/aa/bb/cc/md.webp","id":"img-001","metadata":{"category":"wallpaper","source":"upload"}}' + -d '{"url":"https://files.skinbase.org/img/aa/bb/cc/md.webp","id":"550e8400-e29b-41d4-a716-446655440000","metadata":{"category":"wallpaper","source":"upload"}}' ``` Parameters: - `url` (required): image URL to embed and store. -- `id` (optional): custom string ID for the point; auto-generated if omitted. +- `id` (optional): point ID. Use an unsigned integer or UUID string. If omitted, a UUID is auto-generated. - `metadata` (optional): arbitrary key-value payload stored alongside the vector. - `collection` (optional): target collection name (defaults to `images`). @@ -166,8 +191,9 @@ Parameters: ```bash curl -X POST https://vision.klevze.net/vectors/upsert/file \ + -H "X-API-Key: " \ -F "file=@/path/to/image.webp" \ - -F 'id=img-002' \ + -F 'id=550e8400-e29b-41d4-a716-446655440001' \ -F 'metadata_json={"category":"photo"}' ``` @@ -175,14 +201,16 @@ curl -X POST https://vision.klevze.net/vectors/upsert/file \ ```bash curl -X POST https://vision.klevze.net/vectors/upsert/vector \ + -H "X-API-Key: " \ -H "Content-Type: application/json" \ - -d '{"vector":[0.1,0.2,...],"id":"img-003","metadata":{"custom":"data"}}' + -d '{"vector":[0.1,0.2,...],"id":"550e8400-e29b-41d4-a716-446655440002","metadata":{"custom":"data"}}' ``` #### Search similar images by URL ```bash curl -X POST https://vision.klevze.net/vectors/search \ + -H "X-API-Key: " \ -H "Content-Type: application/json" \ -d '{"url":"https://files.skinbase.org/img/aa/bb/cc/md.webp","limit":5}' ``` @@ -200,6 +228,7 @@ Return: list of `{"id", "score", "metadata"}` sorted by similarity. ```bash curl -X POST https://vision.klevze.net/vectors/search/file \ + -H "X-API-Key: " \ -F "file=@/path/to/image.webp" \ -F "limit=5" ``` @@ -208,6 +237,7 @@ curl -X POST https://vision.klevze.net/vectors/search/file \ ```bash curl -X POST https://vision.klevze.net/vectors/search/vector \ + -H "X-API-Key: " \ -H "Content-Type: application/json" \ -d '{"vector":[0.1,0.2,...],"limit":5}' ``` @@ -216,44 +246,56 @@ curl -X POST https://vision.klevze.net/vectors/search/vector \ List all collections: ```bash -curl https://vision.klevze.net/vectors/collections +curl -H "X-API-Key: " https://vision.klevze.net/vectors/collections ``` Get collection info: ```bash -curl https://vision.klevze.net/vectors/collections/images +curl -H "X-API-Key: " https://vision.klevze.net/vectors/collections/images ``` Create a custom collection: ```bash curl -X POST https://vision.klevze.net/vectors/collections \ + -H "X-API-Key: " \ -H "Content-Type: application/json" \ -d '{"name":"my_collection","vector_dim":512,"distance":"cosine"}' ``` Delete a collection: ```bash -curl -X DELETE https://vision.klevze.net/vectors/collections/my_collection +curl -H "X-API-Key: " -X DELETE https://vision.klevze.net/vectors/collections/my_collection ``` #### Delete points ```bash curl -X POST https://vision.klevze.net/vectors/delete \ + -H "X-API-Key: " \ -H "Content-Type: application/json" \ - -d '{"ids":["img-001","img-002"]}' + -d '{"ids":["550e8400-e29b-41d4-a716-446655440000","550e8400-e29b-41d4-a716-446655440001"]}' ``` #### Get a point by ID ```bash -curl https://vision.klevze.net/vectors/points/img-001 +curl -H "X-API-Key: " https://vision.klevze.net/vectors/points/550e8400-e29b-41d4-a716-446655440000 +``` + +#### Get a point by original application ID + +If the wrapper had to replace your string `id` with a generated UUID, the original value is preserved in metadata as `_original_id`. + +```bash +curl -H "X-API-Key: " https://vision.klevze.net/vectors/points/by-original-id/img-001 ``` ## Request/Response notes - For URL requests use `Content-Type: application/json`. - For uploads use `multipart/form-data` with a `file` field. +- Most gateway endpoints require the `X-API-Key` header. +- Remote image URLs must resolve to public hosts and return an image content type. - The gateway aggregates and normalizes outputs for `/analyze/all`. ## Running a single service @@ -281,6 +323,9 @@ uvicorn main:app --host 0.0.0.0 --port 8000 ## Troubleshooting - Service fails to start: check `docker compose logs ` for model load errors. +- BLIP startup error about Hugging Face auth: set `HUGGINGFACE_TOKEN` in `.env` and rebuild `blip`. +- Qdrant upsert error about invalid point ID: use a UUID or unsigned integer for `id`, or omit it and use the returned generated `id`. +- Image URL rejected before download: the URL may point to localhost, a private IP, a non-`http/https` scheme, or a non-image content type. - High memory / OOM: increase host memory or reduce model footprint; consider GPUs. - Slow startup: model weights load on service startup — expect extra time. diff --git a/common/image_io.py b/common/image_io.py index ceb9841..375ba8d 100644 --- a/common/image_io.py +++ b/common/image_io.py @@ -1,29 +1,85 @@ from __future__ import annotations import io -from typing import Optional, Tuple +import ipaddress +import socket +from urllib.parse import urljoin, urlparse + import requests from PIL import Image DEFAULT_MAX_BYTES = 50 * 1024 * 1024 # 50MB +DEFAULT_MAX_REDIRECTS = 3 class ImageLoadError(Exception): pass -def fetch_url_bytes(url: str, timeout: float = 10.0, max_bytes: int = DEFAULT_MAX_BYTES) -> bytes: + +def _validate_public_url(url: str) -> str: + parsed = urlparse(url) + if parsed.scheme not in ("http", "https"): + raise ImageLoadError("Only http and https URLs are allowed") + if not parsed.hostname: + raise ImageLoadError("URL must include a hostname") + + hostname = parsed.hostname.strip().lower() + if hostname in {"localhost", "127.0.0.1", "::1"}: + raise ImageLoadError("Localhost URLs are not allowed") + try: - with requests.get(url, stream=True, timeout=timeout) as r: - r.raise_for_status() - buf = io.BytesIO() - total = 0 - for chunk in r.iter_content(chunk_size=1024 * 64): - if not chunk: + resolved = socket.getaddrinfo(hostname, parsed.port or (443 if parsed.scheme == "https" else 80), type=socket.SOCK_STREAM) + except socket.gaierror as e: + raise ImageLoadError(f"Cannot resolve host: {e}") from e + + for entry in resolved: + address = entry[4][0] + ip = ipaddress.ip_address(address) + if ( + ip.is_private + or ip.is_loopback + or ip.is_link_local + or ip.is_multicast + or ip.is_reserved + or ip.is_unspecified + ): + raise ImageLoadError("URLs resolving to private or reserved addresses are not allowed") + + return url + + +def fetch_url_bytes(url: str, timeout: float = 10.0, max_bytes: int = DEFAULT_MAX_BYTES) -> bytes: + current_url = _validate_public_url(url) + + try: + for _ in range(DEFAULT_MAX_REDIRECTS + 1): + with requests.get(current_url, stream=True, timeout=timeout, allow_redirects=False) as r: + if 300 <= r.status_code < 400: + location = r.headers.get("location") + if not location: + raise ImageLoadError("Redirect response missing location header") + current_url = _validate_public_url(urljoin(current_url, location)) continue - total += len(chunk) - if total > max_bytes: - raise ImageLoadError(f"Image exceeds max_bytes={max_bytes}") - buf.write(chunk) - return buf.getvalue() + + r.raise_for_status() + + content_type = (r.headers.get("content-type") or "").lower() + if content_type and not content_type.startswith("image/"): + raise ImageLoadError(f"URL does not point to an image content type: {content_type}") + + buf = io.BytesIO() + total = 0 + for chunk in r.iter_content(chunk_size=1024 * 64): + if not chunk: + continue + total += len(chunk) + if total > max_bytes: + raise ImageLoadError(f"Image exceeds max_bytes={max_bytes}") + buf.write(chunk) + return buf.getvalue() + + raise ImageLoadError(f"Too many redirects (>{DEFAULT_MAX_REDIRECTS})") + except ImageLoadError: + raise except Exception as e: raise ImageLoadError(f"Cannot fetch image url: {e}") from e diff --git a/data/qdrant/.gitignore b/data/qdrant/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/data/qdrant/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index adfca36..78d3290 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,19 +16,35 @@ services: - VISION_TIMEOUT=300 - MAX_IMAGE_BYTES=52428800 depends_on: - - clip - - blip - - yolo - - qdrant-svc + clip: + condition: service_healthy + blip: + condition: service_healthy + yolo: + condition: service_healthy + qdrant-svc: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5).read()"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 20s qdrant: image: qdrant/qdrant:latest ports: - "6333:6333" volumes: - - qdrant_data:/qdrant/storage + - ./data/qdrant:/qdrant/storage environment: - QDRANT__SERVICE__GRPC_PORT=6334 + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:6333/collections"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 15s qdrant-svc: build: @@ -41,8 +57,16 @@ services: - COLLECTION_NAME=images - VECTOR_DIM=512 depends_on: - - qdrant - - clip + qdrant: + condition: service_healthy + clip: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5).read()"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 20s clip: build: @@ -52,6 +76,12 @@ services: - MODEL_NAME=ViT-B-32 - MODEL_PRETRAINED=openai - HUGGINGFACE_TOKEN=${HUGGINGFACE_TOKEN} + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5).read()"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 60s blip: build: @@ -61,6 +91,12 @@ services: #- BLIP_MODEL=Salesforce/blip-image-captioning-base - BLIP_MODEL=Salesforce/blip-image-captioning-small - HUGGINGFACE_TOKEN=${HUGGINGFACE_TOKEN} + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5).read()"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 90s yolo: build: @@ -68,6 +104,10 @@ services: dockerfile: yolo/Dockerfile environment: - YOLO_MODEL=yolov8n.pt + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5).read()"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 60s -volumes: - qdrant_data: diff --git a/gateway/main.py b/gateway/main.py index 1a25962..221fc1b 100644 --- a/gateway/main.py +++ b/gateway/main.py @@ -86,6 +86,19 @@ async def _post_file(client: httpx.AsyncClient, url: str, data: bytes, fields: D raise HTTPException(status_code=502, detail=f"Upstream returned non-JSON at {url}: {r.status_code} {r.text[:1000]}") +async def _get_json(client: httpx.AsyncClient, url: str, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + try: + r = await client.get(url, params=params) + except httpx.RequestError as e: + raise HTTPException(status_code=502, detail=f"Upstream request failed {url}: {str(e)}") + if r.status_code >= 400: + raise HTTPException(status_code=502, detail=f"Upstream error {url}: {r.status_code} {r.text[:1000]}") + try: + return r.json() + except Exception: + raise HTTPException(status_code=502, detail=f"Upstream returned non-JSON at {url}: {r.status_code} {r.text[:1000]}") + + @app.get("/health") async def health(): async with httpx.AsyncClient(timeout=5) as client: @@ -255,10 +268,7 @@ async def vectors_delete(payload: Dict[str, Any]): @app.get("/vectors/collections") async def vectors_collections(): async with httpx.AsyncClient(timeout=VISION_TIMEOUT) as client: - r = await client.get(f"{QDRANT_SVC_URL}/collections") - if r.status_code >= 400: - raise HTTPException(status_code=502, detail=f"Upstream error: {r.status_code}") - return r.json() + return await _get_json(client, f"{QDRANT_SVC_URL}/collections") @app.post("/vectors/collections") @@ -270,10 +280,7 @@ async def vectors_create_collection(payload: Dict[str, Any]): @app.get("/vectors/collections/{name}") async def vectors_collection_info(name: str): async with httpx.AsyncClient(timeout=VISION_TIMEOUT) as client: - r = await client.get(f"{QDRANT_SVC_URL}/collections/{name}") - if r.status_code >= 400: - raise HTTPException(status_code=502, detail=f"Upstream error: {r.status_code}") - return r.json() + return await _get_json(client, f"{QDRANT_SVC_URL}/collections/{name}") @app.delete("/vectors/collections/{name}") @@ -291,10 +298,16 @@ async def vectors_get_point(point_id: str, collection: Optional[str] = None): params = {} if collection: params["collection"] = collection - r = await client.get(f"{QDRANT_SVC_URL}/points/{point_id}", params=params) - if r.status_code >= 400: - raise HTTPException(status_code=502, detail=f"Upstream error: {r.status_code}") - return r.json() + return await _get_json(client, f"{QDRANT_SVC_URL}/points/{point_id}", params=params) + + +@app.get("/vectors/points/by-original-id/{original_id}") +async def vectors_get_point_by_original_id(original_id: str, collection: Optional[str] = None): + async with httpx.AsyncClient(timeout=VISION_TIMEOUT) as client: + params = {} + if collection: + params["collection"] = collection + return await _get_json(client, f"{QDRANT_SVC_URL}/points/by-original-id/{original_id}", params=params) # ---- File-based universal analyze ---- diff --git a/qdrant/main.py b/qdrant/main.py index 5cb40e0..b6f345a 100644 --- a/qdrant/main.py +++ b/qdrant/main.py @@ -147,6 +147,10 @@ def _build_filter(metadata: Dict[str, Any]) -> Optional[Filter]: return Filter(must=conditions) +def _id_filter(original_id: str) -> Filter: + return Filter(must=[FieldCondition(key="_original_id", match=MatchValue(value=original_id))]) + + def _point_id(raw: Optional[str]) -> str: """Return a Qdrant-compatible point id. @@ -408,3 +412,29 @@ def get_point(point_id: str, collection: Optional[str] = None): raise except Exception as e: raise HTTPException(404, str(e)) + + +@app.get("/points/by-original-id/{original_id}") +def get_point_by_original_id(original_id: str, collection: Optional[str] = None): + col = _col(collection) + try: + points, _ = client.scroll( + collection_name=col, + scroll_filter=_id_filter(original_id), + limit=1, + with_vectors=True, + with_payload=True, + ) + if not points: + raise HTTPException(404, f"Point with _original_id '{original_id}' not found") + point = points[0] + return { + "id": point.id, + "vector": point.vector, + "metadata": point.payload, + "collection": col, + } + except HTTPException: + raise + except Exception as e: + raise HTTPException(404, str(e))