FastChat Docker & Self-Hosted Deployment

GitHub

An open platform for training, serving, and evaluating large language models. Release repo for Vicuna and Chatbot Arena.

39,548 stars Python Python 1 file · ~23,358 tokens
RAW Compose
1. Pull Image (DOCKERHUB) docker pull library/fastchat
2. Docker Run (1-Click Launch) docker run -d --name fastchat -p 21001:21001 --restart always fastchat:latest
3. Podman Run podman run -d --name fastchat -p 21001:21001 --restart always fastchat:latest
4. Skopeo Inspect (Zero-Pull Manifest) skopeo inspect docker://docker.io/library/fastchat:latest
5. ORAS Pull (OCI Artifact) oras pull fastchat:latest
6. Compose Deployment docker compose up -d
docker/docker-compose.yml
Download .yml 
version: "3.9"

services:
  fastchat-controller:
    build:
      context: .
      dockerfile: Dockerfile
    image: fastchat:latest
    ports:
      - "21001:21001"
    entrypoint: ["python3.9", "-m", "fastchat.serve.controller", "--host", "0.0.0.0", "--port", "21001"]
  fastchat-model-worker:
    build:
      context: .
      dockerfile: Dockerfile
    volumes:
      - huggingface:/root/.cache/huggingface
    image: fastchat:latest
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 1
              capabilities: [gpu]
    entrypoint: ["python3.9", "-m", "fastchat.serve.model_worker", "--model-names", "${FASTCHAT_WORKER_MODEL_NAMES:-vicuna-7b-v1.5}", "--model-path", "${FASTCHAT_WORKER_MODEL_PATH:-lmsys/vicuna-7b-v1.5}", "--worker-address", "http://fastchat-model-worker:21002", "--controller-address", "http://fastchat-controller:21001", "--host", "0.0.0.0", "--port", "21002"]
  fastchat-api-server:
    build:
      context: .
      dockerfile: Dockerfile
    image: fastchat:latest
    ports:
      - "8000:8000"
    entrypoint: ["python3.9", "-m", "fastchat.serve.openai_api_server", "--controller-address", "http://fastchat-controller:21001", "--host", "0.0.0.0", "--port", "8000"]
volumes:
  huggingface: