IMAGE="${IMAGE:-llama-turboquant:cuda}" HOST_PORT="${HOST_PORT:-8080}" NETWORK="${NETWORK:-host}" STATIC_IP="${STATIC_IP:-172.18.0.10}" # 65 blocks (64 layers + 1 MTP) MODEL_FILE="${MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}" MMPROJ_FILE="${MMPROJ_FILE:-mmproj-Qwen3.8-27B-F16.gguf}" DRAFT_FILE="${DRAFT_FILE:-Qwen3.8-27B-DFlash2-Q4_K_M-mrope.gguf}" N_GPU_LAYERS="${N_GPU_LAYERS:-999}" PARALLEL="${PARALLEL:-1}" CONTEXT="${CONTEXT:-140000}" NAME="${NAME:-llama-turboquant}" if docker inspect "${NAME}" >/dev/null 2>&1; then docker rm -f "${NAME}" >/dev/null fi docker create \ --name "${NAME}" \ --restart=unless-stopped \ --gpus all \ -e CUDA_DEVICE_ORDER=PCI_BUS_ID \ -e CUDA_VISIBLE_DEVICES=1,0 \ -e GGML_CUDA_DISABLE_GRAPHS=1 \ --memory=30g \ --memory-swap=46g \ --cap-add=IPC_LOCK \ --ulimit memlock=-1:-1 \ --ulimit core=0 \ --network "${NETWORK}" \ -v "$(pwd)/models:/models:ro" \ -v "$(pwd)/scripts:/scripts:ro" \ --entrypoint /scripts/entrypoint.sh \ "${IMAGE}" \ --model "/models/${MODEL_FILE}" \ --model-draft "/models/${DRAFT_FILE}" \ --mmproj "/models/${MMPROJ_FILE}" \ --mmproj-offload \ --image-min-tokens 1024 \ --metrics \ --n-gpu-layers "${N_GPU_LAYERS}" \ --main-gpu 0 \ --split-mode layer \ --tensor-split 39,26 \ -fit off \ --flash-attn on \ -c "${CONTEXT}" \ -n -1 \ --parallel "${PARALLEL}" \ --kv-unified \ -ctk q8_0 \ -ctv q8_0 \ -ctkd q8_0 \ -ctvd q8_0 \ --load-mode mlock \ --jinja \ --reasoning on \ --reasoning-preserve \ --reasoning-effort medium \ --spec-type draft-dflash,ngram-mod \ --spec-draft-n-max 5 \ --spec-ngram-mod-n-match 24 \ --spec-ngram-mod-n-min 4 \ --spec-ngram-mod-n-max 48 \ --temp 1.0 \ --top-p 0.95 \ --top-k 20 \ --min-p 0.0 \ --presence-penalty 0.0 \ --repeat-penalty 1.0 \ -b 2048 \ -ub 256 \ --cache-idle-slots \ --cache-ram 12288 \ -ctxcp 16 \ --threads 8 \ --cpu-range 0-7 \ --timeout 360 \ "$@" >/dev/null docker start -a "${NAME}"