IMAGE="${IMAGE:-llama-turboquant:cuda}"
HOST_PORT="${HOST_PORT:-8080}"
NETWORK="${NETWORK:-host}"
STATIC_IP="${STATIC_IP:-172.18.0.10}"
# 65 blocks (64 layers + 1 MTP)
MODEL_FILE="${MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}"
MMPROJ_FILE="${MMPROJ_FILE:-mmproj-Qwen3.8-27B-F16.gguf}"
DRAFT_FILE="${DRAFT_FILE:-Qwen3.8-27B-DFlash2-Q4_K_M-mrope.gguf}"
N_GPU_LAYERS="${N_GPU_LAYERS:-999}"
 
PARALLEL="${PARALLEL:-1}"
CONTEXT="${CONTEXT:-140000}"
 
NAME="${NAME:-llama-turboquant}"
 
if docker inspect "${NAME}" >/dev/null 2>&1; then
    docker rm -f "${NAME}" >/dev/null
fi
 
docker create \

    --name "${NAME}" \

    --restart=unless-stopped \

    --gpus all \

    -e CUDA_DEVICE_ORDER=PCI_BUS_ID \

    -e CUDA_VISIBLE_DEVICES=1,0 \

    -e GGML_CUDA_DISABLE_GRAPHS=1 \

    --memory=30g \

    --memory-swap=46g \

    --cap-add=IPC_LOCK \

    --ulimit memlock=-1:-1 \

    --ulimit core=0 \

    --network "${NETWORK}" \

    -v "$(pwd)/models:/models:ro" \

    -v "$(pwd)/scripts:/scripts:ro" \

    --entrypoint /scripts/entrypoint.sh \

    "${IMAGE}" \

    --model "/models/${MODEL_FILE}" \

    --model-draft "/models/${DRAFT_FILE}" \

    --mmproj "/models/${MMPROJ_FILE}" \

    --mmproj-offload \

    --image-min-tokens 1024 \

    --metrics \

    --n-gpu-layers "${N_GPU_LAYERS}" \

    --main-gpu 0 \

    --split-mode layer \

    --tensor-split 39,26 \

    -fit off \

    --flash-attn on \

    -c "${CONTEXT}" \

    -n -1 \

    --parallel "${PARALLEL}" \

    --kv-unified \

    -ctk q8_0 \

    -ctv q8_0 \

    -ctkd q8_0 \

    -ctvd q8_0 \

    --load-mode mlock \

    --jinja \

    --reasoning on \

    --reasoning-preserve \

    --reasoning-effort medium \

    --spec-type draft-dflash,ngram-mod \

    --spec-draft-n-max 5 \

    --spec-ngram-mod-n-match 24 \

    --spec-ngram-mod-n-min 4 \

    --spec-ngram-mod-n-max 48 \

    --temp 1.0 \

    --top-p 0.95 \

    --top-k 20 \

    --min-p 0.0 \

    --presence-penalty 0.0 \

    --repeat-penalty 1.0 \

    -b 2048 \

    -ub 256 \

    --cache-idle-slots \

    --cache-ram 12288 \

    -ctxcp 16 \

    --threads 8 \

    --cpu-range 0-7 \

    --timeout 360 \

    "$@" >/dev/null
 
docker start -a "${NAME}"

Изменить пасту