#!/bin/bash

export VLLM_ALLOW_RUNTIME_LORA_UPDATING=True

vllm serve $MODEL \
     --enable-lora \
     --dtype bfloat16 \
     --max-lora-rank 64 \
     --enable-prefix-caching \
     > $(readlink -ef $(dirname $0))/vllm.log \
     2> $(readlink -ef $(dirname $0))/vllm.err


# --lora-modules "{\"name\": \"myadapter\", \"path\": \"$ADAPTER\"}" \                                                                                                                                                                   
# --lora-modules "{\"name\": \"myadapter\", \"path\": \"$ADAPTER\", \"base_model_name\": \"$MODEL\"}" \