update to support lora, added new examples, updated prompt

LLaVA

@@ -1 +1 @@
-Subproject commit a54459c2dd993c4fd2f571cd16bf73cb8dbdcb00
+Subproject commit 30e73a40fe42f392dac3ad9466b3d62e1a40ad07

app.py

@@ -25,7 +25,7 @@ def start_controller():
  return subprocess.Popen(controller_command)
 
 
-def start_worker(model_path: str, bits=16, revision='main', port=21002):
+def start_worker(model_path: str, bits=16, revision='main', model_base = None, port=21002):
  print(f"Starting the model worker for the model {model_path}")
  model_name = model_path.strip("/").split("/")[-1]
  assert bits in [4, 8, 16], "It can be only loaded with 16-bit, 8-bit, and 4-bit."
@@ -49,8 +49,13 @@ def start_worker(model_path: str, bits=16, revision='main', port=21002):
  model_name,
  "--use-flash-attn",
  "--revision",
- revision
+ revision,
  ]
+ if model_base:
+ worker_command += [
+ "--model-base",
+ model_base
+ ]
  if bits != 16:
  worker_command += [f"--load-{bits}bit"]
  print(worker_command)
@@ -84,6 +89,7 @@ Set the environment variable `model` to change the model:
  print(f"args: {gws.args}")
 
  model_paths = os.getenv("model", "nvandal/LLaVA-Med-v1.5-7b")
+ model_base = os.getenv("model_base", '')
  revisions = os.getenv("revision", "main")
  bits = int(os.getenv("bits", 4))
  concurrency_count = int(os.getenv("concurrency_count", 5))
@@ -93,11 +99,12 @@ Set the environment variable `model` to change the model:
 
  model_paths = model_paths.split(';')
  revisions = revisions.split(';')
+ model_base = model_base.split(';')
  assert(len(model_paths)==len(revisions))
  worker_proc = [None]*len(model_paths)
- for i, (model_path, revision) in enumerate(zip(model_paths,revisions)):
- print(model_path, revision)
- worker_proc[i] = start_worker(model_path, bits=bits, revision=revision, port=str(start_worker_port+i))
+ for i, (model_path, revision, model_base) in enumerate(zip(model_paths,revisions,model_base)):
+ print(model_path, revision, model_base)
+ worker_proc[i] = start_worker(model_path, bits=bits, revision=revision, model_base=model_base, port=str(start_worker_port+i))
 
  # Wait for worker and controller to start
  time.sleep(10)