DeepSeek/DeepSeek-VL
1
0
Fork 0
mirror of https://github.com/deepseek-ai/DeepSeek-VL synced 2025-06-26 18:27:43 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

load model on detected device and use correct dtype

Browse Source
This commit is contained in:
Nicola Dall'Asen
2024-03-13 12:32:14 +01:00
parent 494e622544
commit 40ec6491d4
3 changed files with 11 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
cli_chat.py
View File

@@ -8,7 +8,7 @@ from threading import Thread
import torch
from transformers import TextIteratorStreamer
from deepseek_vl.utils.io import load_pretrained_model
from deepseek_vl.utils.io import load_pretrained_model, get_device_and_dtype
def load_image(image_file):
@@ -34,13 +34,13 @@ def get_help_message(image_token):
@torch.inference_mode()
def response(args, conv, pil_images, tokenizer, vl_chat_processor, vl_gpt, generation_config):
_, dtype = get_device_and_dtype()
prompt = conv.get_prompt()
prepare_inputs = vl_chat_processor.__call__(
prompt=prompt,
images=pil_images,
force_batchify=True
).to(vl_gpt.device)
).to(vl_gpt.device, dtype=dtype)
# run image encoder to get the image embeddings
inputs_embeds = vl_gpt.prepare_inputs_embeds(**prepare_inputs)