update

Gradio Demo Example, Incremental Prefilling and VLMEvalKit Support

deepseek_vl2/serve/app_modules/presets.py

@@ -21,7 +21,7 @@
 import gradio as gr
 
 title = """<h1 align="left" style="min-width:200px; margin-top:0;">Chat with DeepSeek-VL2 </h1>"""
-description_top = """"""
+description_top = """Special Tokens: `<image>`,     Visual Grounding: `<|ref|>{query}<|/ref|>`,    Grounding Conversation: `<|grounding|>{question}`"""
 description = """"""
 CONCURRENT_COUNT = 1
 MAX_EVENTS = 10

deepseek_vl2/serve/app_modules/utils.py

@@ -242,7 +242,9 @@ def pil_to_base64(
     alt: str = "user upload image",
     resize: bool = True,
     max_size: int = MAX_IMAGE_SIZE,
-    min_size: int = MIN_IMAGE_SIZE
+    min_size: int = MIN_IMAGE_SIZE,
+    format: str = "JPEG",
+    quality: int = 95
 ) -> str:
 
     if resize:
@@ -258,15 +260,16 @@ def pil_to_base64(
         image = image.resize((W, H))
 
     buffered = io.BytesIO()
-    image.save(buffered, format="JPEG")
+    image.save(buffered, format=format, quality=quality)
     img_b64_str = base64.b64encode(buffered.getvalue()).decode()
     img_str = f'<img src="data:image/png;base64,{img_b64_str}" alt="{alt}" />'
 
     return img_str
 
 
-def parse_ref_bbox(response, image):
+def parse_ref_bbox(response, image: Image.Image):
     try:
+        image = image.copy()
         image_h, image_w = image.size
         draw = ImageDraw.Draw(image)
 
@@ -275,7 +278,7 @@ def parse_ref_bbox(response, image):
         assert len(ref) == len(bbox)
 
         if len(ref) == 0:
-            return
+            return None
 
         boxes, labels = [], []
         for box, label in zip(bbox, ref):
@@ -301,9 +304,30 @@ def parse_ref_bbox(response, image):
             text_x = box[0]
             text_y = box[1] - 20
             text_color = box_color
-            font = ImageFont.truetype('./deepseek_vl2/serve/assets/simsun.ttc', size=20)
+            font = ImageFont.truetype("deepseek_vl2/serve/assets/simsun.ttc", size=20)
             draw.text((text_x, text_y), label, font=font, fill=text_color)
 
+        # print(f"boxes = {boxes}, labels = {labels}, re-render = {image}")
         return image
     except:
-        return
+        return None
+
+
+def display_example(image_list):
+    images_html = ""
+    for i, img_path in enumerate(image_list):
+        image = Image.open(img_path)
+        buffered = io.BytesIO()
+        image.save(buffered, format="PNG", quality=100)
+        img_b64_str = base64.b64encode(buffered.getvalue()).decode()
+        img_str = f'<img src="data:image/png;base64,{img_b64_str}" alt="{img_path}" style="height:80px; margin-right: 10px;" />'
+        images_html += img_str
+
+    result_html = f"""
+    <div style="display: flex; align-items: center; margin-bottom: 10px;">
+        <div style="flex: 1; margin-right: 10px;">{images_html}</div>
+    </div>
+    """
+
+    return result_html
+

deepseek_vl2/serve/inference.py

@@ -47,24 +47,27 @@ def load_model(model_path, dtype=torch.bfloat16):
 
 def convert_conversation_to_prompts(conversation: Conversation):
     conv_prompts = []
-    pil_images = []
+
+    last_image = None
+
     messages = conversation.messages
     for i in range(0, len(messages), 2):
 
         if isinstance(messages[i][1], tuple):
             text, images = messages[i][1]
+            last_image = images[-1]
         else:
             text, images = messages[i][1], []
-        pil_images.extend(images)
 
         prompt = {
             "role": messages[i][0],
             "content": text,
+            "images": images
         }
         response = {"role": messages[i + 1][0], "content": messages[i + 1][1]}
         conv_prompts.extend([prompt, response])
 
-    return conv_prompts, pil_images
+    return conv_prompts, last_image
 
 
 class StoppingCriteriaSub(StoppingCriteria):
@@ -86,8 +89,7 @@ class StoppingCriteriaSub(StoppingCriteria):
 
 @torch.inference_mode()
 def deepseek_generate(
-    conv_prompts: list,
-    pil_images: list,
+    conversations: list,
     vl_gpt: torch.nn.Module,
     vl_chat_processor: DeepseekVLV2Processor,
     tokenizer: transformers.PreTrainedTokenizer,
@@ -95,11 +97,17 @@ def deepseek_generate(
     max_length: int = 256,
     temperature: float = 1.0,
     top_p: float = 1.0,
-    repetition_penalty=1.1,
+    repetition_penalty: float = 1.1,
+    chunk_size: int = -1
 ):
+    pil_images = []
+    for message in conversations:
+        if "images" not in message:
+            continue
+        pil_images.extend(message["images"])
 
     prepare_inputs = vl_chat_processor.__call__(
-        conversations=conv_prompts,
+        conversations=conversations,
         images=pil_images,
         inference_mode=True,
         force_batchify=True,
@@ -110,11 +118,12 @@ def deepseek_generate(
         vl_gpt,
         tokenizer,
         prepare_inputs,
-        max_length,
-        temperature,
-        repetition_penalty,
-        top_p,
-        stop_words,
+        max_gen_len=max_length,
+        temperature=temperature,
+        repetition_penalty=repetition_penalty,
+        top_p=top_p,
+        stop_words=stop_words,
+        chunk_size=chunk_size
     )
 
 
@@ -128,11 +137,10 @@ def generate(
     repetition_penalty=1.1,
     top_p: float = 0.95,
     stop_words: List[str] = [],
+    chunk_size: int = -1
 ):
     """Stream the text output from the multimodality model with prompt and image inputs."""
-    inputs_embeds = vl_gpt.prepare_inputs_embeds(**prepare_inputs)
-
-    streamer = TextIteratorStreamer(tokenizer)
+    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True)
 
     stop_words_ids = [
         torch.tensor(tokenizer.encode(stop_word)) for stop_word in stop_words
@@ -141,9 +149,27 @@ def generate(
         [StoppingCriteriaSub(stops=stop_words_ids)]
     )
 
+    if chunk_size != -1:
+        inputs_embeds, past_key_values = vl_gpt.incremental_prefilling(
+            input_ids=prepare_inputs.input_ids,
+            images=prepare_inputs.images,
+            images_seq_mask=prepare_inputs.images_seq_mask,
+            images_spatial_crop=prepare_inputs.images_spatial_crop,
+            attention_mask=prepare_inputs.attention_mask,
+            chunk_size=chunk_size
+        )
+    else:
+        inputs_embeds = vl_gpt.prepare_inputs_embeds(**prepare_inputs)
+        past_key_values = None
+
     generation_config = dict(
         inputs_embeds=inputs_embeds,
+        input_ids=prepare_inputs.input_ids,
+        images=prepare_inputs.images,
+        images_seq_mask=prepare_inputs.images_seq_mask,
+        images_spatial_crop=prepare_inputs.images_spatial_crop,
         attention_mask=prepare_inputs.attention_mask,
+        past_key_values=past_key_values,
         pad_token_id=tokenizer.eos_token_id,
         bos_token_id=tokenizer.bos_token_id,
         eos_token_id=tokenizer.eos_token_id,