feat: add multiple images (or in-context learning) conversation examples (#47)

Co-authored-by: Bo Liu <benjaminliu.eecs@gmail.com>
This commit is contained in:
StevenLiuWen
2024-04-16 13:58:43 +08:00
committed by GitHub
parent 3c02b24219
commit 9bb02cc50d
8 changed files with 121 additions and 5 deletions

View File

@@ -132,18 +132,34 @@ tokenizer = vl_chat_processor.tokenizer
vl_gpt: MultiModalityCausalLM = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True)
vl_gpt = vl_gpt.to(torch.bfloat16).cuda().eval()
## single image conversation example
conversation = [
{
"role": "User",
"content": "<image_placeholder>Describe each stage of this image.",
"images": ["./images/training_pipelines.jpg"]
"images": ["./images/training_pipelines.jpg"],
},
{
"role": "Assistant",
"content": ""
}
{"role": "Assistant", "content": ""},
]
## multiple images (or in-context learning) conversation example
# conversation = [
# {
# "role": "User",
# "content": "<image_placeholder>A dog wearing nothing in the foreground, "
# "<image_placeholder>a dog wearing a santa hat, "
# "<image_placeholder>a dog wearing a wizard outfit, and "
# "<image_placeholder>what's the dog wearing?",
# "images": [
# "images/dog_a.png",
# "images/dog_b.png",
# "images/dog_c.png",
# "images/dog_d.png",
# ],
# },
# {"role": "Assistant", "content": ""}
# ]
# load images and prepare for inputs
pil_images = load_pil_images(conversation)
prepare_inputs = vl_chat_processor(