diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 18180ca..8a6b072 --- a/README.md +++ b/README.md @@ -45,11 +45,12 @@

- 📥 Model Download | - ⚡ Quick Start | - 📜 License | - 📖 Citation
+ 📥 Model Download | + ⚡ Quick Start | + 📜 License | + 📖 Citation
📄 Paper Link | + 🤗 Online Demo

@@ -61,11 +62,14 @@ Janus is a novel autoregressive framework that unifies multimodal understanding image +## 2. News -## 2. Model Download +**2024.10.20**: (1) Fix a bug in [tokenizer_config.json](https://huggingface.co/deepseek-ai/Janus-1.3B/blob/main/tokenizer_config.json). The previous version caused classifier-free guidance to not function properly, resulting in relatively poor visual generation quality. (2) Release Gradio demo ([online demo](https://huggingface.co/spaces/deepseek-ai/Janus-1.3B) and [local](#gradio-demo)). + +## 3. Model Download We release Janus to the public to support a broader and more diverse range of research within both academic and commercial communities. -Please note that the use of this model is subject to the terms outlined in [License section](#4-license). Commercial usage is +Please note that the use of this model is subject to the terms outlined in [License section](#5-license). Commercial usage is permitted under these terms. ### Huggingface @@ -77,7 +81,7 @@ permitted under these terms. -## 3. Quick Start +## 4. Quick Start ### Installation @@ -87,6 +91,7 @@ On the basis of `Python >= 3.8` environment, install the necessary dependencies pip install -e . ``` + ### Simple Inference Example #### Multimodal Understanding @@ -243,11 +248,25 @@ generate( ) ``` -## 4. License +### Gradio Demo +We have deployed online demo in [Huggingface](https://huggingface.co/spaces/deepseek-ai/Janus-1.3B). + + +For the local gradio demo, you can run with the following command: + +``` +pip install -e .[gradio] + +python demo/app.py +``` + +Have Fun! + +## 5. License This code repository is licensed under [the MIT License](https://github.com/deepseek-ai/DeepSeek-LLM/blob/HEAD/LICENSE-CODE). The use of Janus models is subject to [DeepSeek Model License](https://github.com/deepseek-ai/DeepSeek-LLM/blob/HEAD/LICENSE-MODEL). -## 5. Citation +## 6. Citation ``` @misc{wu2024janus, @@ -261,6 +280,6 @@ This code repository is licensed under [the MIT License](https://github.com/deep } ``` -## 6. Contact +## 7. Contact If you have any questions, please raise an issue or contact us at [service@deepseek.com](mailto:service@deepseek.com). diff --git a/Janus_demo.ipynb b/demo/Janus_colab_demo.ipynb similarity index 100% rename from Janus_demo.ipynb rename to demo/Janus_colab_demo.ipynb diff --git a/demo/app.py b/demo/app.py new file mode 100644 index 0000000..7dbc59f --- /dev/null +++ b/demo/app.py @@ -0,0 +1,224 @@ +import gradio as gr +import torch +from transformers import AutoConfig, AutoModelForCausalLM +from janus.models import MultiModalityCausalLM, VLChatProcessor +from PIL import Image + +import numpy as np + + +# Load model and processor +model_path = "deepseek-ai/Janus-1.3B" +config = AutoConfig.from_pretrained(model_path) +language_config = config.language_config +language_config._attn_implementation = 'eager' +vl_gpt = AutoModelForCausalLM.from_pretrained(model_path, + language_config=language_config, + trust_remote_code=True) +vl_gpt = vl_gpt.to(torch.bfloat16).cuda() + +vl_chat_processor = VLChatProcessor.from_pretrained(model_path) +tokenizer = vl_chat_processor.tokenizer +cuda_device = 'cuda' if torch.cuda.is_available() else 'cpu' +# Multimodal Understanding function +@torch.inference_mode() +# Multimodal Understanding function +def multimodal_understanding(image, question, seed, top_p, temperature): + # Clear CUDA cache before generating + torch.cuda.empty_cache() + + # set seed + torch.manual_seed(seed) + np.random.seed(seed) + torch.cuda.manual_seed(seed) + + conversation = [ + { + "role": "User", + "content": f"\n{question}", + "images": [image], + }, + {"role": "Assistant", "content": ""}, + ] + + pil_images = [Image.fromarray(image)] + prepare_inputs = vl_chat_processor( + conversations=conversation, images=pil_images, force_batchify=True + ).to(cuda_device, dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float16) + + + inputs_embeds = vl_gpt.prepare_inputs_embeds(**prepare_inputs) + + outputs = vl_gpt.language_model.generate( + inputs_embeds=inputs_embeds, + attention_mask=prepare_inputs.attention_mask, + pad_token_id=tokenizer.eos_token_id, + bos_token_id=tokenizer.bos_token_id, + eos_token_id=tokenizer.eos_token_id, + max_new_tokens=512, + do_sample=False if temperature == 0 else True, + use_cache=True, + temperature=temperature, + top_p=top_p, + ) + + answer = tokenizer.decode(outputs[0].cpu().tolist(), skip_special_tokens=True) + return answer + + +def generate(input_ids, + width, + height, + temperature: float = 1, + parallel_size: int = 5, + cfg_weight: float = 5, + image_token_num_per_image: int = 576, + patch_size: int = 16): + # Clear CUDA cache before generating + torch.cuda.empty_cache() + + tokens = torch.zeros((parallel_size * 2, len(input_ids)), dtype=torch.int).to(cuda_device) + for i in range(parallel_size * 2): + tokens[i, :] = input_ids + if i % 2 != 0: + tokens[i, 1:-1] = vl_chat_processor.pad_id + inputs_embeds = vl_gpt.language_model.get_input_embeddings()(tokens) + generated_tokens = torch.zeros((parallel_size, image_token_num_per_image), dtype=torch.int).to(cuda_device) + + pkv = None + for i in range(image_token_num_per_image): + outputs = vl_gpt.language_model.model(inputs_embeds=inputs_embeds, + use_cache=True, + past_key_values=pkv) + pkv = outputs.past_key_values + hidden_states = outputs.last_hidden_state + logits = vl_gpt.gen_head(hidden_states[:, -1, :]) + logit_cond = logits[0::2, :] + logit_uncond = logits[1::2, :] + logits = logit_uncond + cfg_weight * (logit_cond - logit_uncond) + probs = torch.softmax(logits / temperature, dim=-1) + next_token = torch.multinomial(probs, num_samples=1) + generated_tokens[:, i] = next_token.squeeze(dim=-1) + next_token = torch.cat([next_token.unsqueeze(dim=1), next_token.unsqueeze(dim=1)], dim=1).view(-1) + img_embeds = vl_gpt.prepare_gen_img_embeds(next_token) + inputs_embeds = img_embeds.unsqueeze(dim=1) + patches = vl_gpt.gen_vision_model.decode_code(generated_tokens.to(dtype=torch.int), + shape=[parallel_size, 8, width // patch_size, height // patch_size]) + + return generated_tokens.to(dtype=torch.int), patches + +def unpack(dec, width, height, parallel_size=5): + dec = dec.to(torch.float32).cpu().numpy().transpose(0, 2, 3, 1) + dec = np.clip((dec + 1) / 2 * 255, 0, 255) + + visual_img = np.zeros((parallel_size, width, height, 3), dtype=np.uint8) + visual_img[:, :, :] = dec + + return visual_img + + + +@torch.inference_mode() +def generate_image(prompt, + seed=None, + guidance=5): + # Clear CUDA cache and avoid tracking gradients + torch.cuda.empty_cache() + # Set the seed for reproducible results + if seed is not None: + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + np.random.seed(seed) + width = 384 + height = 384 + parallel_size = 5 + + with torch.no_grad(): + messages = [{'role': 'User', 'content': prompt}, + {'role': 'Assistant', 'content': ''}] + text = vl_chat_processor.apply_sft_template_for_multi_turn_prompts(conversations=messages, + sft_format=vl_chat_processor.sft_format, + system_prompt='') + text = text + vl_chat_processor.image_start_tag + input_ids = torch.LongTensor(tokenizer.encode(text)) + output, patches = generate(input_ids, + width // 16 * 16, + height // 16 * 16, + cfg_weight=guidance, + parallel_size=parallel_size) + images = unpack(patches, + width // 16 * 16, + height // 16 * 16) + + return [Image.fromarray(images[i]).resize((1024, 1024), Image.LANCZOS) for i in range(parallel_size)] + + + +# Gradio interface +with gr.Blocks() as demo: + gr.Markdown(value="# Multimodal Understanding") + # with gr.Row(): + with gr.Row(): + image_input = gr.Image() + with gr.Column(): + question_input = gr.Textbox(label="Question") + und_seed_input = gr.Number(label="Seed", precision=0, value=42) + top_p = gr.Slider(minimum=0, maximum=1, value=0.95, step=0.05, label="top_p") + temperature = gr.Slider(minimum=0, maximum=1, value=0.1, step=0.05, label="temperature") + + understanding_button = gr.Button("Chat") + understanding_output = gr.Textbox(label="Response") + + examples_inpainting = gr.Examples( + label="Multimodal Understanding examples", + examples=[ + [ + "explain this meme", + "images/doge.png", + ], + [ + "Convert the formula into latex code.", + "images/equation.png", + ], + ], + inputs=[question_input, image_input], + ) + + + gr.Markdown(value="# Text-to-Image Generation") + + + + with gr.Row(): + cfg_weight_input = gr.Slider(minimum=1, maximum=10, value=5, step=0.5, label="CFG Weight") + + prompt_input = gr.Textbox(label="Prompt") + seed_input = gr.Number(label="Seed (Optional)", precision=0, value=12345) + + generation_button = gr.Button("Generate Images") + + image_output = gr.Gallery(label="Generated Images", columns=2, rows=2, height=300) + + examples_t2i = gr.Examples( + label="Text to image generation examples. (Tips for designing prompts: Adding description like 'digital art' at the end of the prompt or writing the prompt in more detail can help produce better images!)", + examples=[ + "Master shifu racoon wearing drip attire as a street gangster.", + "A cute and adorable baby fox with big brown eyes, autumn leaves in the background enchanting,immortal,fluffy, shiny mane,Petals,fairyism,unreal engine 5 and Octane Render,highly detailed, photorealistic, cinematic, natural colors.", + "The image features an intricately designed eye set against a circular backdrop adorned with ornate swirl patterns that evoke both realism and surrealism. At the center of attention is a strikingly vivid blue iris surrounded by delicate veins radiating outward from the pupil to create depth and intensity. The eyelashes are long and dark, casting subtle shadows on the skin around them which appears smooth yet slightly textured as if aged or weathered over time.\n\nAbove the eye, there's a stone-like structure resembling part of classical architecture, adding layers of mystery and timeless elegance to the composition. This architectural element contrasts sharply but harmoniously with the organic curves surrounding it. Below the eye lies another decorative motif reminiscent of baroque artistry, further enhancing the overall sense of eternity encapsulated within each meticulously crafted detail. \n\nOverall, the atmosphere exudes a mysterious aura intertwined seamlessly with elements suggesting timelessness, achieved through the juxtaposition of realistic textures and surreal artistic flourishes. Each component\u2014from the intricate designs framing the eye to the ancient-looking stone piece above\u2014contributes uniquely towards creating a visually captivating tableau imbued with enigmatic allure.", + ], + inputs=prompt_input, + ) + + understanding_button.click( + multimodal_understanding, + inputs=[image_input, question_input, und_seed_input, top_p, temperature], + outputs=understanding_output + ) + + generation_button.click( + fn=generate_image, + inputs=[prompt_input, seed_input, cfg_weight_input], + outputs=image_output + ) + +demo.launch(share=True) \ No newline at end of file diff --git a/generation_inference.py b/generation_inference.py index 39ce59c..31cc61e 100644 --- a/generation_inference.py +++ b/generation_inference.py @@ -38,7 +38,7 @@ vl_gpt = vl_gpt.to(torch.bfloat16).cuda().eval() conversation = [ { "role": "User", - "content": "A stunning princess from kabul in red, white traditional clothing, blue eyes, brown hair", + "content": "A close-up high-contrast photo of Sydney Opera House sitting next to Eiffel tower, under a blue night sky of roiling energy, exploding yellow stars, and radiating swirls of blue.", }, {"role": "Assistant", "content": ""}, ] diff --git a/images/doge.png b/images/doge.png new file mode 100644 index 0000000..dd27c74 Binary files /dev/null and b/images/doge.png differ