import{s as te,n as ae,o as le}from"../chunks/scheduler.8c3d61f6.js";import{S as se,i as ie,g as p,s,r as b,A as ne,h as c,f as a,c as i,j as K,u as U,x as _,k as Q,y as me,a as l,v as f,d as W,t as Z,w as j}from"../chunks/index.da70eac4.js";import{C as H}from"../chunks/CodeBlock.00a903b3.js";import{D as oe}from"../chunks/DocNotebookDropdown.02900f6b.js";import{H as ee,E as pe}from"../chunks/EditOnGithub.1e64e623.js";function ce(q){let n,B,G,R,r,x,g,N,d,L=`Multimodal instructions mean you can input arbitrarily interleaved text and image inputs as conditions to guide image generation.
You can input multiple images and use prompts to describe the desired output.
This approach is more flexible than using only text or images.`,V,u,v,J,A="Take <code>OmniGenPipeline</code> as an example: the input can be a text-image sequence to create new images, he input can be a text-image sequence, with images inserted into the text prompt via special placeholder <code>&lt;img&gt;&lt;|image_i|&gt;&lt;/img&gt;</code>.",S,T,X,m,D='<div class="flex-1"><img class="rounded-xl" src="https://raw.githubusercontent.com/VectorSpaceLab/OmniGen/main/imgs/docs_img/3.png"/> <figcaption class="mt-2 text-center text-sm text-gray-500">input_image_1</figcaption></div> <div class="flex-1"><img class="rounded-xl" src="https://raw.githubusercontent.com/VectorSpaceLab/OmniGen/main/imgs/docs_img/4.png"/> <figcaption class="mt-2 text-center text-sm text-gray-500">input_image_2</figcaption></div> <div class="flex-1"><img class="rounded-xl" src="https://raw.githubusercontent.com/VectorSpaceLab/OmniGen/main/imgs/docs_img/id2.png"/> <figcaption class="mt-2 text-center text-sm text-gray-500">generated image</figcaption></div>',k,h,C,o,P='<div class="flex-1"><img class="rounded-xl" src="https://raw.githubusercontent.com/VectorSpaceLab/OmniGen/main/imgs/docs_img/emma.jpeg"/> <figcaption class="mt-2 text-center text-sm text-gray-500">person image</figcaption></div> <div class="flex-1"><img class="rounded-xl" src="https://raw.githubusercontent.com/VectorSpaceLab/OmniGen/main/imgs/docs_img/dress.jpg"/> <figcaption class="mt-2 text-center text-sm text-gray-500">clothe image</figcaption></div> <div class="flex-1"><img class="rounded-xl" src="https://raw.githubusercontent.com/VectorSpaceLab/OmniGen/main/imgs/docs_img/tryon.png"/> <figcaption class="mt-2 text-center text-sm text-gray-500">generated image</figcaption></div>',Y,y,O='The output image is a <a href="https://pillow.readthedocs.io/en/stable/reference/Image.html?highlight=image#the-image-class" rel="nofollow"><code>PIL.Image</code></a> object that can be saved:',$,M,E,w,z,I,F;return r=new ee({props:{title:"Multi-modal instruction to image",local:"multi-modal-instruction-to-image",headingTag:"h1"}}),g=new oe({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers_doc/en/multimodal2img.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers_doc/en/pytorch/multimodal2img.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers_doc/en/tensorflow/multimodal2img.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/diffusers_doc/en/multimodal2img.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/diffusers_doc/en/pytorch/multimodal2img.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/diffusers_doc/en/tensorflow/multimodal2img.ipynb"}]}}),u=new ee({props:{title:"Examples",local:"examples",headingTag:"h2"}}),T=new H({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwT21uaUdlblBpcGVsaW5lJTBBZnJvbSUyMGRpZmZ1c2Vycy51dGlscyUyMGltcG9ydCUyMGxvYWRfaW1hZ2UlMjAlMEElMEFwaXBlJTIwJTNEJTIwT21uaUdlblBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJTaGl0YW8lMkZPbW5pR2VuLXYxLWRpZmZ1c2VycyUyMiUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYlMEEpJTBBcGlwZS50byglMjJjdWRhJTIyKSUwQSUwQXByb21wdCUzRCUyMkElMjBtYW4lMjBhbmQlMjBhJTIwd29tYW4lMjBhcmUlMjBzaXR0aW5nJTIwYXQlMjBhJTIwY2xhc3Nyb29tJTIwZGVzay4lMjBUaGUlMjBtYW4lMjBpcyUyMHRoZSUyMG1hbiUyMHdpdGglMjB5ZWxsb3clMjBoYWlyJTIwaW4lMjAlM0NpbWclM0UlM0MlN0NpbWFnZV8xJTdDJTNFJTNDJTJGaW1nJTNFLiUyMFRoZSUyMHdvbWFuJTIwaXMlMjB0aGUlMjB3b21hbiUyMG9uJTIwdGhlJTIwbGVmdCUyMG9mJTIwJTNDaW1nJTNFJTNDJTdDaW1hZ2VfMiU3QyUzRSUzQyUyRmltZyUzRSUyMiUwQWlucHV0X2ltYWdlXzElMjAlM0QlMjBsb2FkX2ltYWdlKCUyMmh0dHBzJTNBJTJGJTJGcmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSUyRlZlY3RvclNwYWNlTGFiJTJGT21uaUdlbiUyRm1haW4lMkZpbWdzJTJGZG9jc19pbWclMkYzLmpwZyUyMiklMEFpbnB1dF9pbWFnZV8yJTIwJTNEJTIwbG9hZF9pbWFnZSglMjJodHRwcyUzQSUyRiUyRnJhdy5naXRodWJ1c2VyY29udGVudC5jb20lMkZWZWN0b3JTcGFjZUxhYiUyRk9tbmlHZW4lMkZtYWluJTJGaW1ncyUyRmRvY3NfaW1nJTJGNC5qcGclMjIpJTBBaW5wdXRfaW1hZ2VzJTNEJTVCaW5wdXRfaW1hZ2VfMSUyQyUyMGlucHV0X2ltYWdlXzIlNUQlMEFpbWFnZSUyMCUzRCUyMHBpcGUoJTBBJTIwJTIwJTIwJTIwcHJvbXB0JTNEcHJvbXB0JTJDJTIwJTBBJTIwJTIwJTIwJTIwaW5wdXRfaW1hZ2VzJTNEaW5wdXRfaW1hZ2VzJTJDJTIwJTBBJTIwJTIwJTIwJTIwaGVpZ2h0JTNEMTAyNCUyQyUwQSUyMCUyMCUyMCUyMHdpZHRoJTNEMTAyNCUyQyUwQSUyMCUyMCUyMCUyMGd1aWRhbmNlX3NjYWxlJTNEMi41JTJDJTIwJTBBJTIwJTIwJTIwJTIwaW1nX2d1aWRhbmNlX3NjYWxlJTNEMS42JTJDJTBBJTIwJTIwJTIwJTIwZ2VuZXJhdG9yJTNEdG9yY2guR2VuZXJhdG9yKGRldmljZSUzRCUyMmNwdSUyMikubWFudWFsX3NlZWQoNjY2KSkuaW1hZ2VzJTVCMCU1RCUwQWltYWdl",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> OmniGenPipeline
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image 

pipe = OmniGenPipeline.from_pretrained(
    <span class="hljs-string">&quot;Shitao/OmniGen-v1-diffusers&quot;</span>,
    torch_dtype=torch.bfloat16
)
pipe.to(<span class="hljs-string">&quot;cuda&quot;</span>)

prompt=<span class="hljs-string">&quot;A man and a woman are sitting at a classroom desk. The man is the man with yellow hair in &lt;img&gt;&lt;|image_1|&gt;&lt;/img&gt;. The woman is the woman on the left of &lt;img&gt;&lt;|image_2|&gt;&lt;/img&gt;&quot;</span>
input_image_1 = load_image(<span class="hljs-string">&quot;https://raw.githubusercontent.com/VectorSpaceLab/OmniGen/main/imgs/docs_img/3.jpg&quot;</span>)
input_image_2 = load_image(<span class="hljs-string">&quot;https://raw.githubusercontent.com/VectorSpaceLab/OmniGen/main/imgs/docs_img/4.jpg&quot;</span>)
input_images=[input_image_1, input_image_2]
image = pipe(
    prompt=prompt, 
    input_images=input_images, 
    height=<span class="hljs-number">1024</span>,
    width=<span class="hljs-number">1024</span>,
    guidance_scale=<span class="hljs-number">2.5</span>, 
    img_guidance_scale=<span class="hljs-number">1.6</span>,
    generator=torch.Generator(device=<span class="hljs-string">&quot;cpu&quot;</span>).manual_seed(<span class="hljs-number">666</span>)).images[<span class="hljs-number">0</span>]
image`,wrap:!1}}),h=new H({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwT21uaUdlblBpcGVsaW5lJTBBZnJvbSUyMGRpZmZ1c2Vycy51dGlscyUyMGltcG9ydCUyMGxvYWRfaW1hZ2UlMjAlMEElMEFwaXBlJTIwJTNEJTIwT21uaUdlblBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJTaGl0YW8lMkZPbW5pR2VuLXYxLWRpZmZ1c2VycyUyMiUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYlMEEpJTBBcGlwZS50byglMjJjdWRhJTIyKSUwQSUwQSUwQXByb21wdCUzRCUyMkElMjB3b21hbiUyMGlzJTIwd2Fsa2luZyUyMGRvd24lMjB0aGUlMjBzdHJlZXQlMkMlMjB3ZWFyaW5nJTIwYSUyMHdoaXRlJTIwbG9uZy1zbGVldmUlMjBibG91c2UlMjB3aXRoJTIwbGFjZSUyMGRldGFpbHMlMjBvbiUyMHRoZSUyMHNsZWV2ZXMlMkMlMjBwYWlyZWQlMjB3aXRoJTIwYSUyMGJsdWUlMjBwbGVhdGVkJTIwc2tpcnQuJTIwVGhlJTIwd29tYW4lMjBpcyUyMCUzQ2ltZyUzRSUzQyU3Q2ltYWdlXzElN0MlM0UlM0MlMkZpbWclM0UuJTIwVGhlJTIwbG9uZy1zbGVldmUlMjBibG91c2UlMjBhbmQlMjBhJTIwcGxlYXRlZCUyMHNraXJ0JTIwYXJlJTIwJTNDaW1nJTNFJTNDJTdDaW1hZ2VfMiU3QyUzRSUzQyUyRmltZyUzRS4lMjIlMEFpbnB1dF9pbWFnZV8xJTIwJTNEJTIwbG9hZF9pbWFnZSglMjIlMkZzaGFyZSUyRmp1bmppZSUyRmNvZGUlMkZWSVNUQTIlMkZwcm9kdWNlX2RhdGElMkZsYWlvbl9uZXQlMkZkaWZmZ3B0JTJGT21uaUdlbiUyRmRvY3NfaW1nJTJGZW1tYS5qcGVnJTIyKSUwQWlucHV0X2ltYWdlXzIlMjAlM0QlMjBsb2FkX2ltYWdlKCUyMiUyRnNoYXJlJTJGanVuamllJTJGY29kZSUyRlZJU1RBMiUyRnByb2R1Y2VfZGF0YSUyRmxhaW9uX25ldCUyRmRpZmZncHQlMkZPbW5pR2VuJTJGZG9jc19pbWclMkZkcmVzcy5qcGclMjIpJTBBaW5wdXRfaW1hZ2VzJTNEJTVCaW5wdXRfaW1hZ2VfMSUyQyUyMGlucHV0X2ltYWdlXzIlNUQlMEFpbWFnZSUyMCUzRCUyMHBpcGUoJTBBJTIwJTIwJTIwJTIwcHJvbXB0JTNEcHJvbXB0JTJDJTIwJTBBJTIwJTIwJTIwJTIwaW5wdXRfaW1hZ2VzJTNEaW5wdXRfaW1hZ2VzJTJDJTIwJTBBJTIwJTIwJTIwJTIwaGVpZ2h0JTNEMTAyNCUyQyUwQSUyMCUyMCUyMCUyMHdpZHRoJTNEMTAyNCUyQyUwQSUyMCUyMCUyMCUyMGd1aWRhbmNlX3NjYWxlJTNEMi41JTJDJTIwJTBBJTIwJTIwJTIwJTIwaW1nX2d1aWRhbmNlX3NjYWxlJTNEMS42JTJDJTBBJTIwJTIwJTIwJTIwZ2VuZXJhdG9yJTNEdG9yY2guR2VuZXJhdG9yKGRldmljZSUzRCUyMmNwdSUyMikubWFudWFsX3NlZWQoNjY2KSkuaW1hZ2VzJTVCMCU1RA==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> OmniGenPipeline
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image 

pipe = OmniGenPipeline.from_pretrained(
    <span class="hljs-string">&quot;Shitao/OmniGen-v1-diffusers&quot;</span>,
    torch_dtype=torch.bfloat16
)
pipe.to(<span class="hljs-string">&quot;cuda&quot;</span>)


prompt=<span class="hljs-string">&quot;A woman is walking down the street, wearing a white long-sleeve blouse with lace details on the sleeves, paired with a blue pleated skirt. The woman is &lt;img&gt;&lt;|image_1|&gt;&lt;/img&gt;. The long-sleeve blouse and a pleated skirt are &lt;img&gt;&lt;|image_2|&gt;&lt;/img&gt;.&quot;</span>
input_image_1 = load_image(<span class="hljs-string">&quot;/share/junjie/code/VISTA2/produce_data/laion_net/diffgpt/OmniGen/docs_img/emma.jpeg&quot;</span>)
input_image_2 = load_image(<span class="hljs-string">&quot;/share/junjie/code/VISTA2/produce_data/laion_net/diffgpt/OmniGen/docs_img/dress.jpg&quot;</span>)
input_images=[input_image_1, input_image_2]
image = pipe(
    prompt=prompt, 
    input_images=input_images, 
    height=<span class="hljs-number">1024</span>,
    width=<span class="hljs-number">1024</span>,
    guidance_scale=<span class="hljs-number">2.5</span>, 
    img_guidance_scale=<span class="hljs-number">1.6</span>,
    generator=torch.Generator(device=<span class="hljs-string">&quot;cpu&quot;</span>).manual_seed(<span class="hljs-number">666</span>)).images[<span class="hljs-number">0</span>]`,wrap:!1}}),M=new H({props:{code:"aW1hZ2Uuc2F2ZSglMjJnZW5lcmF0ZWRfaW1hZ2UucG5nJTIyKQ==",highlighted:'image.save(<span class="hljs-string">&quot;generated_image.png&quot;</span>)',wrap:!1}}),w=new pe({props:{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/using-diffusers/multimodal2img.md"}}),{c(){n=p("meta"),B=s(),G=p("p"),R=s(),b(r.$$.fragment),x=s(),b(g.$$.fragment),N=s(),d=p("p"),d.textContent=L,V=s(),b(u.$$.fragment),v=s(),J=p("p"),J.innerHTML=A,S=s(),b(T.$$.fragment),X=s(),m=p("div"),m.innerHTML=D,k=s(),b(h.$$.fragment),C=s(),o=p("div"),o.innerHTML=P,Y=s(),y=p("p"),y.innerHTML=O,$=s(),b(M.$$.fragment),E=s(),b(w.$$.fragment),z=s(),I=p("p"),this.h()},l(e){const t=ne("svelte-u9bgzb",document.head);n=c(t,"META",{name:!0,content:!0}),t.forEach(a),B=i(e),G=c(e,"P",{}),K(G).forEach(a),R=i(e),U(r.$$.fragment,e),x=i(e),U(g.$$.fragment,e),N=i(e),d=c(e,"P",{"data-svelte-h":!0}),_(d)!=="svelte-1oh3x8a"&&(d.textContent=L),V=i(e),U(u.$$.fragment,e),v=i(e),J=c(e,"P",{"data-svelte-h":!0}),_(J)!=="svelte-tr2se5"&&(J.innerHTML=A),S=i(e),U(T.$$.fragment,e),X=i(e),m=c(e,"DIV",{class:!0,"data-svelte-h":!0}),_(m)!=="svelte-q7ysaf"&&(m.innerHTML=D),k=i(e),U(h.$$.fragment,e),C=i(e),o=c(e,"DIV",{class:!0,"data-svelte-h":!0}),_(o)!=="svelte-164h1l6"&&(o.innerHTML=P),Y=i(e),y=c(e,"P",{"data-svelte-h":!0}),_(y)!=="svelte-1ut9jwr"&&(y.innerHTML=O),$=i(e),U(M.$$.fragment,e),E=i(e),U(w.$$.fragment,e),z=i(e),I=c(e,"P",{}),K(I).forEach(a),this.h()},h(){Q(n,"name","hf:doc:metadata"),Q(n,"content",re),Q(m,"class","flex flex-row gap-4"),Q(o,"class","flex flex-row gap-4")},m(e,t){me(document.head,n),l(e,B,t),l(e,G,t),l(e,R,t),f(r,e,t),l(e,x,t),f(g,e,t),l(e,N,t),l(e,d,t),l(e,V,t),f(u,e,t),l(e,v,t),l(e,J,t),l(e,S,t),f(T,e,t),l(e,X,t),l(e,m,t),l(e,k,t),f(h,e,t),l(e,C,t),l(e,o,t),l(e,Y,t),l(e,y,t),l(e,$,t),f(M,e,t),l(e,E,t),f(w,e,t),l(e,z,t),l(e,I,t),F=!0},p:ae,i(e){F||(W(r.$$.fragment,e),W(g.$$.fragment,e),W(u.$$.fragment,e),W(T.$$.fragment,e),W(h.$$.fragment,e),W(M.$$.fragment,e),W(w.$$.fragment,e),F=!0)},o(e){Z(r.$$.fragment,e),Z(g.$$.fragment,e),Z(u.$$.fragment,e),Z(T.$$.fragment,e),Z(h.$$.fragment,e),Z(M.$$.fragment,e),Z(w.$$.fragment,e),F=!1},d(e){e&&(a(B),a(G),a(R),a(x),a(N),a(d),a(V),a(v),a(J),a(S),a(X),a(m),a(k),a(C),a(o),a(Y),a(y),a($),a(E),a(z),a(I)),a(n),j(r,e),j(g,e),j(u,e),j(T,e),j(h,e),j(M,e),j(w,e)}}}const re='{"title":"Multi-modal instruction to image","local":"multi-modal-instruction-to-image","sections":[{"title":"Examples","local":"examples","sections":[],"depth":2}],"depth":1}';function ge(q){return le(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ye extends se{constructor(n){super(),ie(this,n,ge,ce,te,{})}}export{ye as component};
