SDXL_InstantID_Img2Img_Face_to_All.ipynb

Kernel: Python 3

# @title Install requirements
!git clone https://github.com/InstantID/InstantID
%cd InstantID
!pip install -r gradio_demo/requirements.txt
!pip install timm==0.6.7
!pip install diffusers==0.27.2

# @title Download required models

!python gradio_demo/download_models.py -y

# @title Set up the pipeline

import diffusers
from diffusers.utils import load_image
from diffusers.models import ControlNetModel
from diffusers import AutoencoderKL, DPMSolverMultistepScheduler
import cv2
import torch
import numpy as np
from PIL import Image

from insightface.app import FaceAnalysis
from pipeline_stable_diffusion_xl_instantid_img2img import StableDiffusionXLInstantIDImg2ImgPipeline, draw_kps
from controlnet_aux import ZoeDetector

# prepare 'antelopev2' under ./models
app = FaceAnalysis(name='antelopev2', root='./', providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
app.prepare(ctx_id=0, det_size=(640, 640))

# prepare models under ./checkpoints
face_adapter = f'./checkpoints/ip-adapter.bin'
controlnet_path = f'./checkpoints/ControlNetModel'

# load IdentityNet
identitynet = ControlNetModel.from_pretrained(controlnet_path, torch_dtype=torch.float16)
zoedepthnet = ControlNetModel.from_pretrained("diffusers/controlnet-zoe-depth-sdxl-1.0",torch_dtype=torch.float16)
vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16)
pipe = StableDiffusionXLInstantIDImg2ImgPipeline.from_pretrained("rubbrband/albedobaseXL_v21",
                                                                 vae=vae,
                                                                 controlnet=[identitynet, zoedepthnet],
                                                                 torch_dtype=torch.float16)
pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config, use_karras_sigmas=True)
#pipe.to("cuda")
pipe.load_ip_adapter_instantid(face_adapter)
pipe.set_ip_adapter_scale(0.8)

# @title Enter an image of your face
# @markdown Upload an image of a face to the Files tab
# load adapter
image_path = "your_face.png" # @param {type:"string"}

face_image = load_image(f"../{image_path}")

#prepare face emb and kps
face_info = app.get(cv2.cvtColor(np.array(face_image), cv2.COLOR_RGB2BGR))
face_info = sorted(face_info, key=lambda x:(x['bbox'][2]-x['bbox'][0])*x['bbox'][3]-x['bbox'][1])[-1] # only use the maximum face
face_emb = face_info['embedding']
face_kps = draw_kps(face_image, face_info['kps'])

#prepare face zoe
zoe = ZoeDetector.from_pretrained("lllyasviel/Annotators")
image_zoe = zoe(face_image)

width, height = face_kps.size
images = [face_kps, image_zoe.resize((height, width))]

# @title Load your LoRA!
# @markdown You can load a LoRA directly from Hugging Face by browsing [here](https://huggingface.co/models?library=diffusers&other=lora), or download a LoRA from CivitAI/Tensor.Art and place it on the colab folder.
pipe.load_lora_weights(
    "Norod78/sdxl-chalkboarddrawing-lora",
    weight_name="SDXL_ChalkBoardDrawing_LoRA_r8.safetensors"
)
pipe.enable_sequential_cpu_offload()

# @title Generate!

prompt = "A colorful ChalkBoardDrawing of a man" # @param {type:"string"}
negative_prompt = "blurry, ultra-realism, detailed" # @param {type:"string"}
# @markdown The higher the `denoising_strength`, more similar to the original image.
denoising_strength = 0.85 # @param {type:"slider", min:0, max:1, step:0.01}
guidance_scale = 7 # @param {type:"number"}
face_control_strength = 0.8 # @param {type:"slider", min:0, max:1, step: 0.01}
depth_control_strength = 0.8 # @param {type:"slider", min:0, max:1, step: 0.01}

image = pipe(
    prompt,
    negative_prompt=negative_prompt,
    width=1024,
    height=1024,
    image_embeds=face_emb,
    image=face_image,
    strength=denoising_strength,
    control_image=images,
    num_inference_steps=20,
    guidance_scale = guidance_scale,
    controlnet_conditioning_scale=[face_control_strength, depth_control_strength],
).images[0]
image