Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
automatic1111
GitHub Repository: automatic1111/stable-diffusion-webui
 Path: blob/master/modules/sd_hijack_open_clip.py
4923 views
1
import open_clip.tokenizer

2
import torch

3


4
from modules import sd_hijack_clip, devices

5
from modules.shared import opts

6


7
tokenizer = open_clip.tokenizer._tokenizer

8


9


10
class FrozenOpenCLIPEmbedderWithCustomWords(sd_hijack_clip.FrozenCLIPEmbedderWithCustomWordsBase):

11
    def __init__(self, wrapped, hijack):

12
        super().__init__(wrapped, hijack)

13


14
        self.comma_token = [v for k, v in tokenizer.encoder.items() if k == ',</w>'][0]

15
        self.id_start = tokenizer.encoder["<start_of_text>"]

16
        self.id_end = tokenizer.encoder["<end_of_text>"]

17
        self.id_pad = 0

18


19
    def tokenize(self, texts):

20
        assert not opts.use_old_emphasis_implementation, 'Old emphasis implementation not supported for Open Clip'

21


22
        tokenized = [tokenizer.encode(text) for text in texts]

23


24
        return tokenized

25


26
    def encode_with_transformers(self, tokens):

27
        # set self.wrapped.layer_idx here according to opts.CLIP_stop_at_last_layers

28
        z = self.wrapped.encode_with_transformer(tokens)

29


30
        return z

31


32
    def encode_embedding_init_text(self, init_text, nvpt):

33
        ids = tokenizer.encode(init_text)

34
        ids = torch.asarray([ids], device=devices.device, dtype=torch.int)

35
        embedded = self.wrapped.model.token_embedding.wrapped(ids).squeeze(0)

36


37
        return embedded

38


39


40
class FrozenOpenCLIPEmbedder2WithCustomWords(sd_hijack_clip.FrozenCLIPEmbedderWithCustomWordsBase):

41
    def __init__(self, wrapped, hijack):

42
        super().__init__(wrapped, hijack)

43


44
        self.comma_token = [v for k, v in tokenizer.encoder.items() if k == ',</w>'][0]

45
        self.id_start = tokenizer.encoder["<start_of_text>"]

46
        self.id_end = tokenizer.encoder["<end_of_text>"]

47
        self.id_pad = 0

48


49
    def tokenize(self, texts):

50
        assert not opts.use_old_emphasis_implementation, 'Old emphasis implementation not supported for Open Clip'

51


52
        tokenized = [tokenizer.encode(text) for text in texts]

53


54
        return tokenized

55


56
    def encode_with_transformers(self, tokens):

57
        d = self.wrapped.encode_with_transformer(tokens)

58
        z = d[self.wrapped.layer]

59


60
        pooled = d.get("pooled")

61
        if pooled is not None:

62
            z.pooled = pooled

63


64
        return z

65


66
    def encode_embedding_init_text(self, init_text, nvpt):

67
        ids = tokenizer.encode(init_text)

68
        ids = torch.asarray([ids], device=devices.device, dtype=torch.int)

69
        embedded = self.wrapped.model.token_embedding.wrapped(ids.to(self.wrapped.model.token_embedding.wrapped.weight.device)).squeeze(0)

70


71
        return embedded

72


73