section3_tf.ipynb

Kernel: Unknown Kernel

Публикация предобученных моделей в общий доступ (TensorFlow)

Install the Transformers, Datasets, and Evaluate libraries to run this notebook.

!pip install datasets evaluate transformers[sentencepiece]
!apt install git-lfs

You will need to setup git, adapt your email and name in the following cell.

!git config --global user.email "[email protected]"
!git config --global user.name "Your Name"

You will also need to be logged in to the Hugging Face Hub. Execute the following and enter your credentials.

from huggingface_hub import notebook_login

notebook_login()

from huggingface_hub import notebook_login

notebook_login()

from transformers import PushToHubCallback

callback = PushToHubCallback(
    "bert-finetuned-mrpc", save_strategy="epoch", tokenizer=tokenizer
)

from transformers import TFAutoModelForMaskedLM, AutoTokenizer

checkpoint = "camembert-base"

model = TFAutoModelForMaskedLM.from_pretrained(checkpoint)
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

model.push_to_hub("dummy-model")

tokenizer.push_to_hub("dummy-model")

tokenizer.push_to_hub("dummy-model", organization="huggingface")

tokenizer.push_to_hub("dummy-model", organization="huggingface", use_auth_token="<TOKEN>")

from huggingface_hub import (
    # Пользовательские настройки
    login,
    logout,
    whoami,

    # Создание и управление репозиториями
    create_repo,
    delete_repo,
    update_repo_visibility,

    # И несколько способов для получения или изменения информации о содержимом
    list_models,
    list_datasets,
    list_metrics,
    list_repo_files,
    upload_file,
    delete_file,
)

from huggingface_hub import create_repo

create_repo("dummy-model")

from huggingface_hub import create_repo

create_repo("dummy-model", organization="huggingface")

from huggingface_hub import upload_file

upload_file(
    "<path_to_file>/config.json",
    path_in_repo="config.json",
    repo_id="<namespace>/dummy-model",
)

from huggingface_hub import Repository

repo = Repository("<path_to_dummy_folder>", clone_from="<namespace>/dummy-model")

repo.git_pull()
repo.git_add()
repo.git_commit()
repo.git_push()
repo.git_tag()

repo.git_pull()

model.save_pretrained("<path_to_dummy_folder>")
tokenizer.save_pretrained("<path_to_dummy_folder>")

repo.git_add()
repo.git_commit("Add model and tokenizer files")
repo.git_push()

from transformers import TFAutoModelForMaskedLM, AutoTokenizer

checkpoint = "camembert-base"

model = TFAutoModelForMaskedLM.from_pretrained(checkpoint)
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

# Do whatever with the model, train it, fine-tune it...

model.save_pretrained("<path_to_dummy_folder>")
tokenizer.save_pretrained("<path_to_dummy_folder>")