section6_pt.ipynb

Kernel: Unknown Kernel

Colocando tudo junto (PyTorch)

Install the Transformers, Datasets, and Evaluate libraries to run this notebook.

!pip install datasets evaluate transformers[sentencepiece]

from transformers import AutoTokenizer

checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

sequence = "I've been waiting for a HuggingFace course my whole life."

model_inputs = tokenizer(sequence)

sequence = "I've been waiting for a HuggingFace course my whole life."

model_inputs = tokenizer(sequence)

sequences = ["I've been waiting for a HuggingFace course my whole life.", "So have I!"]

model_inputs = tokenizer(sequences)

# Irá preencher as sequências até o comprimento máximo da sequência
model_inputs = tokenizer(sequences, padding="longest")

# Irá preencher as sequências até o comprimento máximo do modelo
# (512 para o modelo BERT ou DistilBERT)
model_inputs = tokenizer(sequences, padding="max_length")

# Irá preencher as sequências até o comprimento máximo especificado
model_inputs = tokenizer(sequences, padding="max_length", max_length=8)

sequences = ["I've been waiting for a HuggingFace course my whole life.", "So have I!"]

# Irá preencher as sequências até o comprimento máximo do modelo
# (512 para o modelo BERT ou DistilBERT)
model_inputs = tokenizer(sequences, truncation=True)

# Truncará as sequências que são mais longas do que o comprimento máximo especificado
model_inputs = tokenizer(sequences, max_length=8, truncation=True)

sequences = ["I've been waiting for a HuggingFace course my whole life.", "So have I!"]

# Retorna tensores PyTorch
model_inputs = tokenizer(sequences, padding=True, return_tensors="pt")

# Retorna tensores TensorFlow
model_inputs = tokenizer(sequences, padding=True, return_tensors="tf")

# Retorna NumPy arrays
model_inputs = tokenizer(sequences, padding=True, return_tensors="np")

sequence = "I've been waiting for a HuggingFace course my whole life."

model_inputs = tokenizer(sequence)
print(model_inputs["input_ids"])

tokens = tokenizer.tokenize(sequence)
ids = tokenizer.convert_tokens_to_ids(tokens)
print(ids)

[101, 1045, 1005, 2310, 2042, 3403, 2005, 1037, 17662, 12172, 2607, 2026, 2878, 2166, 1012, 102]
[1045, 1005, 2310, 2042, 3403, 2005, 1037, 17662, 12172, 2607, 2026, 2878, 2166, 1012]

print(tokenizer.decode(model_inputs["input_ids"]))
print(tokenizer.decode(ids))

"[CLS] i've been waiting for a huggingface course my whole life. [SEP]"
"i've been waiting for a huggingface course my whole life."

import torch
from transformers import AutoTokenizer, AutoModelForSequenceClassification

checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = ["I've been waiting for a HuggingFace course my whole life.", "So have I!"]

tokens = tokenizer(sequences, padding=True, truncation=True, return_tensors="pt")
output = model(**tokens)