# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Tokenization classes for fast tokenizers (provided by HuggingFace's tokenizers library). For slow (python) tokenizers
see tokenization_utils.py
"""
import copy
import json
import os
from collections import defaultdict
from collections.abc import Iterable
from shutil import copyfile
from typing import Any
import tokenizers.pre_tokenizers as pre_tokenizers_fast
from huggingface_hub import is_offline_mode
from tokenizers import AddedToken, processors
from tokenizers import Encoding as EncodingFast
from tokenizers import Tokenizer as TokenizerFast
from tokenizers.decoders import Decoder as DecoderFast
from tokenizers.models import BPE, Unigram
from tokenizers.trainers import BpeTrainer, UnigramTrainer, WordLevelTrainer, WordPieceTrainer
from transformers.utils.hub import cached_file
from .integrations.ggml import convert_gguf_tokenizer
from .modeling_gguf_pytorch_utils import load_gguf_checkpoint
from .tokenization_utils_base import (
INIT_TOKENIZER_DOCSTRING,
BatchEncoding,
PreTokenizedInput,
PreTrainedTokenizerBase,
TextInput,
TruncationStrategy,
generate_merges,
)
from .utils import PaddingStrategy, add_end_docstrings, logging
logger = logging.get_logger(__name__)
# Fast tokenizers (provided by HuggingFace tokenizer's library) can be saved in a single file
TOKENIZER_FILE = "tokenizer.json"
SPECIAL_TOKENS_MAP_FILE = "special_tokens_map.json"
TOKENIZER_CONFIG_FILE = "tokenizer_config.json"
TIKTOKEN_VOCAB_FILE = "tokenizer.model"
# Slow tokenizers have an additional added tokens files
ADDED_TOKENS_FILE = "added_tokens.json"
INIT_TOKENIZER_DOCSTRING += """
tokenizer_object ([`tokenizers.Tokenizer`]):
A [`tokenizers.Tokenizer`] object from 🤗 tokenizers to instantiate from. See [Using tokenizers from 🤗
tokenizers](../fast_tokenizers) for more information.
tokenizer_file ([`str`]):
A path to a local JSON file representing a previously serialized [`tokenizers.Tokenizer`] object from 🤗
tokenizers.
"""
MODEL_TO_TRAINER_MAPPING = {
"BPE": BpeTrainer,
"Unigram": UnigramTrainer,
"WordLevel": WordLevelTrainer,
"WordPiece": WordPieceTrainer,
}
VOCAB_FILES_NAMES = {"tokenizer_file": TOKENIZER_FILE, "vocab_file": TIKTOKEN_VOCAB_FILE}
@add_end_docstrings(INIT_TOKENIZER_DOCSTRING)
class TokenizersBackend(PreTrainedTokenizerBase):
"""
Base class for all fast tokenizers (wrapping HuggingFace tokenizers library).
Inherits from [`~tokenization_utils_base.PreTrainedTokenizerBase`].
Handles all the shared methods for tokenization and special tokens, as well as methods for
downloading/caching/loading pretrained tokenizers, as well as adding tokens to the vocabulary.
This class also contains the added tokens in a unified way on top of all tokenizers so we don't have to handle the
specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...).
"""
vocab_files_names = VOCAB_FILES_NAMES
model = None
_tokenizer = None
@classmethod
def convert_to_native_format(cls, trust_remote_code=False, **kwargs):
"""s
Build a `tokenizers.Tokenizer` backend from the available serialization files (tokenizer.json, sentencepiece
models, tekken.json, vocab/merges).
"""
# Preserve kwargs for possible downstream use
local_kwargs = dict(kwargs)
fast_tokenizer_file = local_kwargs.pop("tokenizer_file", None)
if (
fast_tokenizer_file is not None
and os.path.isfile(fast_tokenizer_file)
and (cls is TokenizersBackend or "__init__" not in cls.__dict__ or trust_remote_code)
):
local_kwargs["tokenizer_object"] = TokenizerFast.from_file(fast_tokenizer_file)
return local_kwargs
elif fast_tokenizer_file is not None and os.path.isfile(fast_tokenizer_file):
# we extract vocab / merges from the tokenizer file to pass them to __init__
processor = TokenizerFast.from_file(fast_tokenizer_file).post_processor
with open(fast_tokenizer_file, encoding="utf-8") as tokenizer_handle:
tokenizer_json = json.load(tokenizer_handle)
vocab = tokenizer_json.get("model", {}).get("vocab", None)
if cls.model is None:
if isinstance(vocab, list):
vocab = list(map(tuple, vocab)) # TODO just for now
elif cls.model.__name__ == "Unigram":
if vocab and isinstance(vocab[0], (list, tuple)):
vocab = [tuple(item) for item in vocab]
elif cls.model.__name__ == "WordLevel":
vocab = {token: i for i, token in enumerate(vocab)}
elif cls.model.__name__ == "BPE" or cls.model.__name__ == "WordPiece":
if isinstance(vocab, list):
vocab = {token[0] if isinstance(token, list) else token: i for i, token in enumerate(vocab)}
local_kwargs["vocab"] = vocab
model_type = getattr(cls, "model", None)
if "merges" in tokenizer_json.get("model", {}) and (model_type and model_type.__name__ == "BPE"):
merges = tokenizer_json["model"]["merges"]
merges = [tuple(merge.split(" ")) if isinstance(merge, str) else tuple(merge) for merge in merges]
local_kwargs["merges"] = merges
if processor is not None:
local_kwargs["post_processor"] = processor
return local_kwargs
vocab_file = local_kwargs.get("vocab_file")
merges_file = local_kwargs.get("merges_file")
vocab = local_kwargs.get("vocab")
merges = local_kwargs.get("merges")
# Tekken converter (Mistral)
if isinstance(vocab_file, str) and vocab_file.endswith("tekken.json") and os.path.isfile(vocab_file):
from .convert_slow_tokenizer import MistralConverter
local_kwargs["vocab"], local_kwargs["merges"] = MistralConverter(
vocab_file=vocab_file
).extract_vocab_merges_from_model(vocab_file)
return local_kwargs
# SentencePiece model (with TikToken fallback)
if isinstance(vocab_file, str) and os.path.isfile(vocab_file) and vocab_file.endswith(".model"):
try:
from .convert_slow_tokenizer import SentencePieceExtractor
local_kwargs = SentencePieceExtractor(vocab_file).extract(cls.model, **local_kwargs)
try:
from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
converter_class = SLOW_TO_FAST_CONVERTERS.get(cls.__name__)
if converter_class is not None and hasattr(converter_class, "convert_from_spm"):
local_kwargs = converter_class.convert_from_spm(**local_kwargs)
except Exception as e:
logger.warning(
f"Could not reorder vocab using converter for {cls.__name__} due to {e}. Falling back to raw SentencePiece extraction."
)
# what used to be in `convert_slow`
if hasattr(cls, "convert_from_spm_model"):
local_kwargs = cls.convert_from_spm_model(**local_kwargs)
except Exception as e: # TODO only catch deserialization error here!
logger.warning(
f"Could not extract SentencePiece model from {vocab_file} using sentencepiece library due to {e}. "
"Falling back to TikToken extractor."
)
from .convert_slow_tokenizer import TikTokenConverter
local_kwargs["vocab"], local_kwargs["merges"] = TikTokenConverter(
vocab_file=vocab_file, extra_special_tokens=local_kwargs.get("extra_special_tokens")
).extract_vocab_merges_from_model(vocab_file)
return local_kwargs
# Fallback to standard vocab/merges files if they existed!
if vocab is None and isinstance(vocab_file, str) and os.path.isfile(vocab_file):
local_kwargs["vocab"] = vocab_file
vocab = local_kwargs["vocab"]
if merges is None and isinstance(merges_file, str) and os.path.isfile(merges_file):
local_kwargs["merges"] = merges_file
merges = local_kwargs["merges"]
# Generate merges automatically when not provided for BPE tokenizers
if merges is None and cls.model is not None and cls.model.__name__ == "BPE" and isinstance(vocab, dict):
# Gather special tokens from kwargs to skip in merge generation
def _iter_special_tokens(values: Iterable[Any]) -> list[str]:
collected: list[str] = []
for val in values:
if val is None:
continue
if isinstance(val, (list, tuple)):
collected.extend(_iter_special_tokens(val))
else:
collected.append(str(val))
return collected
special_tokens_keys = [
"pad_token",
"unk_token",
"bos_token",
"eos_token",
"sep_token",
"cls_token",
"mask_token",
"additional_special_tokens",
"extra_special_tokens",
]
skip_tokens: set[str] = set()
for key in special_tokens_keys:
if key in local_kwargs:
skip_tokens.update(_iter_special_tokens([local_kwargs[key]]))
merges = generate_merges(vocab, skip_tokens=skip_tokens)
local_kwargs["merges"] = merges
return local_kwargs
def __init__(self, *args, **kwargs):
tokenizer_object = kwargs.pop("tokenizer_object", None)
gguf_file = kwargs.pop("gguf_file", None)
fast_tokenizer_file = kwargs.pop("tokenizer_file", None)
# Note: added_tokens_decoder is NOT popped - it's passed to super().__init__() for processing
added_tokens_decoder = kwargs.get("added_tokens_decoder", {})
# Store add_prefix_space before super().__init__() to ensure it's not overridden
add_prefix_space = kwargs.get("add_prefix_space", False)
vocab_file = kwargs.get("vocab_file")
vocab = kwargs.get("vocab")
merges = kwargs.get("merges")
fast_tokenizer = None
if tokenizer_object is not None:
fast_tokenizer = copy.deepcopy(tokenizer_object)
elif fast_tokenizer_file is not None and os.path.isfile(fast_tokenizer_file):
# We have a serialization from tokenizers which let us directly build the backend
fast_tokenizer = TokenizerFast.from_file(fast_tokenizer_file)
elif gguf_file is not None:
# We need to convert a slow tokenizer to build the backend
gguf_path = cached_file(kwargs.get("name_or_path", ""), gguf_file, **kwargs)
gguf_param = load_gguf_checkpoint(gguf_path)
architecture = gguf_param["config"]["model_type"]
tokenizer_dict = gguf_param["tokenizer"]
tokenizer_config = gguf_param["tokenizer_config"]
fast_tokenizer, additional_kwargs = convert_gguf_tokenizer(architecture, tokenizer_dict)
kwargs.update(tokenizer_config)
if len(additional_kwargs) > 0:
kwargs.update(additional_kwargs)
elif self._tokenizer is None and vocab is not None:
# Build from vocab/merges extracted by convert_to_native_format
if merges is not None:
vocab_dict = vocab if isinstance(vocab, dict) else {w: i for i, (w, _) in enumerate(vocab)}
fast_tokenizer = TokenizerFast(BPE(vocab=vocab_dict, merges=merges, fuse_unk=True, dropout=None))
elif isinstance(vocab, dict):
fast_tokenizer = TokenizerFast(BPE(vocab=vocab, merges=[], fuse_unk=True, dropout=None))
elif isinstance(vocab, list) and vocab and isinstance(vocab[0], (tuple, list)):
fast_tokenizer = TokenizerFast(Unigram(vocab=vocab, unk_id=kwargs.get("unk_id", 0)))
elif self._tokenizer is None:
raise ValueError(
"Couldn't instantiate the backend tokenizer from one of: \n"
"(1) a `tokenizers` library serialization file, \n"
"(2) a slow tokenizer instance to convert or \n"
"(3) an equivalent slow tokenizer class to instantiate and convert. \n"
"You need to have sentencepiece or tiktoken installed to convert a slow tokenizer to a fast one."
)
# Only set defaults when creating TokenizersBackend from scratch
if fast_tokenizer_file is None and tokenizer_object is None and self._tokenizer is None:
kwargs.setdefault("bos_token", "")
kwargs.setdefault("eos_token", "")
if fast_tokenizer is not None:
self._tokenizer = fast_tokenizer
if self._tokenizer is None:
raise ValueError("The backend tokenizer is not correctly initialized.")
_truncation = self._tokenizer.truncation
if _truncation is not None:
self._tokenizer.enable_truncation(**_truncation)
kwargs.setdefault("max_length", _truncation["max_length"])
kwargs.setdefault("truncation_side", _truncation["direction"])
kwargs.setdefault("stride", _truncation["stride"])
kwargs.setdefault("truncation_strategy", _truncation["strategy"])
else:
self._tokenizer.no_truncation()
_padding = self._tokenizer.padding
if _padding is not None:
self._tokenizer.enable_padding(**_padding)
kwargs.setdefault("pad_token", _padding["pad_token"])
kwargs.setdefault("pad_token_type_id", _padding["pad_type_id"])
kwargs.setdefault("padding_side", _padding["direction"])
kwargs.setdefault("max_length", _padding["length"])
kwargs.setdefault("pad_to_multiple_of", _padding["pad_to_multiple_of"])
# Set backend to "tokenizers" if not already set
if "backend" not in kwargs:
kwargs["backend"] = "tokenizers"
explicit_bos_eos_in_kwargs = "add_bos_token" in kwargs or "add_eos_token" in kwargs
self._add_bos_token = kwargs.get("add_bos_token", False)
self._add_eos_token = kwargs.get("add_eos_token", False)
if post_processor := kwargs.pop("post_processor", None): # most reliable way to get the post-processor
self._tokenizer.post_processor = post_processor
self._should_update_post_processor = explicit_bos_eos_in_kwargs or self._tokenizer.post_processor is None
# We call this after having initialized the backend tokenizer because we update it.
super().__init__(**kwargs)
if vocab_file is not None:
self.vocab_file = vocab_file
# Ensure add_prefix_space is set correctly after parent init
self.add_prefix_space = add_prefix_space
self._tokenizer.encode_special_tokens = self.split_special_tokens
added_tokens_decoder_hash = {hash(repr(token)) for token in self.added_tokens_decoder}
tokens_to_add = [
token
for index, token in sorted(added_tokens_decoder.items(), key=lambda x: x[0])
if hash(repr(token)) not in added_tokens_decoder_hash
]
encoder = list(self.added_tokens_encoder.keys()) + [str(token) for token in tokens_to_add]
# if some of the special tokens are not already in the tokenizer, add them
# V5: Check both named special tokens and extra special tokens
# Iterate over _special_tokens_map to preserve AddedToken properties (lstrip, rstrip, etc.)
for special_token_value in self._special_tokens_map.values():
if special_token_value is None:
continue
if str(special_token_value) not in encoder and special_token_value not in tokens_to_add:
tokens_to_add.append(special_token_value)
# Also check extra special tokens
for token in self._extra_special_tokens:
if str(token) not in encoder and token not in tokens_to_add:
tokens_to_add.append(token)
if len(tokens_to_add) > 0:
tokens = []
all_named_tokens = [str(t) for t in self._special_tokens_map.values() if t]
for token in tokens_to_add:
if isinstance(token, str):
# Convert string to AddedToken, assuming it's special
token = AddedToken(token, special=True)
elif isinstance(token, AddedToken):
# Ensure the special flag is set correctly for special tokens
if not token.special and str(token) in all_named_tokens:
token.special = True
tokens.append(token)
if tokens:
# These tokens are from the special tokens map
self.add_tokens(tokens)
try:
vocab_size = self._tokenizer.get_vocab_size()
except NotImplementedError:
vocab_size = 0
# Optionally patches mistral tokenizers with wrong regex
if vocab_size > 100000 and getattr(self._tokenizer, "pre_tokenizer", None) is not None:
kwargs.pop("tokenizer", None)
self._tokenizer = self._patch_mistral_regex(
self._tokenizer,
self.init_kwargs.get("name_or_path", None),
init_kwargs=self.init_kwargs,
fix_mistral_regex=kwargs.get("fix_mistral_regex"),
**kwargs,
)
self._should_update_post_processor = (
self._should_update_post_processor or self._tokenizer.post_processor is None
)
if self._should_update_post_processor:
self.update_post_processor()
@property
def is_fast(self) -> bool:
return True
@property
def can_save_slow_tokenizer(self) -> bool:
"""
`bool`: Whether or not the slow tokenizer can be saved. For a sentencepiece based slow tokenizer, this
can only be `True` if the original `"sentencepiece.model"` was not deleted.
"""
if "vocab_file" in self.vocab_files_names and self.vocab_files_names["vocab_file"].endswith(".model"):
if hasattr(self, "vocab_file") and self.vocab_file:
# If the vocab file is a sentencepiece model, we can save it
return os.path.isfile(self.vocab_file)
return False
else:
return True
def save_vocabulary(self, save_directory: str, filename_prefix: str | None = None) -> tuple[str]:
if not os.path.isdir(save_directory):
logger.error(f"Vocabulary path ({save_directory}) should be a directory")
return
out_vocab_file = os.path.join(
save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
)
if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
copyfile(self.vocab_file, out_vocab_file)
return (out_vocab_file,)
def update_post_processor(self):
"""
Updates the underlying post processor with the current `bos_token` and `eos_token`.
"""
bos = self.bos_token
bos_token_id = self.bos_token_id
if bos is None and self.add_bos_token:
raise ValueError("add_bos_token = True but bos_token = None")
eos = self.eos_token
eos_token_id = self.eos_token_id
# If eos_token is None and add_eos_token is True, silently disable add_eos_token
# This allows tokenizers to set add_eos_token even if eos_token is not configured
if eos is None and self.add_eos_token:
self.add_eos_token = False
return
single = f"{(bos + ':0 ') if self.add_bos_token else ''}$A:0{(' ' + eos + ':0') if self.add_eos_token else ''}"
pair = f"{single}{(' ' + bos + ':1') if self.add_bos_token else ''} $B:1{(' ' + eos + ':1') if self.add_eos_token else ''}"
special_tokens = []
if self.add_bos_token:
special_tokens.append((bos, bos_token_id))
if self.add_eos_token:
special_tokens.append((eos, eos_token_id))
self._tokenizer.post_processor = processors.TemplateProcessing(
single=single, pair=pair, special_tokens=special_tokens
)
@property
def add_eos_token(self):
return getattr(self, "_add_eos_token", False)
@property
def add_bos_token(self):
return getattr(self, "_add_bos_token", False)
@add_eos_token.setter
def add_eos_token(self, value):
object.__setattr__(self, "_add_eos_token", value)
self.update_post_processor()
@add_bos_token.setter
def add_bos_token(self, value):
object.__setattr__(self, "_add_bos_token", value)
self.update_post_processor()
def _post_init(self):
"""
Post-initialization hook that runs after the tokenizer is fully set up.
This is called by from_pretrained() after loading the tokenizer, which allows
us to add any special tokens that may have been passed as AddedToken objects.
Child classes should call super()._post_init() if they override this method.
"""
tokens_to_add = []
# V5: Check named special tokens
for token_value in self._special_tokens_map.values():
if token_value is None:
continue
if isinstance(token_value, AddedToken):
tokens_to_add.append(token_value)
elif isinstance(token_value, str):
tokens_to_add.append(AddedToken(token_value, special=True, normalized=False))
# V5: Check extra special tokens
for token in self._extra_special_tokens:
if isinstance(token, AddedToken):
tokens_to_add.append(token)
elif isinstance(token, str):
tokens_to_add.append(AddedToken(token, special=True, normalized=False))
if tokens_to_add:
# Ensure special tokens are added as such to the backend
self.add_tokens(tokens_to_add, special_tokens=True)
if getattr(self, "_should_update_post_processor", True) or self._tokenizer.post_processor is None:
self.update_post_processor()
@property
def vocab_size(self) -> int:
"""
`int`: Size of the base vocabulary (without the added tokens).
"""
return self._tokenizer.get_vocab_size(with_added_tokens=False)
def get_vocab(self) -> dict[str, int]:
return self._tokenizer.get_vocab(with_added_tokens=True)
@property
def vocab(self) -> dict[str, int]:
return self.get_vocab()
@property
def added_tokens_encoder(self) -> dict[str, int]:
"""
Returns the sorted mapping from string to index. The added tokens encoder is cached for performance
optimisation in `self._added_tokens_encoder` for the slow tokenizers.
"""
return {k.content: v for v, k in sorted(self.added_tokens_decoder.items(), key=lambda item: item[0])}
@property
def added_tokens_decoder(self) -> dict[int, AddedToken]:
"""
Returns the added tokens in the vocabulary as a dictionary of index to AddedToken.
Returns:
`dict[str, int]`: The added tokens.
"""
return self._tokenizer.get_added_tokens_decoder()
def get_added_vocab(self) -> dict[str, int]:
"""
Returns the added tokens in the vocabulary as a dictionary of token to index.
Returns:
`dict[str, int]`: The added tokens.
"""
return {k.content: v for v, k in sorted(self.added_tokens_decoder.items(), key=lambda item: item[0])}
def __bool__(self) -> bool:
"""
Returns True, to avoid expensive `assert tokenizer` gotchas.
"""
return True
def __len__(self) -> int:
"""
Size of the full vocabulary with the added tokens.
"""
return self._tokenizer.get_vocab_size(with_added_tokens=True)
@property
def backend_tokenizer(self) -> TokenizerFast:
"""
`tokenizers.implementations.BaseTokenizer`: The Rust tokenizer used as a backend.
"""
return self._tokenizer
@property
def decoder(self) -> DecoderFast:
"""
`tokenizers.decoders.Decoder`: The Rust decoder for this tokenizer.
"""
return self._tokenizer.decoder
def _convert_encoding(
self,
encoding: EncodingFast,
return_token_type_ids: bool | None = None,
return_attention_mask: bool | None = None,
return_overflowing_tokens: bool = False,
return_special_tokens_mask: bool = False,
return_offsets_mapping: bool = False,
return_length: bool = False,
verbose: bool = True,
) -> tuple[dict[str, Any], list[EncodingFast]]:
"""
Convert the encoding representation (from low-level HuggingFace tokenizer output) to a python Dict and a list
of encodings, take care of building a batch from overflowing tokens.
Overflowing tokens are converted to additional examples (like batches) so the output values of the dict are
lists (overflows) of lists (tokens).
Output shape: (overflows, sequence length)
"""
if return_token_type_ids is None:
return_token_type_ids = "token_type_ids" in self.model_input_names
if return_attention_mask is None:
return_attention_mask = "attention_mask" in self.model_input_names
if return_overflowing_tokens and encoding.overflowing is not None:
encodings = [encoding] + encoding.overflowing
else:
encodings = [encoding]
encoding_dict = defaultdict(list)
for e in encodings:
encoding_dict["input_ids"].append(e.ids)
if return_token_type_ids:
encoding_dict["token_type_ids"].append(e.type_ids)
if return_attention_mask:
encoding_dict["attention_mask"].append(e.attention_mask)
if return_special_tokens_mask:
encoding_dict["special_tokens_mask"].append(e.special_tokens_mask)
if return_offsets_mapping:
encoding_dict["offset_mapping"].append(e.offsets)
if return_length:
encoding_dict["length"].append(len(e.ids))
return encoding_dict, encodings
def _convert_token_to_id_with_added_voc(self, token: str) -> int:
index = self._tokenizer.token_to_id(token)
if index is None:
return self.unk_token_id
return index
def _convert_id_to_token(self, index: int) -> str | None:
return self._tokenizer.id_to_token(int(index))
def _add_tokens(self, new_tokens: list[str | AddedToken], special_tokens=False) -> int:
if special_tokens:
return self._tokenizer.add_special_tokens(new_tokens)
return self._tokenizer.add_tokens(new_tokens)
def num_special_tokens_to_add(self, pair: bool = False) -> int:
"""
Returns the number of added tokens when encoding a sequence with special tokens.
This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not put
this inside your training loop.
Args:
pair (`bool`, *optional*, defaults to `False`):
Whether the number of added tokens should be computed in the case of a sequence pair or a single
sequence.
Returns:
`int`: Number of special tokens added to sequences.
"""
return self._tokenizer.num_special_tokens_to_add(pair)
def convert_ids_to_tokens(self, ids: int | list[int], skip_special_tokens: bool = False) -> str | list[str]:
"""
Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and
added tokens.
Args:
ids (`int` or `list[int]`):
The token id (or token ids) to convert to tokens.
skip_special_tokens (`bool`, *optional*, defaults to `False`):
Whether or not to remove special tokens in the decoding.
Returns:
`str` or `list[str]`: The decoded token(s).
"""
if isinstance(ids, int):
return self._tokenizer.id_to_token(ids)
tokens = []
# self.all_special_ids is an @property which may be slow, so only compute it once before the loop
ids_to_skip = set(self.all_special_ids) if skip_special_tokens else set()
for index in ids:
index = int(index)
if index in ids_to_skip:
continue
tokens.append(self._tokenizer.id_to_token(index))
return tokens
def tokenize(self, text: str, pair: str | None = None, add_special_tokens: bool = False, **kwargs) -> list[str]:
return self._encode_plus(text=text, text_pair=pair, add_special_tokens=add_special_tokens, **kwargs).tokens()
def set_truncation_and_padding(
self,
padding_strategy: PaddingStrategy,
truncation_strategy: TruncationStrategy,
max_length: int,
stride: int,
pad_to_multiple_of: int | None,
padding_side: str | None,
):
"""
Define the truncation and the padding strategies for fast tokenizers (provided by HuggingFace tokenizers
library) and restore the tokenizer settings afterwards.
The provided tokenizer has no padding / truncation strategy before the managed section. If your tokenizer set a
padding / truncation strategy before, then it will be reset to no padding / truncation when exiting the managed
section.
Args:
padding_strategy ([`~utils.PaddingStrategy`]):
The kind of padding that will be applied to the input
truncation_strategy ([`~tokenization_utils_base.TruncationStrategy`]):
The kind of truncation that will be applied to the input
max_length (`int`):
The maximum size of a sequence.
stride (`int`):
The stride to use when handling overflow.
pad_to_multiple_of (`int`, *optional*):
If set will pad the sequence to a multiple of the provided value. This is especially useful to enable
the use of Tensor Cores on NVIDIA hardware with compute capability `>= 7.5` (Volta).
padding_side (`str`, *optional*):
The side on which the model should have padding applied. Should be selected between ['right', 'left'].
Default value is picked from the class attribute of the same name.
"""
_truncation = self._tokenizer.truncation
_padding = self._tokenizer.padding
# Set truncation and padding on the backend tokenizer
if truncation_strategy == TruncationStrategy.DO_NOT_TRUNCATE:
if _truncation is not None:
self._tokenizer.no_truncation()
else:
target = {
"max_length": max_length,
"stride": stride,
"strategy": truncation_strategy.value,
"direction": self.truncation_side,
}
# _truncation might contain more keys that the target `transformers`
# supports. Use only the target keys to trigger `enable_truncation`.
# This should enable this code to works on various `tokenizers`
# targets.
if _truncation is None:
current = None
else:
current = {k: _truncation.get(k, None) for k in target}
if current != target:
self._tokenizer.enable_truncation(**target)
if padding_strategy == PaddingStrategy.DO_NOT_PAD:
if _padding is not None:
self._tokenizer.no_padding()
else:
length = max_length if padding_strategy == PaddingStrategy.MAX_LENGTH else None
target = {
"length": length,
"direction": padding_side if padding_side is not None else self.padding_side,
"pad_id": self.pad_token_id,
"pad_token": self.pad_token,
"pad_type_id": self.pad_token_type_id,
"pad_to_multiple_of": pad_to_multiple_of,
}
if _padding != target:
self._tokenizer.enable_padding(**target)
def _encode_plus(
self,
text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput],
text_pair: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] | None = None,
add_special_tokens: bool = True,
padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
max_length: int | None = None,
stride: int = 0,
is_split_into_words: bool = False,
pad_to_multiple_of: int | None = None,
padding_side: str | None = None,
return_tensors: bool | None = None,
return_token_type_ids: bool | None = None,
return_attention_mask: bool | None = None,
return_overflowing_tokens: bool = False,
return_special_tokens_mask: bool = False,
return_offsets_mapping: bool = False,
return_length: bool = False,
verbose: bool = True,
split_special_tokens: bool | None = None,
**kwargs,
) -> BatchEncoding:
# Input validation (from _call_one)
def _is_valid_text_input(t):
if isinstance(t, str):
return True
elif isinstance(t, (list, tuple)):
if len(t) == 0:
return True
elif isinstance(t[0], str):
return True
elif isinstance(t[0], (list, tuple)):
if len(t[0]) == 0 or isinstance(t[0][0], str):
return True
elif isinstance(t[0][0], (list, tuple)):
return len(t[0][0]) == 0 or isinstance(t[0][0][0], str)
else:
return False
else:
return False
else:
return False
if not _is_valid_text_input(text):
raise ValueError(
"text input must be of type `str` (single example), `list[str]` (batch or single pretokenized example) "
"or `list[list[str]]` (batch of pretokenized examples) or `list[tuple[list[str], list[str]]]` (batch of pretokenized sequence pairs)."
)
if text_pair is not None and not _is_valid_text_input(text_pair):
raise ValueError(
"text input must be of type `str` (single example), `list[str]` (batch or single pretokenized example) "
"or `list[list[str]]` (batch of pretokenized examples) or `list[tuple[list[str], list[str]]]` (batch of pretokenized sequence pairs)."
)
# Batch detection (from _call_one)
if is_split_into_words:
is_batched = isinstance(text, (list, tuple)) and text and isinstance(text[0], (list, tuple))
else:
is_batched = isinstance(text, (list, tuple))
if is_batched:
# Batch validation
if isinstance(text_pair, str):
raise TypeError(
"when tokenizing batches of text, `text_pair` must be a list or tuple with the same length as"
" `text`."
)
if text_pair is not None and len(text) != len(text_pair):
raise ValueError(
f"batch length of `text`: {len(text)} does not match batch length of `text_pair`:"
f" {len(text_pair)}."
)
batch_text_or_text_pairs = list(zip(text, text_pair)) if text_pair is not None else text
else:
# Single input - convert to batch format
batch_text_or_text_pairs = [(text, text_pair)] if text_pair else [text]
# Set tokenizer configuration (from _batch_encode_plus)
if not isinstance(batch_text_or_text_pairs, (tuple, list)):
raise TypeError(
f"batch_text_or_text_pairs has to be a list or a tuple (got {type(batch_text_or_text_pairs)})"
)
self.set_truncation_and_padding(
padding_strategy=padding_strategy,
truncation_strategy=truncation_strategy,
max_length=max_length,
stride=stride,
pad_to_multiple_of=pad_to_multiple_of,
padding_side=padding_side,
)
# Use self.split_special_tokens as default if not explicitly provided
if split_special_tokens is None:
split_special_tokens = self.split_special_tokens
if self._tokenizer.encode_special_tokens != split_special_tokens:
self._tokenizer.encode_special_tokens = split_special_tokens
# Direct rust backend call
encodings = self._tokenizer.encode_batch(
batch_text_or_text_pairs,
add_special_tokens=add_special_tokens,
is_pretokenized=is_split_into_words,
)
# Convert encodings to BatchEncoding format
tokens_and_encodings = [
self._convert_encoding(
encoding=encoding,
return_token_type_ids=return_token_type_ids,
return_attention_mask=return_attention_mask,
return_overflowing_tokens=return_overflowing_tokens,
return_special_tokens_mask=return_special_tokens_mask,
return_offsets_mapping=return_offsets_mapping,
return_length=return_length,
verbose=verbose,
)
for encoding in encodings
]
# Convert the output to have dict[list] from list[dict]
sanitized_tokens = {}
for key in tokens_and_encodings[0][0]:
stack = [e for item, _ in tokens_and_encodings for e in item[key]]
sanitized_tokens[key] = stack
sanitized_encodings = [e for _, item in tokens_and_encodings for e in item]
# If returning overflowing tokens, we need to return a mapping
if return_overflowing_tokens:
overflow_to_sample_mapping = []
for i, (toks, _) in enumerate(tokens_and_encodings):
overflow_to_sample_mapping += [i] * len(toks["input_ids"])
sanitized_tokens["overflow_to_sample_mapping"] = overflow_to_sample_mapping
for input_ids in sanitized_tokens["input_ids"]:
self._eventual_warn_about_too_long_sequence(input_ids, max_length, verbose)
batched_output = BatchEncoding(sanitized_tokens, sanitized_encodings, tensor_type=return_tensors)
# If single input, remove the batch dimension (unless returning overflowing tokens)
if not is_batched and return_tensors is None and not return_overflowing_tokens:
batched_output = BatchEncoding(
{
key: (value[0] if len(value) > 0 and isinstance(value[0], list) else value)
for key, value in batched_output.items()
},
batched_output.encodings,
)
return batched_output
def convert_tokens_to_string(self, tokens: list[str]) -> str:
return (
self.backend_tokenizer.decoder.decode(tokens)
if self.backend_tokenizer.decoder is not None
else " ".join(tokens)
)
def _decode(
self,
token_ids: int | list[int],
skip_special_tokens: bool = False,
clean_up_tokenization_spaces: bool | None = None,
**kwargs,
) -> str:
# Removed: use_source_tokenizer parameter (unused)
kwargs.pop("use_source_tokenizer", None) # Pop if present to avoid errors
if isinstance(token_ids, int):
token_ids = [token_ids]
if isinstance(token_ids, dict):
token_ids = token_ids["input_ids"]
text = self._tokenizer.decode(token_ids, skip_special_tokens=skip_special_tokens)
clean_up_tokenization_spaces = (
clean_up_tokenization_spaces
if clean_up_tokenization_spaces is not None
else self.clean_up_tokenization_spaces
)
if clean_up_tokenization_spaces:
# Call custom cleanup method if it exists
if hasattr(self, "clean_up_tokenization") and callable(self.clean_up_tokenization):
text = self.clean_up_tokenization(text)
else:
# Apply standard cleanup
text = (
text.replace(" .", ".")
.replace(" ?", "?")
.replace(" !", "!")
.replace(" ,", ",")
.replace(" ' ", "'")
.replace(" n't", "n't")
.replace(" 'm", "'m")
.replace(" 's", "'s")
.replace(" 've", "'ve")
.replace(" 're", "'re")
)
return text
def _save_pretrained(
self,
save_directory: str | os.PathLike,
file_names: tuple[str, ...],
legacy_format: bool | None = None,
filename_prefix: str | None = None,
) -> tuple[str, ...]:
save_directory = str(save_directory)
tokenizer_file = os.path.join(
save_directory, (filename_prefix + "-" if filename_prefix else "") + TOKENIZER_FILE
)
self.backend_tokenizer.save(tokenizer_file)
file_names = file_names + (tokenizer_file,)
return file_names
def train_new_from_iterator(
self,
text_iterator,
vocab_size,
length=None,
new_special_tokens=None,
special_tokens_map=None,
**kwargs,
):
"""
Trains a tokenizer on a new corpus with the same defaults (in terms of special tokens or tokenization pipeline)
as the current one.
Args:
text_iterator (generator of `list[str]`):
The training corpus. Should be a generator of batches of texts, for instance a list of lists of texts
if you have everything in memory.
vocab_size (`int`):
The size of the vocabulary you want for your tokenizer.
length (`int`, *optional*):
The total number of sequences in the iterator. This is used to provide meaningful progress tracking
new_special_tokens (list of `str` or `AddedToken`, *optional*):
A list of new special tokens to add to the tokenizer you are training.
special_tokens_map (`dict[str, str]`, *optional*):
If you want to rename some of the special tokens this tokenizer uses, pass along a mapping old special
token name to new special token name in this argument.
kwargs (`dict[str, Any]`, *optional*):
Additional keyword arguments passed along to the trainer from the 🤗 Tokenizers library.
Returns:
[`PreTrainedTokenizerFast`]: A new tokenizer of the same type as the original one, trained on
`text_iterator`.
"""
tokenizer_json = json.loads(self._tokenizer.to_str())
# Remove added tokens for now (uses IDs of tokens)
added_tokens = tokenizer_json.pop("added_tokens")
# Remove post processor for now (uses IDs of tokens)
post_processor = tokenizer_json.pop("post_processor")
unk_token = None
# Remove vocab
if tokenizer_json["model"]["type"] == "BPE":
tokenizer_json["model"]["vocab"] = {}
tokenizer_json["model"]["merges"] = []
elif tokenizer_json["model"]["type"] == "Unigram":
if tokenizer_json["model"]["unk_id"] is not None:
unk_id = tokenizer_json["model"]["unk_id"]
unk_token = tokenizer_json["model"]["vocab"][unk_id][0]
if special_tokens_map is not None and unk_token in special_tokens_map:
unk_token = special_tokens_map[unk_token]
tokenizer_json["model"]["unk_id"] = 0
tokenizer_json["model"]["vocab"] = [[unk_token, 0.0]]
elif tokenizer_json["model"]["type"] in ["WordLevel", "WordPiece"]:
tokenizer_json["model"]["vocab"] = {}
else:
raise ValueError(
f"This method does not support this type of tokenizer (found {tokenizer_json['model']['type']}) "
"only BPE, Unigram, WordLevel and WordPiece."
)
if (
special_tokens_map is not None
and "unk_token" in tokenizer_json["model"]
and tokenizer_json["model"]["unk_token"] in special_tokens_map
):
tokenizer_json["model"]["unk_token"] = special_tokens_map[tokenizer_json["model"]["unk_token"]]
tokenizer = TokenizerFast.from_str(json.dumps(tokenizer_json))
# Get the special tokens from the current tokenizer if none are specified.
special_tokens = []
for added_token in added_tokens:
special = added_token.pop("special", None)
_ = added_token.pop("id", None)
if tokenizer_json["model"]["type"] != "Unigram" and not special:
continue
if special_tokens_map is not None and added_token["content"] in special_tokens_map:
added_token["content"] = special_tokens_map[added_token["content"]]
special_tokens.append(AddedToken(**added_token))
if new_special_tokens is not None:
special_tokens.extend(new_special_tokens)
# Trainer needs to know the end of word / continuing subword thingies in BPE
if (
tokenizer_json["model"]["type"] == "BPE"
and "continuing_subword_prefix" not in kwargs
and tokenizer_json["model"]["continuing_subword_prefix"] is not None
):
kwargs["continuing_subword_prefix"] = tokenizer_json["model"]["continuing_subword_prefix"]
if (
tokenizer_json["model"]["type"] == "BPE"
and "end_of_word_suffix" not in kwargs
and tokenizer_json["model"]["end_of_word_suffix"] is not None
):
kwargs["end_of_word_suffix"] = tokenizer_json["model"]["end_of_word_suffix"]
if tokenizer_json["model"]["type"] == "Unigram" and unk_token is not None:
kwargs["unk_token"] = unk_token
if tokenizer_json["pre_tokenizer"] is not None:
if (
tokenizer_json["pre_tokenizer"]["type"] == "ByteLevel"
or tokenizer_json["pre_tokenizer"]["type"] == "Sequence"
and "pretokenizers" in tokenizer_json["pre_tokenizer"]
and any(
pretokenizer["type"] == "ByteLevel"
for pretokenizer in tokenizer_json["pre_tokenizer"]["pretokenizers"]
)
):
kwargs["initial_alphabet"] = pre_tokenizers_fast.ByteLevel.alphabet()
trainer_class = MODEL_TO_TRAINER_MAPPING[tokenizer_json["model"]["type"]]
trainer = trainer_class(vocab_size=vocab_size, special_tokens=special_tokens, **kwargs)
tokenizer.train_from_iterator(text_iterator, length=length, trainer=trainer)
if post_processor is not None:
trained_tokenizer_json = json.loads(tokenizer.to_str())
# Almost done, we just have to adjust the token IDs in the post processor
if "special_tokens" in post_processor:
for key in post_processor["special_tokens"]:
tokens = post_processor["special_tokens"][key]["tokens"]
if special_tokens_map is not None:
tokens = [special_tokens_map.get(token, token) for token in tokens]
post_processor["special_tokens"][key]["tokens"] = tokens
for token in tokens:
token_id = tokenizer.token_to_id(token)
if token_id is None:
raise ValueError(
"Attempted to set a token in the post processor that does not exist in the mapping"
)
post_processor["special_tokens"][key]["ids"] = [tokenizer.token_to_id(token) for token in tokens]
for special_token in ["cls", "sep"]:
if special_token in post_processor:
token, _ = post_processor[special_token]
if special_tokens_map is not None and token in special_tokens_map:
token = special_tokens_map[token]
token_id = tokenizer.token_to_id(token)
if token_id is None:
raise ValueError(
"Attempted to set a token in the post processor that does not exist in the mapping"
)
post_processor[special_token] = [token, token_id]
trained_tokenizer_json["post_processor"] = post_processor
tokenizer = TokenizerFast.from_str(json.dumps(trained_tokenizer_json))
kwargs = self.init_kwargs.copy()
# V5: Map pad/cls/mask token at the Transformers level (named tokens only)
for token in PreTrainedTokenizerBase.SPECIAL_TOKENS_ATTRIBUTES:
if getattr(self, token) is not None:
special_token = getattr(self, token)
if special_tokens_map is not None and special_token in special_tokens_map:
special_token = special_tokens_map[special_token]
special_token_full = self._special_tokens_map.get(token, None)
if isinstance(special_token_full, AddedToken):
# Create an added token with the same parameters except the content
kwargs[token] = AddedToken(
special_token,
single_word=special_token_full.single_word,
lstrip=special_token_full.lstrip,
rstrip=special_token_full.rstrip,
normalized=special_token_full.normalized,
special=True,
)
else:
kwargs[token] = special_token
# V5: Handle extra special tokens
extra_special_tokens = self.extra_special_tokens.copy() if self.extra_special_tokens else []
if new_special_tokens is not None:
extra_special_tokens.extend(new_special_tokens)
if len(extra_special_tokens) > 0:
kwargs["extra_special_tokens"] = extra_special_tokens
# Always try to pass tokenizer_object in kwargs first (standard TokenizersBackend usage)
# If the class creates its own tokenizer and passes it explicitly to super().__init__(),
# this will cause a TypeError, which we catch and handle by removing tokenizer_object
# from kwargs and setting _tokenizer directly after initialization.
kwargs["tokenizer_object"] = tokenizer
try:
return self.__class__(**kwargs)
except TypeError as e:
# Check if the error is due to multiple values for tokenizer_object
if "multiple values for keyword argument 'tokenizer_object'" in str(e):
# Class creates its own tokenizer and passes it explicitly (like LayoutLMv3Tokenizer)
# Remove tokenizer_object from kwargs and set _tokenizer directly
kwargs.pop("tokenizer_object", None)
new_tokenizer = self.__class__(**kwargs)
new_tokenizer._tokenizer = tokenizer
return new_tokenizer
else:
# Some other TypeError, re-raise it
raise
@classmethod
def _patch_mistral_regex(
cls,
tokenizer,
pretrained_model_name_or_path,
token=None,
cache_dir=None,
local_files_only=False,
_commit_hash=None,
is_local=False,
init_kwargs=None,
fix_mistral_regex=None,
**kwargs,
):
"""
Patches mistral related tokenizers with incorrect regex if detected
1) Local file with an associated config saved next to it
>> Model type one of the mistral models (on older versions)
2) Remote models on the hub from official mistral models
>> Tags including `base_model:.*mistralai`
"""
import re
from huggingface_hub import model_info
from packaging import version
from transformers.utils.hub import cached_file
def is_base_mistral(model_id: str) -> bool:
model = model_info(model_id)
if model.tags is not None:
if re.search("base_model:.*mistralai", "".join(model.tags)):
return True
return False
if is_offline_mode():
is_local = True
if pretrained_model_name_or_path is not None and (
is_local or (not is_local and is_base_mistral(pretrained_model_name_or_path))
):
_config_file = cached_file(
pretrained_model_name_or_path,
"config.json",
cache_dir=cache_dir,
token=token,
local_files_only=local_files_only,
_raise_exceptions_for_missing_entries=False,
_raise_exceptions_for_connection_errors=False,
_commit_hash=_commit_hash,
)
# Detected using a (local) mistral tokenizer
mistral_config_detected = False
if _config_file is not None:
with open(_config_file, encoding="utf-8") as f:
_config = json.load(f)
transformers_version = _config.get("transformers_version")
transformers_model_type = _config.get("model_type")
# Detect if we can skip the mistral fix by
# a) having a non-mistral tokenizer
# b) fixed version of transformers
if transformers_version and version.parse(transformers_version) <= version.parse("4.57.2"):
if (
is_local
and transformers_model_type is not None
and transformers_model_type
not in [
"mistral",
"mistral3",
"voxtral",
"ministral",
"pixtral",
]
):
return tokenizer
elif transformers_version and version.parse(transformers_version) > version.parse("4.57.3"):
return tokenizer
mistral_config_detected = True
if mistral_config_detected or (not is_local and is_base_mistral(pretrained_model_name_or_path)):
# Expose the `fix_mistral_regex` flag on the tokenizer when provided, even if no correction is applied.
if init_kwargs and "fix_mistral_regex" in init_kwargs:
setattr(tokenizer, "fix_mistral_regex", init_kwargs["fix_mistral_regex"])
# only warn if its not explicitly passed
if fix_mistral_regex is None and not getattr(tokenizer, "fix_mistral_regex", False):
setattr(tokenizer, "fix_mistral_regex", False)
logger.warning(
f"The tokenizer you are loading from '{pretrained_model_name_or_path}'"
f" with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e."
" This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue."
)
elif fix_mistral_regex is True or getattr(tokenizer, "fix_mistral_regex", False):
setattr(tokenizer, "fix_mistral_regex", True)
import tokenizers
split_pretokenizer = tokenizers.pre_tokenizers.Split(
pattern=tokenizers.Regex(
r"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+"
),
behavior="isolated",
)
current_pretokenizer = tokenizer.backend_tokenizer.pre_tokenizer
# Check if it's already a Sequence
if isinstance(current_pretokenizer, tokenizers.pre_tokenizers.Sequence):
# Replace the first element (the Split pattern)
tokenizer.backend_tokenizer.pre_tokenizer[0] = split_pretokenizer
else:
# Replace Metaspace with ByteLevel when adding Split, as Metaspace(split=False) doesn't
# work correctly with the Split pre-tokenizer and causes spaces to be lost during encoding
if isinstance(current_pretokenizer, tokenizers.pre_tokenizers.Metaspace):
current_pretokenizer = tokenizers.pre_tokenizers.ByteLevel(
add_prefix_space=False, use_regex=False
)
# Not a Sequence, so create one with Split + current pretokenizer
tokenizer.backend_tokenizer.pre_tokenizer = tokenizers.pre_tokenizers.Sequence(
[
split_pretokenizer,
current_pretokenizer,
]
)
return tokenizer
# Backward-compatible alias: allow referring to TokenizersBackend as PreTrainedTokenizerFast
PreTrainedTokenizerFast = TokenizersBackend