You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
17 lines
598 B
17 lines
598 B
# Generated content DO NOT EDIT
|
|
from .. import pre_tokenizers
|
|
|
|
PreTokenizer = pre_tokenizers.PreTokenizer
|
|
BertPreTokenizer = pre_tokenizers.BertPreTokenizer
|
|
ByteLevel = pre_tokenizers.ByteLevel
|
|
CharDelimiterSplit = pre_tokenizers.CharDelimiterSplit
|
|
Digits = pre_tokenizers.Digits
|
|
FixedLength = pre_tokenizers.FixedLength
|
|
Metaspace = pre_tokenizers.Metaspace
|
|
Punctuation = pre_tokenizers.Punctuation
|
|
Sequence = pre_tokenizers.Sequence
|
|
Split = pre_tokenizers.Split
|
|
UnicodeScripts = pre_tokenizers.UnicodeScripts
|
|
Whitespace = pre_tokenizers.Whitespace
|
|
WhitespaceSplit = pre_tokenizers.WhitespaceSplit
|