You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
30 lines
841 B
30 lines
841 B
from .. import normalizers
|
|
|
|
|
|
Normalizer = normalizers.Normalizer
|
|
BertNormalizer = normalizers.BertNormalizer
|
|
NFD = normalizers.NFD
|
|
NFKD = normalizers.NFKD
|
|
NFC = normalizers.NFC
|
|
NFKC = normalizers.NFKC
|
|
Sequence = normalizers.Sequence
|
|
Lowercase = normalizers.Lowercase
|
|
Prepend = normalizers.Prepend
|
|
Strip = normalizers.Strip
|
|
StripAccents = normalizers.StripAccents
|
|
Nmt = normalizers.Nmt
|
|
Precompiled = normalizers.Precompiled
|
|
Replace = normalizers.Replace
|
|
ByteLevel = normalizers.ByteLevel
|
|
|
|
NORMALIZERS = {"nfc": NFC, "nfd": NFD, "nfkc": NFKC, "nfkd": NFKD}
|
|
|
|
|
|
def unicode_normalizer_from_str(normalizer: str) -> Normalizer:
|
|
if normalizer not in NORMALIZERS:
|
|
raise ValueError(
|
|
"{} is not a known unicode normalizer. Available are {}".format(normalizer, NORMALIZERS.keys())
|
|
)
|
|
|
|
return NORMALIZERS[normalizer]()
|