You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
9 lines
257 B
9 lines
257 B
2 months ago
|
def test_tl_simple_punct(tl_tokenizer):
|
||
|
|
text = "Sige, punta ka dito"
|
||
|
|
tokens = tl_tokenizer(text)
|
||
|
|
assert tokens[0].idx == 0
|
||
|
|
assert tokens[1].idx == 4
|
||
|
|
assert tokens[2].idx == 6
|
||
|
|
assert tokens[3].idx == 12
|
||
|
|
assert tokens[4].idx == 15
|