Files
guru/dev-python/tiktoken/files/tiktoken-0.12.0-special-token-roudtrip.patch
Florian Albrechtskirchinger d999e1e10d dev-python/tiktoken: new package, add 0.12.0
Signed-off-by: Florian Albrechtskirchinger <falbrechtskirchinger@gmail.com>
2026-03-26 05:39:06 +01:00

14 lines
486 B
Diff

diff --git a/tests/test_encoding.py b/tests/test_encoding.py
index b77ca13..7ef65d9 100644
--- a/tests/test_encoding.py
+++ b/tests/test_encoding.py
@@ -152,7 +152,7 @@ def test_basic_roundtrip(make_enc):
def test_hyp_roundtrip(make_enc: Callable[[], tiktoken.Encoding], text):
enc = make_enc()
- assert text == enc.decode(enc.encode(text))
+ assert text == enc.decode(enc.encode(text, allowed_special="all"))
@pytest.mark.parametrize("make_enc", ENCODING_FACTORIES)