mirror of
https://github.com/xtekky/gpt4free.git
synced 2024-09-12 08:29:30 -04:00
9 lines
272 B
Python
9 lines
272 B
Python
|
import tiktoken
|
||
|
|
from typing import Union
|
||
|
|
|
||
|
|
def tokenize(text: str, model: str = 'gpt-3.5-turbo') -> Union[int, str]:
|
||
|
|
encoding = tiktoken.encoding_for_model(model)
|
||
|
|
encoded = encoding.encode(text)
|
||
|
|
num_tokens = len(encoded)
|
||
|
|
|
||
|
|
return num_tokens, encoded
|