diff --git a/services/tokenizationService.py b/services/tokenizationService.py index 877198c2d3b712dfdd00e08256cbb2749a5e6d6c..90cd85a889befeb86ff1d74b05512b2ec86e6eb3 100644 --- a/services/tokenizationService.py +++ b/services/tokenizationService.py @@ -37,7 +37,7 @@ class TokenizationService: model_config: ModelConfig, ) -> PreTrainedTokenizer | PreTrainedTokenizerFast: model_id = model_config.huggingface_id - return AutoTokenizer.from_pretrained(model_id) + return AutoTokenizer.from_pretrained(model_id, trust_remote_code=True) def tokenCount(self, text: str) -> int: """