Skip to content
Snippets Groups Projects
Verified Commit 0b3976c9 authored by Nico Jeske's avatar Nico Jeske
Browse files

feat(LLM): :bug: Allow loading models that execute custom code

parent b386fbd6
No related branches found
No related tags found
No related merge requests found
Pipeline #262827 failed
......@@ -37,7 +37,7 @@ class TokenizationService:
model_config: ModelConfig,
) -> PreTrainedTokenizer | PreTrainedTokenizerFast:
model_id = model_config.huggingface_id
return AutoTokenizer.from_pretrained(model_id)
return AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
def tokenCount(self, text: str) -> int:
"""
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment