Skip to content
Snippets Groups Projects
Verified Commit 0b3976c9 authored by Nico Jeske's avatar Nico Jeske
Browse files

feat(LLM): :bug: Allow loading models that execute custom code

parent b386fbd6
No related branches found
No related tags found
No related merge requests found
Pipeline #262827 failed
...@@ -37,7 +37,7 @@ class TokenizationService: ...@@ -37,7 +37,7 @@ class TokenizationService:
model_config: ModelConfig, model_config: ModelConfig,
) -> PreTrainedTokenizer | PreTrainedTokenizerFast: ) -> PreTrainedTokenizer | PreTrainedTokenizerFast:
model_id = model_config.huggingface_id model_id = model_config.huggingface_id
return AutoTokenizer.from_pretrained(model_id) return AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
def tokenCount(self, text: str) -> int: def tokenCount(self, text: str) -> int:
""" """
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment