From 0b3976c9003e0b16c2032819f82a5d14eadf737d Mon Sep 17 00:00:00 2001
From: Nico Jeske <nico.jeske@tu-dortmund.de>
Date: Tue, 26 Nov 2024 19:54:36 +0000
Subject: [PATCH] feat(LLM): :bug: Allow loading models that execute custom
code
---
services/tokenizationService.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/services/tokenizationService.py b/services/tokenizationService.py
index 877198c..90cd85a 100644
--- a/services/tokenizationService.py
+++ b/services/tokenizationService.py
@@ -37,7 +37,7 @@ class TokenizationService:
model_config: ModelConfig,
) -> PreTrainedTokenizer | PreTrainedTokenizerFast:
model_id = model_config.huggingface_id
- return AutoTokenizer.from_pretrained(model_id)
+ return AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
def tokenCount(self, text: str) -> int:
"""
--
GitLab