third_codes/eland
1
0
Fork 0
mirror of https://github.com/elastic/eland.git synced 2025-07-11 00:02:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

Upgrade transformers to 4.47 (#752)

Browse Source 
The upgrade fixes a crash tracing the baai/bge-m3 model
This commit is contained in:
David Kyle 2025-02-12 17:30:45 +00:00 committed by GitHub
parent acdeeeded2
commit ee4d701aa4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
eland/ml/pytorch/transformers.py
View File

@ -570,7 +570,7 @@ class _TraceableTextEmbeddingModel(_TransformerTraceableModel):
def _prepare_inputs(self) -> transformers.BatchEncoding: def _prepare_inputs(self) -> transformers.BatchEncoding:
return self._tokenizer( return self._tokenizer(
"This is an example sentence.", "This is an example sentence.",
padding="max_length", padding="longest",
return_tensors="pt", return_tensors="pt",
) )
@ -759,7 +759,7 @@ class TransformerModel:
# a random or very large value. # a random or very large value.
REASONABLE_MAX_LENGTH = 8192 REASONABLE_MAX_LENGTH = 8192
max_len = getattr(self._tokenizer, "model_max_length", None) max_len = getattr(self._tokenizer, "model_max_length", None)
if max_len is not None and max_len < REASONABLE_MAX_LENGTH: if max_len is not None and max_len <= REASONABLE_MAX_LENGTH:
return int(max_len) return int(max_len)
max_sizes = getattr(self._tokenizer, "max_model_input_sizes", dict()) max_sizes = getattr(self._tokenizer, "max_model_input_sizes", dict())

2
setup.py
View File

@ -65,7 +65,7 @@ extras = {
"sentence-transformers>=2.1.0,<=2.7.0", "sentence-transformers>=2.1.0,<=2.7.0",
# sentencepiece is a required dependency for the slow tokenizers # sentencepiece is a required dependency for the slow tokenizers
# https://huggingface.co/transformers/v4.4.2/migration.html#sentencepiece-is-removed-from-the-required-dependencies # https://huggingface.co/transformers/v4.4.2/migration.html#sentencepiece-is-removed-from-the-required-dependencies
"transformers[sentencepiece]>=4.31.0,<4.44.0", "transformers[sentencepiece]>=4.47.0",
], ],
} }
extras["all"] = list({dep for deps in extras.values() for dep in deps}) extras["all"] = list({dep for deps in extras.values() for dep in deps})