third_codes/eland
1
0
Fork 0
mirror of https://github.com/elastic/eland.git synced 2025-07-11 00:02:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

Upgrade transformers to 4.47 (#752) (#759)

Browse Source
This commit is contained in:
github-actions[bot] 2025-02-12 22:23:00 +04:00 committed by GitHub
parent 6759b1db70
commit d50436b01c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
eland/ml/pytorch/transformers.py
View File

@ -570,7 +570,7 @@ class _TraceableTextEmbeddingModel(_TransformerTraceableModel):
def _prepare_inputs(self) -> transformers.BatchEncoding:
return self._tokenizer(
"This is an example sentence.",
padding="max_length",
padding="longest",
return_tensors="pt",
)
@ -759,7 +759,7 @@ class TransformerModel:
# a random or very large value.
REASONABLE_MAX_LENGTH = 8192
max_len = getattr(self._tokenizer, "model_max_length", None)
if max_len is not None and max_len < REASONABLE_MAX_LENGTH:
if max_len is not None and max_len <= REASONABLE_MAX_LENGTH:
return int(max_len)
max_sizes = getattr(self._tokenizer, "max_model_input_sizes", dict())

2
setup.py
View File

@ -65,7 +65,7 @@ extras = {
"sentence-transformers>=2.1.0,<=2.7.0",
# sentencepiece is a required dependency for the slow tokenizers
# https://huggingface.co/transformers/v4.4.2/migration.html#sentencepiece-is-removed-from-the-required-dependencies
"transformers[sentencepiece]>=4.31.0,<4.44.0",
"transformers[sentencepiece]>=4.47.0",
],
}
extras["all"] = list({dep for deps in extras.values() for dep in deps})