third_codes/eland
1
0
Fork 0
mirror of https://github.com/elastic/eland.git synced 2025-07-11 00:02:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix failed import of ST RoBERTa models (#637)

Browse Source 
Fixes an error uploading the sentence-transformers/all-distilroberta-v1 model
which failed with "missing 2 required positional arguments: 'token_type_ids' 
and 'position_ids'". The cause was that the tokenizer type was not recognised 
due to a typo
This commit is contained in:
David Kyle 2023-11-21 12:53:43 +00:00 committed by GitHub
parent af26897313
commit 081250cdec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
eland/ml/pytorch/transformers.py
View File

@ -311,7 +311,7 @@ class _SentenceTransformerWrapperModule(nn.Module): # type: ignore
( (
transformers.BartTokenizer, transformers.BartTokenizer,
transformers.MPNetTokenizer, transformers.MPNetTokenizer,
transformers.RobertaConfig, transformers.RobertaTokenizer,
transformers.XLMRobertaTokenizer, transformers.XLMRobertaTokenizer,
), ),
): ):

8
tests/ml/pytorch/test_pytorch_model_config_pytest.py
View File

@ -77,6 +77,14 @@ pytestmark = [
# have been imported # have been imported
if HAS_PYTORCH and HAS_SKLEARN and HAS_TRANSFORMERS: if HAS_PYTORCH and HAS_SKLEARN and HAS_TRANSFORMERS:
MODEL_CONFIGURATIONS = [ MODEL_CONFIGURATIONS = [
(
"sentence-transformers/all-distilroberta-v1",
"text_embedding",
TextEmbeddingInferenceOptions,
NlpRobertaTokenizationConfig,
512,
768,
),
( (
"intfloat/multilingual-e5-small", "intfloat/multilingual-e5-small",
"text_embedding", "text_embedding",