mirrors/transformers
0
0 Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2026-10-29 01:06:24 +08:00
Code Activity

a couple processors fixes

Browse Source
This commit is contained in:
itazap committed 2025-11-21 17:36:55 +01:00
1 parent ca988b9035
commit f5bc69efcd
2 files changed
+7 -6
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/models/clap/test_processing_clap.py
+3 -2
View File
@@ -18,6 +18,7 @@ import unittest
from transformers import ClapFeatureExtractor, ClapProcessor, RobertaTokenizer, RobertaTokenizerFast
from transformers.testing_utils import require_sentencepiece, require_torchaudio
from transformers.tokenization_utils_tokenizers import TokenizersBackend
from .test_feature_extraction_clap import floats_list
@@ -48,7 +49,7 @@ class ClapProcessorTest(unittest.TestCase):
processor = ClapProcessor.from_pretrained(self.tmpdirname)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer.get_vocab())
self.assertIsInstance(processor.tokenizer, RobertaTokenizerFast)
self.assertIsInstance(processor.tokenizer, TokenizersBackend)
self.assertEqual(processor.feature_extractor.to_json_string(), feature_extractor.to_json_string())
self.assertIsInstance(processor.feature_extractor, ClapFeatureExtractor)
@@ -65,7 +66,7 @@ class ClapProcessorTest(unittest.TestCase):
)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, RobertaTokenizerFast)
self.assertIsInstance(processor.tokenizer, TokenizersBackend)
self.assertEqual(processor.feature_extractor.to_json_string(), feature_extractor_add_kwargs.to_json_string())
self.assertIsInstance(processor.feature_extractor, ClapFeatureExtractor)
tests/models/idefics/test_processing_idefics.py
+4 -4
View File
@@ -159,8 +159,8 @@ class IdeficsProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor = IdeficsProcessor(tokenizer=tokenizer, image_processor=image_processor, return_tensors="pt")
predicted_tokens = [
"<s> Describe this image.\nAssistant:<unk><unk><unk><unk><unk><unk><unk><unk><unk>",
"<s> Describe this image.\nAssistant:<unk><unk><unk><unk><unk><unk><unk><unk><unk><unk>",
"<s>Describe this image.\nAssistant:<unk><unk><unk><unk><unk><unk><unk><unk><unk>",
"<s>Describe this image.\nAssistant:<unk><unk><unk><unk><unk><unk><unk><unk><unk><unk>",
]
predicted_attention_masks = [
([1] * 10) + ([0] * 9),
@@ -188,8 +188,8 @@ class IdeficsProcessorTest(ProcessorTesterMixin, unittest.TestCase):
processor = IdeficsProcessor(tokenizer=tokenizer, image_processor=image_processor)
predicted_tokens = [
"<unk><unk><unk><unk><unk><unk><unk><unk><unk><s> Describe this image.\nAssistant:",
"<unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><s> Describe this image.\nAssistant:",
"<unk><unk><unk><unk><unk><unk><unk><unk><unk><s>Describe this image.\nAssistant:",
"<unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><s>Describe this image.\nAssistant:",
]
predicted_attention_masks = [
([0] * 9) + ([1] * 10),