Upload model

configuration_cxrmate_ed.py

@@ -1,4 +1,5 @@
 import transformers
+from transformers.models.auto import CONFIG_MAPPING
 
 
 class CXRMateEDConfig(transformers.PretrainedConfig):
@@ -21,7 +22,7 @@ class CXRMateEDConfig(transformers.PretrainedConfig):
     ):
         super().__init__(**kwargs)
         self.vision_config = vision_config
-        self.text_config = text_config
+        
         self.index_value_encoder_intermediate_size = index_value_encoder_intermediate_size
         self.include_time_delta = include_time_delta
         self.time_delta_monotonic_inversion = time_delta_monotonic_inversion
@@ -31,6 +32,20 @@ class CXRMateEDConfig(transformers.PretrainedConfig):
         self.prompt_report_sections_filter = prompt_report_sections_filter
         self.pad_token_id = pad_token_id      
 
+        if isinstance(vision_config, dict):
+            vision_config = transformers.AutoConfig.from_pretrained(
+                'aehrc/uniformer_base_tl_384',
+                trust_remote_code=True,
+                **vision_config,
+            )
+            
+        self.vision_config = vision_config
+
+        if isinstance(text_config, dict):
+            text_config['model_type'] = text_config['model_type'] if 'model_type' in text_config else 'llama'
+            text_config = CONFIG_MAPPING[text_config['model_type']](**text_config)
+
+        self.text_config = text_config
 
 # class CXRMateEDConfig(transformers.PretrainedConfig):