refactor: use _docarray_fields everywhere

Signed-off-by: samsja <sami.jaghouar@hotmail.fr>
docarray · samsja · Sep 10, 2023 · Jun 15, 2023 · Jun 15, 2023 · Jun 15, 2023
commit 979edc74ac0eb78ef5e1dbabcf2abc462d1278da
diff --git a/docarray/array/any_array.py b/docarray/array/any_array.py
@@ -60,7 +60,7 @@ def __class_getitem__(cls, item: Union[Type[BaseDoc], TypeVar, str]):
             class _DocArrayTyped(cls):  # type: ignore
                 doc_type: Type[BaseDoc] = cast(Type[BaseDoc], item)
 
-            for field in _DocArrayTyped.doc_type.__fields__.keys():
+            for field in _DocArrayTyped.doc_type._docarray_fields.keys():
 
                 def _property_generator(val: str):
                     def _getter(self):

diff --git a/docarray/array/doc_list/doc_list.py b/docarray/array/doc_list/doc_list.py
@@ -219,7 +219,7 @@ def __class_getitem__(cls, item: Union[Type[BaseDoc], TypeVar, str]):
 
         if (
             not is_union_type(field_type)
-            and self.__class__.doc_type.__fields__[field].required
+            and self.__class__.doc_type._docarray_fields[field].required
             and isinstance(field_type, type)
             and issubclass(field_type, BaseDoc)
         ):

diff --git a/docarray/array/doc_vec/doc_vec.py b/docarray/array/doc_vec/doc_vec.py
@@ -188,12 +188,12 @@ def __init__(
             else DocList.__class_getitem__(self.doc_type)(docs)
         )
 
-        for field_name, field in self.doc_type.__fields__.items():
+        for field_name, field in self.doc_type._docarray_fields.items():
             # here we iterate over the field of the docs schema, and we collect the data
             # from each document and put them in the corresponding column
             field_type = self.doc_type._get_field_type(field_name)
 
-            is_field_required = self.doc_type.__fields__[field_name].required
+            is_field_required = self.doc_type._docarray_fields[field_name].required
 
             first_doc_is_none = getattr(docs[0], field_name) is None
 
@@ -538,7 +538,7 @@ def _set_data_column(
                 if col is not None:
                     validation_class = col.__unparametrizedcls__ or col.__class__
                 else:
-                    validation_class = self.doc_type.__fields__[field].type_
+                    validation_class = self.doc_type._docarray_fields[field].type_
 
                 # TODO shape check should be handle by the tensor validation
 

diff --git a/docarray/base_doc/doc.py b/docarray/base_doc/doc.py
@@ -162,7 +162,7 @@ def is_view(self) -> bool:
         return isinstance(self.__dict__, ColumnStorageView)
 
     def __getattr__(self, item) -> Any:
-        if item in self.__fields__.keys():
+        if item in self._docarray_fields.keys():
             return self.__dict__[item]
         else:
             return super().__getattribute__(item)
@@ -184,10 +184,10 @@ def __eq__(self, other) -> bool:
         if not isinstance(other, BaseDoc):
             return False
 
-        if self.__fields__.keys() != other.__fields__.keys():
+        if self._docarray_fields.keys() != other._docarray_fields.keys():
             return False
 
-        for field_name in self.__fields__:
+        for field_name in self._docarray_fields:
             value1 = getattr(self, field_name)
             value2 = getattr(other, field_name)
 
@@ -363,7 +363,7 @@ def _exclude_doclist(
         self, exclude: ExcludeType
     ) -> Tuple[ExcludeType, ExcludeType, List[str]]:
         doclist_exclude_fields = []
-        for field in self.__fields__.keys():
+        for field in self._docarray_fields.keys():
             from docarray import DocList
 
             type_ = self._get_field_type(field)

diff --git a/docarray/base_doc/mixins/io.py b/docarray/base_doc/mixins/io.py
@@ -125,7 +125,7 @@ class IOMixin(Iterable[Tuple[str, Any]]):
     IOMixin to define all the bytes/protobuf/json related part of BaseDoc
     """
 
-    __fields__: Dict[str, 'ModelField']
+    _docarray_fields: Dict[str, 'ModelField']
 
     class Config:
         _load_extra_fields_from_protobuf: bool
@@ -235,7 +235,7 @@ def from_protobuf(cls: Type[T], pb_msg: 'DocProto') -> T:
         for field_name in pb_msg.data:
             if (
                 not (cls.Config._load_extra_fields_from_protobuf)
-                and field_name not in cls.__fields__.keys()
+                and field_name not in cls._docarray_fields.keys()
             ):
                 continue  # optimization we don't even load the data if the key does not
                 # match any field in the cls or in the mapping
@@ -311,8 +311,8 @@ def _get_content_from_node_proto(
 
             elif content_key in arg_to_container.keys():
                 field_type = (
-                    cls.__fields__[field_name].type_
-                    if field_name and field_name in cls.__fields__
+                    cls._docarray_fields[field_name].type_
+                    if field_name and field_name in cls._docarray_fields
                     else None
                 )
                 return_field = arg_to_container[content_key](
@@ -323,8 +323,8 @@ def _get_content_from_node_proto(
             elif content_key == 'dict':
                 deser_dict: Dict[str, Any] = dict()
                 field_type = (
-                    cls.__fields__[field_name].type_
-                    if field_name and field_name in cls.__fields__
+                    cls._docarray_fields[field_name].type_
+                    if field_name and field_name in cls._docarray_fields
                     else None
                 )
                 for key_name, node in value.dict.data.items():
@@ -393,7 +393,7 @@ def _get_access_paths(cls) -> List[str]:
         from docarray import BaseDoc
 
         paths = []
-        for field in cls.__fields__.keys():
+        for field in cls._docarray_fields.keys():
             field_type = cls._get_field_type(field)
             if not is_union_type(field_type) and safe_issubclass(field_type, BaseDoc):
                 sub_paths = field_type._get_access_paths()

diff --git a/docarray/base_doc/mixins/update.py b/docarray/base_doc/mixins/update.py
@@ -10,7 +10,7 @@
 
 
 class UpdateMixin:
-    __fields__: Dict[str, 'ModelField']
+    _docarray_fields: Dict[str, 'ModelField']
 
     def _get_string_for_regex_filter(self):
         return str(self)
@@ -104,7 +104,7 @@ def _group_fields(doc: 'UpdateMixin') -> _FieldGroups:
             nested_docs_fields: List[str] = []
             nested_docarray_fields: List[str] = []
 
-            for field_name, field in doc.__fields__.items():
+            for field_name, field in doc._docarray_fields.items():
                 if field_name not in FORBIDDEN_FIELDS_TO_UPDATE:
                     field_type = doc._get_field_type(field_name)
 

diff --git a/docarray/display/document_summary.py b/docarray/display/document_summary.py
@@ -61,7 +61,7 @@ def _get_schema(cls: Type['BaseDoc'], doc_name: Optional[str] = None) -> Tree:
         root = cls.__name__ if doc_name is None else f'{doc_name}: {cls.__name__}'
         tree = Tree(root, highlight=True)
 
-        for field_name, value in cls.__fields__.items():
+        for field_name, value in cls._docarray_fields.items():
             if field_name != 'id':
                 field_type = value.annotation
                 field_cls = str(field_type).replace('[', '\[')

diff --git a/docarray/helper.py b/docarray/helper.py
@@ -140,7 +140,7 @@ def _get_field_type_by_access_path(
     from docarray import BaseDoc, DocList
 
     field, _, remaining = access_path.partition('__')
-    field_valid = field in doc_type.__fields__.keys()
+    field_valid = field in doc_type._docarray_fields.keys()
 
     if field_valid:
         if len(remaining) == 0:
@@ -249,7 +249,7 @@ def _shallow_copy_doc(doc):
     field_set = set(doc.__fields_set__)
     object.__setattr__(shallow_copy, '__fields_set__', field_set)
 
-    for field_name, field_ in doc.__fields__.items():
+    for field_name, field_ in doc._docarray_fields.items():
         val = doc.__getattr__(field_name)
         setattr(shallow_copy, field_name, val)
 

diff --git a/docarray/index/abstract.py b/docarray/index/abstract.py
@@ -832,7 +832,7 @@ def _flatten_schema(
         :return: A list of column names, types, and fields
         """
         names_types_fields: List[Tuple[str, Type, 'ModelField']] = []
-        for field_name, field_ in schema.__fields__.items():
+        for field_name, field_ in schema._docarray_fields.items():
             t_ = schema._get_field_type(field_name)
             inner_prefix = name_prefix + field_name + '__'
 
@@ -1041,7 +1041,7 @@ def _convert_dict_to_doc(
         :param schema: The schema of the Document object
         :return: A Document object
         """
-        for field_name, _ in schema.__fields__.items():
+        for field_name, _ in schema._docarray_fields.items():
             t_ = schema._get_field_type(field_name)
 
             if not is_union_type(t_) and issubclass(t_, AnyDocArray):

diff --git a/docarray/store/jac.py b/docarray/store/jac.py
@@ -65,7 +65,7 @@ def _get_raw_summary(self: 'DocList') -> List[Dict[str, Any]]:
         ),
         dict(
             name='Fields',
-            value=tuple(self[0].__class__.__fields__.keys()),
+            value=tuple(self[0].__class__._docarray_fields.keys()),
             description='The fields of the Document',
         ),
         dict(

diff --git a/tests/index/base_classes/test_base_doc_store.py b/tests/index/base_classes/test_base_doc_store.py
@@ -118,21 +118,21 @@ def test_parametrization():
 
     index = DummyDocIndex[SubindexDoc]()
     assert index._schema is SubindexDoc
-    assert list(index._subindices['d']._schema.__fields__.keys()) == [
+    assert list(index._subindices['d']._schema._docarray_fields.keys()) == [
         'id',
         'tens',
         'parent_id',
     ]
 
     index = DummyDocIndex[SubSubindexDoc]()
     assert index._schema is SubSubindexDoc
-    assert list(index._subindices['d_root']._schema.__fields__.keys()) == [
+    assert list(index._subindices['d_root']._schema._docarray_fields.keys()) == [
         'id',
         'd',
         'parent_id',
     ]
     assert list(
-        index._subindices['d_root']._subindices['d']._schema.__fields__.keys()
+        index._subindices['d_root']._subindices['d']._schema._docarray_fields.keys()
     ) == [
         'id',
         'tens',
@@ -306,33 +306,33 @@ def test_create_columns():
 
 def test_flatten_schema():
     index = DummyDocIndex[SimpleDoc]()
-    fields = SimpleDoc.__fields__
+    fields = SimpleDoc._docarray_fields
     assert set(index._flatten_schema(SimpleDoc)) == {
         ('id', ID, fields['id']),
         ('tens', AbstractTensor, fields['tens']),
     }
 
     index = DummyDocIndex[FlatDoc]()
-    fields = FlatDoc.__fields__
+    fields = FlatDoc._docarray_fields
     assert set(index._flatten_schema(FlatDoc)) == {
         ('id', ID, fields['id']),
         ('tens_one', AbstractTensor, fields['tens_one']),
         ('tens_two', AbstractTensor, fields['tens_two']),
     }
 
     index = DummyDocIndex[NestedDoc]()
-    fields = NestedDoc.__fields__
-    fields_nested = SimpleDoc.__fields__
+    fields = NestedDoc._docarray_fields
+    fields_nested = SimpleDoc._docarray_fields
     assert set(index._flatten_schema(NestedDoc)) == {
         ('id', ID, fields['id']),
         ('d__id', ID, fields_nested['id']),
         ('d__tens', AbstractTensor, fields_nested['tens']),
     }
 
     index = DummyDocIndex[DeepNestedDoc]()
-    fields = DeepNestedDoc.__fields__
-    fields_nested = NestedDoc.__fields__
-    fields_nested_nested = SimpleDoc.__fields__
+    fields = DeepNestedDoc._docarray_fields
+    fields_nested = NestedDoc._docarray_fields
+    fields_nested_nested = SimpleDoc._docarray_fields
     assert set(index._flatten_schema(DeepNestedDoc)) == {
         ('id', ID, fields['id']),
         ('d__id', ID, fields_nested['id']),
@@ -341,7 +341,7 @@ def test_flatten_schema():
     }
 
     index = DummyDocIndex[SubindexDoc]()
-    fields = SubindexDoc.__fields__
+    fields = SubindexDoc._docarray_fields
     assert set(index._flatten_schema(SubindexDoc)) == {
         ('id', ID, fields['id']),
         ('d', DocList[SimpleDoc], fields['d']),
@@ -360,7 +360,7 @@ def test_flatten_schema():
     ] == [ID, AbstractTensor, ID]
 
     index = DummyDocIndex[SubSubindexDoc]()
-    fields = SubSubindexDoc.__fields__
+    fields = SubSubindexDoc._docarray_fields
     assert set(index._flatten_schema(SubSubindexDoc)) == {
         ('id', ID, fields['id']),
         ('d_root', DocList[SubindexDoc], fields['d_root']),
@@ -384,8 +384,8 @@ class MyDoc(BaseDoc):
         image: ImageDoc
 
     index = DummyDocIndex[MyDoc]()
-    fields = MyDoc.__fields__
-    fields_image = ImageDoc.__fields__
+    fields = MyDoc._docarray_fields
+    fields_image = ImageDoc._docarray_fields
 
     if torch_imported:
         from docarray.typing.tensor.image.image_torch_tensor import ImageTorchTensor
@@ -409,7 +409,7 @@ class MyDoc3(BaseDoc):
         tensor: Union[NdArray, ImageTorchTensor]
 
     index = DummyDocIndex[MyDoc3]()
-    fields = MyDoc3.__fields__
+    fields = MyDoc3._docarray_fields
     assert set(index._flatten_schema(MyDoc3)) == {
         ('id', ID, fields['id']),
         ('tensor', AbstractTensor, fields['tensor']),