Embedded objects (#329)

* Allow embedded objects to be submitted
* Fix typos

Author: GavinMendelGleason

terminusdb_client/client/Client.py

@@ -193,6 +193,7 @@ def __init__(
         self._branch = None
         self._ref = None
         self._repo = None
+        self._references = {}
 
         # Default headers
         self._default_headers = {"user-agent": user_agent}
@@ -1140,7 +1141,10 @@ def _conv_to_dict(self, obj):
             if hasattr(obj, "_isinstance") and obj._isinstance:
                 if hasattr(obj.__class__, "_subdocument"):
                     raise ValueError("Subdocument cannot be added directly")
-                return obj._obj_to_dict()
+                (d, refs) = obj._obj_to_dict()
+                # merge all refs
+                self._references = {**self._references, **refs}
+                return d
             else:
                 return obj._to_dict()
         else:
@@ -1157,45 +1161,44 @@ def _ref_extract(self, target_key, search_item):
                     for item in value:
                         yield from self._ref_extract(target_key, item)
 
+    def _unseen(self, seen):
+        unseen = []
+        for key in self._references:
+            if key not in seen:
+                unseen.append(self._references[key])
+        return unseen
+
     def _convert_document(self, document, graph_type):
-        if isinstance(document, list):
-            new_doc = []
-            captured = []
-            referenced = []
+        if not isinstance(document, list):
+            document = [document]
 
+        seen = {}
+        objects = []
+        while document != []:
             for item in document:
+                if hasattr(item, "to_dict") and graph_type != "schema":
+                    raise InterfaceError(
+                        "Inserting WOQLSchema object into non-schema graph."
+                    )
                 item_dict = self._conv_to_dict(item)
-                new_doc.append(item_dict)
-                item_capture = item_dict.get("@capture")
-                if item_capture:
-                    captured.append(item_capture)
-                referenced += list(self._ref_extract("@ref", item_dict))
+                if hasattr(item, "_capture"):
+                    seen[item._capture] = item_dict
+                else:
+                    if isinstance(item_dict, list):
+                        objects += item_dict
+                    else:
+                        objects.append(item_dict)
 
-            referenced = list(set(referenced))
+            document = self._unseen(seen)
 
-            for item in referenced:
-                if item not in captured:
-                    raise ValueError(
-                        f"{item} is referenced but not captured. Seems you forgot to submit one or more object(s)."
-                    )
-        else:
-            if hasattr(document, "to_dict") and graph_type != "schema":
-                raise InterfaceError(
-                    "Inserting WOQLSchema object into non-schema graph."
-                )
-            new_doc = self._conv_to_dict(document)
-            if isinstance(new_doc, dict) and list(self._ref_extract("@ref", new_doc)):
-                raise ValueError(
-                    "There are uncaptured references. Seems you forgot to submit one or more object(s)."
-                )
-        return new_doc
+        return list(seen.values()) + objects
 
     def insert_document(
         self,
         document: Union[
             dict,
             List[dict],
-            "WOQLSchema",  # noqa:F821
+            "Schema",  # noqa:F821
             "DocumentTemplate",  # noqa:F821
             List["DocumentTemplate"],  # noqa:F821
         ],
@@ -1249,12 +1252,14 @@ def insert_document(
         if last_data_version is not None:
             headers["TerminusDB-Data-Version"] = last_data_version
 
+        # make sure we track only internal references
+        self._references = {}
         new_doc = self._convert_document(document, graph_type)
+        all_docs = list(self._references.values())
+        self._references = {}
 
         if len(new_doc) == 0:
             return
-        elif not isinstance(new_doc, list):
-            new_doc = [new_doc]
 
         if full_replace:
             if new_doc[0].get("@type") != "@context":
@@ -1289,18 +1294,18 @@ def insert_document(
                 auth=self._auth(),
             )
         result = json.loads(_finish_response(result))
-        if isinstance(document, list):
-            for idx, item in enumerate(document):
+        if isinstance(all_docs, list):
+            for idx, item in enumerate(all_docs):
                 if hasattr(item, "_obj_to_dict") and not hasattr(item, "_backend_id"):
-                    item._backend_id = result[idx][len("terminusdb:///data/") :]
+                    item._backend_id = result[idx]
         return result
 
     def replace_document(
         self,
         document: Union[
             dict,
             List[dict],
-            "WOQLSchema",  # noqa:F821
+            "Schema",  # noqa:F821
             "DocumentTemplate",  # noqa:F821
             List["DocumentTemplate"],  # noqa:F821
         ],
@@ -1346,7 +1351,10 @@ def replace_document(
         if last_data_version is not None:
             headers["TerminusDB-Data-Version"] = last_data_version
 
+        self._references = {}
         new_doc = self._convert_document(document, graph_type)
+        all_docs = list(self._references.values())
+        self._references = {}
 
         json_string = json.dumps(new_doc).encode("utf-8")
         if compress != "never" and len(json_string) > compress:
@@ -1369,8 +1377,8 @@ def replace_document(
                 auth=self._auth(),
             )
         result = json.loads(_finish_response(result))
-        if isinstance(document, list):
-            for idx, item in enumerate(document):
+        if isinstance(all_docs, list):
+            for idx, item in enumerate(all_docs):
                 if hasattr(item, "_obj_to_dict") and not hasattr(item, "_backend_id"):
                     item._backend_id = result[idx][len("terminusdb:///data/") :]
         return result
@@ -1380,7 +1388,7 @@ def update_document(
         document: Union[
             dict,
             List[dict],
-            "WOQLSchema",  # noqa:F821
+            "Schema",  # noqa:F821
             "DocumentTemplate",  # noqa:F821
             List["DocumentTemplate"],  # noqa:F821
         ],
@@ -1449,7 +1457,7 @@ def delete_document(
             document = [document]
         for doc in document:
             if hasattr(doc, "_obj_to_dict"):
-                doc = doc._obj_to_dict()
+                (doc, refs) = doc._obj_to_dict()
             if isinstance(doc, dict) and doc.get("@id"):
                 doc_id.append(doc.get("@id"))
             elif isinstance(doc, str):
@@ -1996,7 +2004,7 @@ def squash(
             self.reset(commit_id)
         return commit_id
 
-    def _convert_diff_dcoument(self, document):
+    def _convert_diff_document(self, document):
         if isinstance(document, list):
             new_doc = []
             for item in document:
@@ -2012,15 +2020,15 @@ def diff(
             str,
             dict,
             List[dict],
-            "WOQLSchema",  # noqa:F821
+            "Schema",  # noqa:F821
             "DocumentTemplate",  # noqa:F821
             List["DocumentTemplate"],  # noqa:F821
         ],
         after: Union[
             str,
             dict,
             List[dict],
-            "WOQLSchema",  # noqa:F821
+            "Schema",  # noqa:F821
             "DocumentTemplate",  # noqa:F821
             List["DocumentTemplate"],  # noqa:F821
         ],
@@ -2047,7 +2055,7 @@ def diff(
             if isinstance(item, str):
                 request_dict[f"{key}_data_version"] = item
             else:
-                request_dict[key] = self._convert_diff_dcoument(item)
+                request_dict[key] = self._convert_diff_document(item)
         if document_id is not None:
             if "before_data_version" in request_dict:
                 if document_id[: len("terminusdb:///data")] == "terminusdb:///data":
@@ -2084,7 +2092,7 @@ def patch(
         before: Union[
             dict,
             List[dict],
-            "WOQLSchema",  # noqa:F821
+            "Schema",  # noqa:F821
             "DocumentTemplate",  # noqa:F821
             List["DocumentTemplate"],  # noqa:F821
         ],
@@ -2109,7 +2117,7 @@ def patch(
         '{ "@id" : "Person/Jane", "@type" : Person", "name" : "Janine"}'"""
 
         request_dict = {
-            "before": self._convert_diff_dcoument(before),
+            "before": self._convert_diff_document(before),
             "patch": patch.content,
         }
 

terminusdb_client/schema/schema.py

@@ -263,7 +263,7 @@ def _id(self, custom_id):
                 f"Customized id is not allowed. {str(self.__class__)} is a subdocument or has set id key scheme."
             )
 
-    def _embeded_rep(self):
+    def _embedded_rep(self):
         """get representation for embedding as object property"""
         if hasattr(self.__class__, "_subdocument"):
             return self._obj_to_dict()
@@ -280,23 +280,40 @@ def _obj_to_dict(self, skip_checking=False):
             result["@id"] = self._id
         elif not hasattr(self, "_subdocument"):
             result["@capture"] = self._capture
-        # elif hasattr(self.__class__, "_key") and hasattr(self.__class__._key, "idgen"):
-        #     result["@id"] = self.__class__._key.idgen(self)
 
+        references = {}
         for item in self._annotations.keys():
             if hasattr(self, item):
                 the_item = eval(f"self.{item}")  # noqa: S307
                 if the_item is not None:
                     # object properties
-                    if hasattr(the_item, "_embeded_rep"):
-                        result[item] = the_item._embeded_rep()
+                    if hasattr(the_item, "_embedded_rep"):
+                        ref_obj = the_item._embedded_rep()
+                        if '@ref' in ref_obj:
+                            references[ref_obj['@ref']] = the_item
+                        elif '@id' in ref_obj:
+                            pass
+                        else:
+                            (sub_item, refs) = ref_obj
+                            references = {**references, **refs}
+                            ref_obj = sub_item
+                        result[item] = ref_obj
                     # handle list and set (set end up passing as list for jsonlize)
                     elif isinstance(the_item, (list, set)):
                         new_item = []
                         for sub_item in the_item:
                             # inner is object properties
-                            if hasattr(sub_item, "_embeded_rep"):
-                                new_item.append(sub_item._embeded_rep())
+                            if hasattr(sub_item, "_embedded_rep"):
+                                ref_obj = sub_item._embedded_rep()
+                                if '@ref' in ref_obj:
+                                    references[ref_obj['@ref']] = sub_item
+                                elif '@id' in ref_obj:
+                                    pass
+                                else:
+                                    (sub_item, refs) = ref_obj
+                                    references = {**references, **refs}
+                                    ref_obj = sub_item
+                                new_item.append(ref_obj)
                             # inner is Enum
                             elif isinstance(sub_item, Enum):
                                 new_item.append(str(sub_item))
@@ -310,7 +327,7 @@ def _obj_to_dict(self, skip_checking=False):
                             result[item] = str(the_item)
                         else:
                             result[item] = wt.datetime_to_woql(the_item)
-        return result
+        return (result, references)
 
 
 class EnumMetaTemplate(EnumMeta):

terminusdb_client/tests/integration_tests/test_schema.py

@@ -88,25 +88,11 @@ def test_insert_cheuk(docker_url, test_schema):
 
     client = Client(docker_url, user_agent=test_user_agent)
     client.connect(db="test_docapi")
-    # client.create_database("test_docapi")
-    # print(cheuk._obj_to_dict())
     with pytest.raises(ValueError) as error:
         client.insert_document(home)
         assert str(error.value) == "Subdocument cannot be added directly"
-    with pytest.raises(ValueError) as error:
-        client.insert_document([cheuk])
-        assert (
-            str(error.value)
-            == f"{uk._capture} is referenced but not captured. Seems you forgot to submit one or more object(s)."
-        )
-    with pytest.raises(ValueError) as error:
-        client.insert_document(cheuk)
-        assert (
-            str(error.value)
-            == "There are uncaptured references. Seems you forgot to submit one or more object(s)."
-        )
     assert cheuk._id is None and uk._id is None
-    client.insert_document([uk, cheuk], commit_msg="Adding cheuk")
+    client.insert_document([cheuk], commit_msg="Adding cheuk")
     assert cheuk._backend_id and cheuk._id
     assert uk._backend_id and uk._id
     result = client.get_all_documents()
@@ -134,7 +120,7 @@ def test_getting_and_deleting_cheuk(docker_url):
     cheuk = new_schema.import_objects(
         client.get_documents_by_type("Employee", as_list=True)
     )[0]
-    result = cheuk._obj_to_dict()
+    result = cheuk._obj_to_dict()[0]
     assert result["address_of"]["postal_code"] == "A12 345"
     assert result["address_of"]["street"] == "123 Abc Street"
     assert result["name"] == "Cheuk"
@@ -183,25 +169,13 @@ def test_insert_cheuk_again(docker_url, test_schema):
     cheuk.member_of = Team.information_technology
     cheuk._id = "Cheuk is back"
 
-    with pytest.raises(ValueError) as error:
-        client.update_document([uk])
-        assert (
-            str(error.value)
-            == f"{location._capture} is referenced but not captured. Seems you forgot to submit one or more object(s)."
-        )
-    with pytest.raises(ValueError) as error:
-        client.insert_document(uk)
-        assert (
-            str(error.value)
-            == "There are uncaptured references. Seems you forgot to submit one or more object(s)."
-        )
-
     client.update_document([location, uk, cheuk], commit_msg="Adding cheuk again")
     assert location._backend_id and location._id
     location.x = -0.7
     result = client.replace_document([location], commit_msg="Fixing location")
     assert len(result) == 1
     result = client.get_all_documents()
+
     for item in result:
         if item.get("@type") == "Country":
             assert item["name"] == "United Kingdom"