Merge pull request #463 from terminusdb/collect

Add WOQL collect predicate

Author: hoijnet

.github/workflows/python.yml

@@ -70,6 +70,3 @@ jobs:
 
     - name: Deploy to pypi
       uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        user: __token__
-        password: ${{ secrets.PYPI_API_TOKEN }}

README.md

@@ -8,8 +8,8 @@
 [![Reddit](https://img.shields.io/reddit/subreddit-subscribers/TerminusDB?style=social)](https://www.reddit.com/r/TerminusDB/)
 [![Twitter](https://img.shields.io/twitter/follow/terminusdb?color=skyblue&label=Follow%20on%20Twitter&logo=twitter&style=flat)](https://twitter.com/TerminusDB)
 
-[![release version](https://img.shields.io/pypi/v/terminusdb-client.svg?logo=pypi)](https://pypi.python.org/pypi/terminusdb-client/)
-[![downloads](https://img.shields.io/pypi/dm/terminusdb-client.svg?logo=pypi)](https://pypi.python.org/pypi/terminusdb-client/)
+[![release version](https://img.shields.io/pypi/v/terminusdb.svg?logo=pypi)](https://pypi.python.org/pypi/terminusdb/)
+[![downloads](https://img.shields.io/pypi/dm/terminusdb.svg?logo=pypi)](https://pypi.python.org/pypi/terminusdb/)
 
 [![build status](https://img.shields.io/github/workflow/status/terminusdb/terminusdb-client-python/Python%20package?logo=github)](https://github.com/terminusdb/terminusdb-client-python/actions)
 [![documentation](https://img.shields.io/github/deployments/terminusdb/terminusdb-client-python/github-pages?label=documentation&logo=github)](https://terminusdb.org/docs/python)
@@ -18,6 +18,10 @@
 
 > Python client for TerminusDB and TerminusCMS.
 
+> **Migrating from `terminusdb-client`?** This package was formerly known as
+> `terminusdb-client`. Simply install `terminusdb` instead — both `import terminusdb`
+> and `import terminusdb_client` continue to work, so no code changes are required.
+
 [**TerminusDB**][terminusdb] is an [open-source][terminusdb-repo] graph database
 and document store. It allows you to link JSON documents in a powerful knowledge
 graph all through a simple document API, with full git-for-data version control.
@@ -28,26 +32,26 @@ graph all through a simple document API, with full git-for-data version control.
 
 ## Requirements
 
-- [TerminusDB v11.1](https://github.com/terminusdb/terminusdb-server)
+- [TerminusDB v12](https://github.com/terminusdb/terminusdb-server)
 - [Python >=3.9](https://www.python.org/downloads)
 
 ## Release Notes and Previous Versions
 
-TerminusDB Client v11.1 works with TerminusDB v11.1 and the [DFRNT cloud service](https://dfrnt.com). Please check the [Release Notes](RELEASE_NOTES.md) to find out what has changed.
+TerminusDB Client v12 works with TerminusDB v12 onwards and the [DFRNT cloud service](https://dfrnt.com). Please check the [Release Notes](RELEASE_NOTES.md) to find out what has changed.
 
 ## Installation
 -  TerminusDB Client can be downloaded from PyPI using pip:
-`python -m pip install terminusdb-client`
+`python -m pip install terminusdb`
 
 This only includes the core Python Client (Client) and WOQLQuery.
 
 If you want to use woqlDataframe or the import and export CSV function in the Scaffolding CLI tool:
 
-`python -m pip install terminusdb-client[dataframe]`
+`python -m pip install terminusdb[dataframe]`
 
 *if you are installing from `zsh` you have to quote the argument like this:*
 
-`python -m pip install 'terminusdb-client[dataframe]'`
+`python -m pip install 'terminusdb[dataframe]'`
 
 - Install from source:
 
@@ -66,19 +70,21 @@ If you want to use woqlDataframe or the import and export CSV function in the Sc
 Connect to local host
 
 ```Python
-from terminusdb_client import Client
+from terminusdb import Client
 
 client = Client("http://127.0.0.1:6363/")
 client.connect()
 ```
 
+The previous import path `from terminusdb_client import Client` also continues to work.
+
 Connect to TerminusDB in the cloud
 
 *check the documentation on the DFRNT support page about how to add your [API token](https://support.dfrnt.com/portal/en/kb/articles/api) to the environment variable*
 
 
 ```Python
-from terminusdb_client import Client
+from terminusdb import Client
 
 team="MyTeam"
 client = Client(f"https://studio.dfrnt.com/api/hosted/{team}/")
@@ -94,7 +100,7 @@ client.create_database("MyDatabase")
 #### Create a schema
 
 ```Python
-from terminusdb_client.schema import Schema, DocumentTemplate, RandomKey
+from terminusdb.schema import Schema, DocumentTemplate, RandomKey
 
 my_schema = Schema()
 

pyproject.toml

@@ -1,10 +1,14 @@
 [tool.poetry]
-name = "terminusdb-client"
+name = "terminusdb"
 version = "12.0.3"
-description = "Python client for Terminus DB"
-authors = ["TerminusDB group"]
+description = "Terminus DB Python client"
+authors = ["TerminusDB group", "DFRNT AB"]
 license = "Apache Software License"
 readme = "README.md"
+packages = [
+    {include = "terminusdb_client"},
+    {include = "terminusdb"},
+]
 
 [tool.poetry.dependencies]
 python = ">=3.9.0,<3.13"

terminusdb/__init__.py

@@ -0,0 +1,4 @@
+# Re-export everything from terminusdb_client for the new import path.
+# Both `import terminusdb` and `import terminusdb_client` are supported.
+from terminusdb_client import *  # noqa
+from terminusdb_client import Client, WOQLClient, WOQLQuery, Var, Vars, Patch, GraphType, WOQLDataFrame, WOQLSchema  # noqa

terminusdb/client/__init__.py

@@ -0,0 +1,2 @@
+from terminusdb_client.client import *  # noqa
+from terminusdb_client.client import Client, GraphType, Patch  # noqa

terminusdb/query_syntax/__init__.py

@@ -0,0 +1 @@
+from terminusdb_client.query_syntax import *  # noqa

terminusdb/schema/__init__.py

@@ -0,0 +1 @@
+from terminusdb_client.schema import *  # noqa

terminusdb_client/scripts/scripts.py

@@ -486,7 +486,6 @@ def importcsv(
         embedded = [x.lower().replace(" ", "_") for x in embedded]
     try:
         pd = import_module("pandas")
-        np = import_module("numpy")
     except ImportError:
         raise ImportError(
             "Library 'pandas' is required to import csv, either install 'pandas' or install woqlDataframe requirements as follows: python -m pip install -U terminus-client-python[dataframe]"
@@ -500,6 +499,48 @@ def importcsv(
     # "not schema" make it always False if adding the schema option
     has_schema = not schema and class_name in client.get_existing_classes()
 
+    def _df_to_schema(class_name, df):
+        class_dict = {"@type": "Class", "@id": class_name}
+        for col, dtype in dict(df.dtypes).items():
+            if embedded and col in embedded:
+                converted_type = class_name
+            else:
+                # Map pandas/numpy dtype to Python type
+                # Uses dtype.kind for compatibility with numpy 2.0+ and pandas 3.0+
+                dtype_kind = getattr(dtype, "kind", "O")
+                if dtype.type is str or dtype_kind in ("U", "O", "S", "T"):
+                    converted_type = str
+                elif dtype_kind in ("i", "u"):
+                    converted_type = int
+                elif dtype_kind == "f":
+                    converted_type = float
+                elif dtype_kind == "b":
+                    converted_type = bool
+                elif dtype_kind == "M":
+                    converted_type = dt.datetime
+                elif dtype_kind == "m":
+                    converted_type = dt.timedelta
+                else:
+                    converted_type = str
+                converted_type = wt.to_woql_type(converted_type)
+
+            if id_ and col == id_:
+                class_dict[col] = converted_type
+            elif na == "optional" and col not in keys:
+                class_dict[col] = {"@type": "Optional", "@class": converted_type}
+            else:
+                class_dict[col] = converted_type
+        # if id_ is not None:
+        #     pass  # don't need key if id is specified
+        # elif keys:
+        #     class_dict["@key"] = {"@type": "Random"}
+        # elif na == "optional":
+        #     # have to use random key cause keys will be optional
+        #     class_dict["@key"] = {"@type": "Random"}
+        # else:
+        #     class_dict["@key"] = {"@type": "Random"}
+        return class_dict
+
     with pd.read_csv(csv_file, sep=sep, chunksize=chunksize, dtype=dtype) as reader:
         for df in tqdm(reader):
             if any(df.isna().any()) and na == "error":
@@ -512,15 +553,7 @@ def importcsv(
                 converted_col = col.lower().replace(" ", "_").replace(".", "_")
                 df.rename(columns={col: converted_col}, inplace=True)
             if not has_schema:
-                class_dict = _df_to_schema(
-                    class_name,
-                    df,
-                    np,
-                    embedded=embedded,
-                    id_col=id_,
-                    na_mode=na,
-                    keys=keys,
-                )
+                class_dict = _df_to_schema(class_name, df)
                 if message is None:
                     schema_msg = f"Schema object insert/ update with {csv_file} by Python client."
                 else:

terminusdb_client/tests/integration_tests/test_conftest.py

@@ -14,20 +14,18 @@ class TestServerDetection:
     """Test server detection helper functions"""
 
     @patch("terminusdb_client.tests.integration_tests.conftest.requests.get")
-    def test_local_server_running_200(self, mock_get):
-        """Test local server detection returns True for HTTP 200"""
-        mock_response = Mock()
-        mock_response.status_code = 200
-        mock_get.return_value = mock_response
+    def test_local_server_running_any_response(self, mock_get):
+        """Test local server detection returns True for any HTTP response"""
+        mock_get.return_value = Mock()
 
         assert is_local_server_running() is True
         mock_get.assert_called_once_with("http://127.0.0.1:6363/api/", timeout=2)
 
     @patch("terminusdb_client.tests.integration_tests.conftest.requests.get")
-    def test_local_server_running_404(self, mock_get):
-        """Test local server detection returns True for HTTP 404"""
+    def test_local_server_running_401(self, mock_get):
+        """Test local server detection returns True for HTTP 401 (unauthorized)"""
         mock_response = Mock()
-        mock_response.status_code = 404
+        mock_response.status_code = 401
         mock_get.return_value = mock_response
 
         assert is_local_server_running() is True
@@ -47,20 +45,18 @@ def test_local_server_not_running_timeout(self, mock_get):
         assert is_local_server_running() is False
 
     @patch("terminusdb_client.tests.integration_tests.conftest.requests.get")
-    def test_docker_server_running_200(self, mock_get):
-        """Test Docker server detection returns True for HTTP 200"""
-        mock_response = Mock()
-        mock_response.status_code = 200
-        mock_get.return_value = mock_response
+    def test_docker_server_running_any_response(self, mock_get):
+        """Test Docker server detection returns True for any HTTP response"""
+        mock_get.return_value = Mock()
 
         assert is_docker_server_running() is True
         mock_get.assert_called_once_with("http://127.0.0.1:6366/api/", timeout=2)
 
     @patch("terminusdb_client.tests.integration_tests.conftest.requests.get")
-    def test_docker_server_running_404(self, mock_get):
-        """Test Docker server detection returns True for HTTP 404"""
+    def test_docker_server_running_401(self, mock_get):
+        """Test Docker server detection returns True for HTTP 401 (unauthorized)"""
         mock_response = Mock()
-        mock_response.status_code = 404
+        mock_response.status_code = 401
         mock_get.return_value = mock_response
 
         assert is_docker_server_running() is True
@@ -73,20 +69,18 @@ def test_docker_server_not_running(self, mock_get):
         assert is_docker_server_running() is False
 
     @patch("terminusdb_client.tests.integration_tests.conftest.requests.get")
-    def test_jwt_server_running_200(self, mock_get):
-        """Test JWT server detection returns True for HTTP 200"""
-        mock_response = Mock()
-        mock_response.status_code = 200
-        mock_get.return_value = mock_response
+    def test_jwt_server_running_any_response(self, mock_get):
+        """Test JWT server detection returns True for any HTTP response"""
+        mock_get.return_value = Mock()
 
         assert is_jwt_server_running() is True
         mock_get.assert_called_once_with("http://127.0.0.1:6367/api/", timeout=2)
 
     @patch("terminusdb_client.tests.integration_tests.conftest.requests.get")
-    def test_jwt_server_running_404(self, mock_get):
-        """Test JWT server detection returns True for HTTP 404"""
+    def test_jwt_server_running_401(self, mock_get):
+        """Test JWT server detection returns True for HTTP 401 (unauthorized)"""
         mock_response = Mock()
-        mock_response.status_code = 404
+        mock_response.status_code = 401
         mock_get.return_value = mock_response
 
         assert is_jwt_server_running() is True