framework/v37.4.2

Author: slrelease

framework/src/framework/common/storage/field/field_data.py

@@ -15,7 +15,7 @@
 
 from __future__ import annotations
 
-from beartype.typing import Generic, TypeVar, get_args
+from beartype.typing import Any, Generic, TypeVar, get_args
 
 from superlinked.framework.common.data_types import Vector
 from superlinked.framework.common.exception import InvalidInputException
@@ -36,22 +36,42 @@ def __init__(self, type_: FieldDataType, name: str, value: FT) -> None:
         self.__validate_value(type_, value)
         self.value = value
 
+    @classmethod
+    def from_field(cls, field: Field, value: FT) -> FieldData:
+        return cls(field.data_type, field.name, value)
+
     def __validate_value(self, data_type: FieldDataType, value: FT) -> None:
         valid_types = FieldTypeConverter.get_valid_node_data_types(data_type)
-        error_msg = "Invalid value {value} for the given field data type {data_type}"
         if not isinstance(value, tuple(valid_types)):
-            raise InvalidInputException(error_msg.format(value=value, data_type=data_type))
-        if isinstance(value, list):
-            # Assuming list types have only 1 valid type
-            valid_type = VALID_TYPE_BY_FIELD_DATA_TYPE[data_type][0]
-            generic_type = get_args(valid_type)[0]
-            # TODO FAB-3719 FieldData validation is slow for large float lists
-            if data_type != FieldDataType.FLOAT_LIST and not all(isinstance(item, generic_type) for item in value):
-                raise InvalidInputException(error_msg.format(value=value, data_type=data_type))
+            self.__raise_validation_exception(data_type, value)
+        self.__validate_list_type(data_type, value)
 
-    @classmethod
-    def from_field(cls, field: Field, value: FT) -> FieldData:
-        return cls(field.data_type, field.name, value)
+    def __validate_list_type(self, data_type: FieldDataType, value: FT) -> None:
+        if not isinstance(value, list):
+            return
+        if data_type == FieldDataType.FLOAT_LIST:
+            if self.__is_valid_float_list(value):
+                return
+            self.__raise_validation_exception(data_type, value)
+        valid_type = VALID_TYPE_BY_FIELD_DATA_TYPE[data_type][0]
+        generic_type = get_args(valid_type)[0]
+        if not all(isinstance(item, generic_type) for item in value):
+            self.__raise_validation_exception(data_type, value)
+
+    def __is_valid_float_list(self, value: list[Any]) -> bool:
+        """
+        This is a performance hot-spot. It needs to be fast as we use it
+        for custom space input validation where we can have long float lists
+        """
+        try:
+            for item in value:
+                float(item)
+            return True
+        except (ValueError, TypeError):
+            return False
+
+    def __raise_validation_exception(self, data_type: FieldDataType, value: Any) -> None:
+        raise InvalidInputException(f"Invalid value {value} for the given field data type {data_type}")
 
 
 class VectorFieldData(FieldData[Vector]):

framework/src/framework/dsl/storage/qdrant_vector_database.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 
 
+from collections.abc import Mapping
+
 from beartype.typing import Any
 
 from superlinked.framework.common.precision import Precision
@@ -40,11 +42,10 @@ def __init__(  # pylint: disable=too-many-arguments
         url: str,
         api_key: str,
         default_query_limit: int = 10,
-        timeout: int | None = None,
         search_algorithm: SearchAlgorithm = SearchAlgorithm.FLAT,
         vector_precision: Precision = Precision.FLOAT16,
-        prefer_grpc: bool | None = None,
-        **extra_params: Any
+        client_params: Mapping[str, Any] | None = None,
+        **extra_params: Any,
     ) -> None:
         """
         Initialize the QdrantVectorDatabase.
@@ -53,18 +54,20 @@ def __init__(  # pylint: disable=too-many-arguments
             url (str): The url of the Qdrant server.
             api_key (str): The api key of the Qdrant cluster.
             default_query_limit (int): Default vector search limit, set to Qdrant's default of 10.
-            timeout (int | None): Timeout in seconds for Qdrant operations. Default is 5 seconds.
             vector_precision (Precision): Precision to use for storing vectors. Defaults to FLOAT16.
-            prefer_grpc (bool | None): Whether to prefer gRPC for Qdrant operations. Default is False.
+            client_params (Mapping[str, Any] | None): Additional parameters for the QdrantClient.
+            These are passed directly to the QdrantClient constructor, so any valid QdrantClient
+            parameter can be used. For example `{"prefer_grpc": True}`. Defaults to None.
             **extra_params (Any): Additional parameters for the Qdrant connection.
         """
         super().__init__()
-        self._connection_params = QdrantConnectionParams(url, api_key, timeout, prefer_grpc, **extra_params)
+        self._connection_params = QdrantConnectionParams(url, api_key, **extra_params)
         self._settings = VDBSettings(
             default_query_limit=default_query_limit,
             search_algorithm=search_algorithm,
             vector_precision=vector_precision,
         )
+        self._client_params = client_params or {}
 
     @property
     def _vdb_connector(self) -> QdrantVDBConnector:
@@ -74,4 +77,4 @@ def _vdb_connector(self) -> QdrantVDBConnector:
         Returns:
             QdrantVDBConnector: The Qdrant vector database connector instance.
         """
-        return QdrantVDBConnector(self._connection_params, self._settings)
+        return QdrantVDBConnector(self._connection_params, self._settings, self._client_params)

framework/src/framework/storage/qdrant/qdrant_connection_params.py

@@ -19,23 +19,16 @@
 
 class QdrantConnectionParams(ConnectionParams):
     def __init__(
-        self, url: str, api_key: str, timeout: int | None = None, prefer_grpc: bool | None = None, **extra_params: Any
+        self,
+        url: str,
+        api_key: str,
+        **extra_params: Any,
     ) -> None:
         super().__init__()
         extra_params_str = self.get_uri_params_string(**extra_params)
         self._connection_string = f"{url}{extra_params_str}"
         self._api_key = api_key
-        self._timeout = timeout
-        self._prefer_grpc = prefer_grpc
 
     @property
     def connection_string(self) -> str:
         return self._connection_string
-
-    @property
-    def timeout(self) -> int | None:
-        return self._timeout
-
-    @property
-    def prefer_grpc(self) -> bool:
-        return bool(self._prefer_grpc)

framework/src/framework/storage/qdrant/qdrant_vdb_connector.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import re
+from collections.abc import Mapping
 
 from beartype.typing import Any, Sequence, cast
 from qdrant_client import QdrantClient
@@ -76,13 +77,18 @@
 
 
 class QdrantVDBConnector(VDBConnector[VDBKNNSearchConfig]):
-    def __init__(self, connection_params: QdrantConnectionParams, vdb_settings: VDBSettings) -> None:
+    def __init__(
+        self,
+        connection_params: QdrantConnectionParams,
+        vdb_settings: VDBSettings,
+        client_params: Mapping[str, Any] | None = None,
+    ) -> None:
         super().__init__(vdb_settings=vdb_settings)
+
         self._client = QdrantClient(
             url=connection_params.connection_string,
             api_key=connection_params._api_key,
-            timeout=connection_params.timeout,
-            prefer_grpc=connection_params.prefer_grpc,
+            **client_params or {},
         )
         self._encoder = QdrantFieldEncoder(self.vector_precision)
         self.__search_index_manager = QdrantSearchIndexManager(self._client)