ADD get_flow, serialization, deserialization of flow

Author: mfeurer

openml/__init__.py

@@ -19,12 +19,13 @@
 from .datasets import OpenMLDataset
 from . import datasets
 from . import runs
+from . import flows
 from .runs import OpenMLRun
 from .tasks import OpenMLTask, OpenMLSplit
 from .flows import OpenMLFlow
 
 
 __version__ = "0.2.1"
 
-__all__ = ['OpenMLDataset', 'OpenMLRun', 'OpenMLSplit',
-           'datasets', 'OpenMLTask', 'OpenMLFlow', 'config', 'runs']
+__all__ = ['OpenMLDataset', 'OpenMLRun', 'OpenMLSplit', 'datasets',
+           'OpenMLTask', 'OpenMLFlow', 'config', 'runs', 'flows']

openml/flows/__init__.py

@@ -1,3 +1,4 @@
 from .flow import OpenMLFlow, create_flow_from_model
+from .functions import get_flow
 
-__all__ = ['OpenMLFlow', 'create_flow_from_model']
+__all__ = ['OpenMLFlow', 'create_flow_from_model', 'get_flow']

openml/flows/flow.py

@@ -2,7 +2,6 @@
 import xmltodict
 
 from .._api_calls import _perform_api_call
-from .functions import _check_flow_exists
 
 
 class OpenMLFlow(object):
@@ -14,8 +13,6 @@ class OpenMLFlow(object):
         The model the flow consists of. The model needs to have fit and predict methods.
     description : string
         Description of the flow (free text).
-    creator : string
-        FIXME
     contributor : string
         FIXME
     tag : string
@@ -28,64 +25,221 @@ class OpenMLFlow(object):
 
     """
     def __init__(self, name, description=None, model=None, components=None,
-                 parameters=None, external_version=None, creator=None,
-                 uploader=None, tag=None, flow_id=None):
+                 parameters=None, parameters_meta_info=None,
+                 external_version=None, uploader=None, tags=None,
+                 binary_url=None, binary_format=None, binary_md5=None,
+                 version=None, upload_date=None, language=None,
+                 dependencies=None, flow_id=None):
         self.name = name
         self.description = description
         self.model = model
 
         if components is None:
             components = OrderedDict()
         elif not isinstance(components, OrderedDict):
-            raise TypeError('Components must be of type OrderedDict, but are %s.' %
+            raise TypeError('components must be of type OrderedDict, but is %s.' %
                             type(components))
         self.components = components
         if parameters is None:
             parameters = OrderedDict()
         elif not isinstance(parameters, OrderedDict):
-            raise TypeError('Parameters must be of type OrderedDict, but are %s.' %
+            raise TypeError('parameters must be of type OrderedDict, but is %s.' %
                             type(parameters))
+        if parameters_meta_info is None:
+            parameters_meta_info = OrderedDict()
+        elif not isinstance(parameters_meta_info, OrderedDict):
+            raise TypeError('parameters_meta_info must be of type OrderedDict, but is %s.' %
+                            type(parameters_meta_info))
+        keys_parameters = set(parameters.keys())
+        keys_parameters_meta_info = set(parameters_meta_info.keys())
+        if len(keys_parameters.difference(keys_parameters_meta_info)) > 0:
+            raise ValueError('Parameter %s only in parameters, but not in'
+                             'parameters_meta_info.' %
+                             str(keys_parameters.difference(keys_parameters_meta_info)))
+        if len(keys_parameters_meta_info.difference(keys_parameters)) > 0:
+            raise ValueError('Parameter %s only in parameters_meta_info, but not in'
+                             'parameters.' %
+                             str(keys_parameters_meta_info.difference(keys_parameters)))
+
         self.parameters = parameters
+        self.parameters_meta_info = parameters_meta_info
 
         self.external_version = external_version
-        self.creator = creator
         self.upoader = uploader
-        self.tag = tag
+
+        if tags is None:
+            tags = []
+        self.tags = tags
+        self.binary_url = binary_url
+        self.binary_format = binary_format
+        self.binary_md5 = binary_md5
+        self.version = version
+        self.upload_date = upload_date
+        self.language = language
+        self.dependencies = dependencies
         self.flow_id = flow_id
 
-    def _generate_flow_xml(self):
+    def _to_xml(self):
         """Generate xml representation of self for upload to server.
 
         Returns
         -------
         flow_xml : string
             Flow represented as XML string.
         """
-        model = self.model
+        flow_dict = self.__to_dict()
+        flow_xml = xmltodict.unparse(flow_dict, pretty=True)
 
+        # A flow may not be uploaded with the encoding specification..
+        flow_xml = flow_xml.split('\n', 1)[-1]
+        return flow_xml
+
+    def __to_dict(self):
         flow_dict = OrderedDict()
         flow_dict['oml:flow'] = OrderedDict()
         flow_dict['oml:flow']['@xmlns:oml'] = 'http://openml.org/openml'
+        if self.flow_id is not None:
+            flow_dict['oml:flow']['oml:id'] = self.flow_id
+        if self.upoader is not None:
+            flow_dict['oml:flow']['oml:uploader'] = self.upoader
         flow_dict['oml:flow']['oml:name'] = self._get_name()
+        if self.version is not None:
+            flow_dict['oml:flow']['oml:version'] = self.version
         flow_dict['oml:flow']['oml:external_version'] = self.external_version
         flow_dict['oml:flow']['oml:description'] = self.description
+        if self.upload_date is not None:
+            flow_dict['oml:flow']['oml:upload_date'] = self.upload_date
+        if self.language is not None:
+            flow_dict['oml:flow']['oml:language'] = self.language
+        if self.dependencies is not None:
+            flow_dict['oml:flow']['oml:dependencies'] = self.dependencies
 
-        clf_params = model.get_params()
         flow_parameters = []
-        for k, v in clf_params.items():
-            # data_type, default_value, description, recommendedRange
-            # type = v.__class__.__name__    Not using this because it doesn't conform standards
-            # eg. int instead of integer
-            param_dict = {'oml:name': k}
+        for key in self.parameters:
+            param_dict = OrderedDict()
+            param_dict['oml:name'] = key
+            if self.parameters_meta_info[key]['data_type'] is not None:
+                param_dict['oml:data_type'] = self.parameters_meta_info[key].get('data_type')
+            param_dict['oml:default_value'] = self.parameters[key],
+            if self.parameters_meta_info[key]['description'] is not None:
+                param_dict['oml:description'] = self.parameters_meta_info[key].get('description')
             flow_parameters.append(param_dict)
 
         flow_dict['oml:flow']['oml:parameter'] = flow_parameters
 
-        flow_xml = xmltodict.unparse(flow_dict, pretty=True)
+        components = []
+        for key in self.components:
+            component_dict = OrderedDict()
+            component_dict['oml:identifier'] = key
+            component_dict['oml:flow'] = self.components[key].__to_dict()['oml:flow']
+            components.append(component_dict)
+        flow_dict['oml:flow']['oml:component'] = components
 
-        # A flow may not be uploaded with the encoding specification..
-        flow_xml = flow_xml.split('\n', 1)[-1]
-        return flow_xml
+        flow_dict['oml:flow']['oml:tag'] = self.tags
+
+        if self.binary_url is not None:
+            flow_dict['oml:flow']['oml:binary_url'] = self.binary_url
+        if self.binary_format is not None:
+            flow_dict['oml:flow']['oml:binary_format'] = self.binary_format
+        if self.binary_md5 is not None:
+            flow_dict['oml:flow']['oml:binary_md5'] = self.binary_md5
+
+        return flow_dict
+
+    @classmethod
+    def _from_xml(cls, xml_dict):
+        dic = xml_dict["oml:flow"]
+        flow_id = int(dic['oml:id']) if 'oml:id' in dic else None
+        uploader = dic.get('oml:uploader')
+        name = dic['oml:name']
+        external_version = dic.get('oml:external_version')
+        description = dic.get('oml:description')
+        upload_date = dic.get('oml:upload_date')
+        language = dic.get('oml:language')
+        dependencies = dic.get('oml:dependencies')
+        version = dic.get('oml:version')
+        binary_url = dic.get('oml:binary_url')
+        binary_format = dic.get('oml:binary_format')
+        binary_md5 = dic.get('oml:binary_md5')
+
+        parameters = OrderedDict()
+        parameters_meta_info = OrderedDict()
+        if 'oml:parameter' in dic:
+            if isinstance(dic['oml:parameter'], dict):
+                oml_parameters = [dic['oml:parameter']]
+            else:
+                oml_parameters = dic['oml:parameter']
+
+            for oml_parameter in oml_parameters:
+                parameter_name = oml_parameter['oml:name']
+                default_value = oml_parameter['oml:default_value']
+                parameters[parameter_name] = default_value
+
+                meta_info = dict()
+                meta_info['description'] = oml_parameter.get('oml:description')
+                meta_info['data_type'] = oml_parameter.get('oml:data_type')
+                parameters_meta_info[parameter_name] = meta_info
+
+        components = OrderedDict()
+        if 'oml:component' in dic:
+            if isinstance(dic['oml:component'], dict):
+                oml_components = [dic['oml:component']]
+            else:
+                oml_components = dic['oml:component']
+
+            for component in oml_components:
+                flow = OpenMLFlow._from_xml(component)
+                components[component['oml:identifier']] = flow
+
+        tags = []
+        if 'oml:tag' in dic and dic['oml:tag'] is not None:
+            if isinstance(dic['oml:tag'], dict):
+                oml_tags = [dic['oml:tag']]
+            else:
+                oml_tags = dic['oml:tag']
+
+            for tag in oml_tags:
+                tags.append(tag)
+
+        return cls(name=name, description=description, model=None,
+                   components=components, parameters=parameters,
+                   parameters_meta_info=parameters_meta_info,
+                   external_version=external_version,
+                   uploader=uploader, tags=tags, version=version,
+                   upload_date=upload_date, language=language,
+                   dependencies=dependencies, binary_url=binary_url,
+                   binary_format=binary_format, binary_md5=binary_md5,
+                   flow_id=flow_id)
+
+    def __eq__(self, other):
+        """Override the default Equals behavior"""
+        if isinstance(other, self.__class__):
+            this_dict = self.__dict__.copy()
+            this_parameters = this_dict['parameters']
+            del this_dict['parameters']
+            this_components = this_dict['components']
+            del this_dict['components']
+            del this_dict['model']
+
+            other_dict = other.__dict__.copy()
+            other_parameters = other_dict['parameters']
+            del other_dict['parameters']
+            other_components = other_dict['components']
+            del other_dict['components']
+            del other_dict['model']
+
+            parameters_equal = this_parameters.keys() == other_parameters.keys() and \
+                               all([this_parameter == other_parameter
+                                    for this_parameter, other_parameter in
+                                    zip(this_parameters, other_parameters)])
+            components_equal = this_components.keys() == other_components.keys() and \
+                               all([this_component == other_component
+                                    for this_component, other_component in
+                                    zip(this_components, other_components)])
+            equal = this_dict == other_dict
+
+            return parameters_equal and components_equal and equal
+        return NotImplemented
 
     def publish(self):
         """Publish flow to OpenML server.
@@ -95,7 +249,7 @@ def publish(self):
         self : OpenMLFlow
 
         """
-        xml_description = self._generate_flow_xml()
+        xml_description = self._to_xml()
 
         file_elements = {'description': xml_description}
         return_code, return_value = _perform_api_call(
@@ -142,3 +296,38 @@ def create_flow_from_model(model, converter, description=None):
         flow.description = description
 
     return flow
+
+
+def _check_flow_exists(name, version):
+    """Retrieves the flow id of the flow uniquely identified by name+version.
+
+    Parameter
+    ---------
+    name : string
+        Name of the flow
+    version : string
+        Version information associated with flow.
+
+    Returns
+    -------
+    flow_exist : int
+        Flow id or -1 if the flow doesn't exist.
+
+    Notes
+    -----
+    see http://www.openml.org/api_docs/#!/flow/get_flow_exists_name_version
+    """
+    if not (type(name) is str and len(name) > 0):
+        raise ValueError('Argument \'name\' should be a non-empty string')
+    if not (type(version) is str and len(version) > 0):
+        raise ValueError('Argument \'version\' should be a non-empty string')
+
+    return_code, xml_response = _perform_api_call(
+        "flow/exists/%s/%s" % (name, version))
+    # TODO check with latest version of code if this raises an exception
+    if return_code != 200:
+        # fixme raise appropriate error
+        raise ValueError("api call failed: %s" % xml_response)
+    xml_dict = xmltodict.parse(xml_response)
+    flow_id = xml_dict['oml:flow_exists']['oml:id']
+    return return_code, xml_response, flow_id

openml/flows/functions.py

@@ -1,38 +1,31 @@
 import xmltodict
 
 from openml._api_calls import _perform_api_call
+from . import OpenMLFlow
+from ..util import URLError
 
 
-def _check_flow_exists(name, version):
-    """Retrieves the flow id of the flow uniquely identified by name+version.
+def get_flow(flow_id):
+    """Download the OpenML flow for a given flow ID.
 
-    Parameter
-    ---------
-    name : string
-        Name of the flow
-    version : string
-        Version information associated with flow.
-
-    Returns
-    -------
-    flow_exist : int
-        Flow id or -1 if the flow doesn't exist.
-
-    Notes
-    -----
-    see http://www.openml.org/api_docs/#!/flow/get_flow_exists_name_version
+    Parameters
+    ----------
+    flow_id : int
+        The OpenML flow id.
     """
-    if not (type(name) is str and len(name) > 0):
-        raise ValueError('Argument \'name\' should be a non-empty string')
-    if not (type(version) is str and len(version) > 0):
-        raise ValueError('Argument \'version\' should be a non-empty string')
+    try:
+        flow_id = int(flow_id)
+    except:
+        raise ValueError("Flow ID is neither an Integer nor can be "
+                         "cast to an Integer.")
+
+    try:
+        return_code, flow_xml = _perform_api_call(
+            "flow/%d" % flow_id)
+    except (URLError, UnicodeEncodeError) as e:
+        print(e)
+        raise e
 
-    return_code, xml_response = _perform_api_call(
-        "flow/exists/%s/%s" % (name, version))
-    # TODO check with latest version of code if this raises an exception
-    if return_code != 200:
-        # fixme raise appropriate error
-        raise ValueError("api call failed: %s" % xml_response)
-    xml_dict = xmltodict.parse(xml_response)
-    flow_id = xml_dict['oml:flow_exists']['oml:id']
-    return return_code, xml_response, flow_id
+    flow_dict = xmltodict.parse(flow_xml)
+    flow = OpenMLFlow._from_xml(flow_dict)
+    return flow