Create OpenMLBase, have most OpenML objects derive from it (#828)

* Create OpenMLBase, have OpenMLFlow derive from it.

* Derive ID and entity_letter based on class type.

* Add #433 open_in_browser.

* Use OpenMLBase in Dataset, Run, Task.

* Use OpenMLBase for Study

* Update tag functions to take into account entity type.

* Quote OpenMLBase typing as it is only imported for type checking.

* Remove _repr_pretty_ as the default __repr__ prints pretty in a notebook anyway.

* Move _to_xml to base

* Fix bug, actually check for instance type to determine entity.

* Provide list to task type description in task __repr__

* Move fetching id to derived classes.

* Share base_url logic. Fix mypy warnings.

* Make child classes responsible for making sure _entity_letter is correct.

* Docstring and type hint changes.

* PEP8

* PEP8

* Fix mypy issues

* Fix CI mypy issues.

* Dont use Py3.6 syntax

* Fix CI mypy issue

Author: PGijsbers

openml/base.py

@@ -0,0 +1,129 @@
+from abc import ABC, abstractmethod
+from collections import OrderedDict
+import re
+from typing import Optional, List, Tuple, Union
+import webbrowser
+
+import xmltodict
+
+import openml.config
+from .utils import _tag_openml_base
+
+
+class OpenMLBase(ABC):
+    """ Base object for functionality that is shared across entities. """
+
+    def __repr__(self):
+        body_fields = self._get_repr_body_fields()
+        return self._apply_repr_template(body_fields)
+
+    @property
+    @abstractmethod
+    def id(self) -> Optional[int]:
+        """ The id of the entity, it is unique for its entity type. """
+        pass
+
+    @property
+    def openml_url(self) -> Optional[str]:
+        """ The URL of the object on the server, if it was uploaded, else None. """
+        if self.id is None:
+            return None
+        return self.__class__.url_for_id(self.id)
+
+    @classmethod
+    def url_for_id(cls, id_: int) -> str:
+        """ Return the OpenML URL for the object of the class entity with the given id. """
+        # Sample url for a flow: openml.org/f/123
+        return "{}/{}/{}".format(openml.config.server_base_url, cls._entity_letter(), id_)
+
+    @classmethod
+    def _entity_letter(cls) -> str:
+        """ Return the letter which represents the entity type in urls, e.g. 'f' for flow."""
+        # We take advantage of the class naming convention (OpenMLX),
+        # which holds for all entities except studies and tasks, which overwrite this method.
+        return cls.__name__.lower()[len('OpenML'):][0]
+
+    @abstractmethod
+    def _get_repr_body_fields(self) -> List[Tuple[str, Union[str, int, List[str]]]]:
+        """ Collect all information to display in the __repr__ body.
+
+        Returns
+        ------
+        body_fields : List[Tuple[str, Union[str, int, List[str]]]]
+            A list of (name, value) pairs to display in the body of the __repr__.
+            E.g.: [('metric', 'accuracy'), ('dataset', 'iris')]
+            If value is a List of str, then each item of the list will appear in a separate row.
+        """
+        # Should be implemented in the base class.
+        pass
+
+    def _apply_repr_template(self, body_fields: List[Tuple[str, str]]) -> str:
+        """ Generates the header and formats the body for string representation of the object.
+
+         Parameters
+         ----------
+         body_fields: List[Tuple[str, str]]
+            A list of (name, value) pairs to display in the body of the __repr__.
+         """
+        # We add spaces between capitals, e.g. ClassificationTask -> Classification Task
+        name_with_spaces = re.sub(r"(\w)([A-Z])", r"\1 \2",
+                                  self.__class__.__name__[len('OpenML'):])
+        header_text = 'OpenML {}'.format(name_with_spaces)
+        header = '{}\n{}\n'.format(header_text, '=' * len(header_text))
+
+        longest_field_name_length = max(len(name) for name, value in body_fields)
+        field_line_format = "{{:.<{}}}: {{}}".format(longest_field_name_length)
+        body = '\n'.join(field_line_format.format(name, value) for name, value in body_fields)
+        return header + body
+
+    @abstractmethod
+    def _to_dict(self) -> 'OrderedDict[str, OrderedDict]':
+        """ Creates a dictionary representation of self.
+
+        Uses OrderedDict to ensure consistent ordering when converting to xml.
+        The return value (OrderedDict) will be used to create the upload xml file.
+        The xml file must have the tags in exactly the order of the object's xsd.
+        (see https://github.com/openml/OpenML/blob/master/openml_OS/views/pages/api_new/v1/xsd/).
+
+        Returns
+        -------
+        OrderedDict
+            Flow represented as OrderedDict.
+
+        """
+        # Should be implemented in the base class.
+        pass
+
+    def _to_xml(self) -> str:
+        """ Generate xml representation of self for upload to server. """
+        dict_representation = self._to_dict()
+        xml_representation = xmltodict.unparse(dict_representation, pretty=True)
+
+        # A task may not be uploaded with the xml encoding specification:
+        # <?xml version="1.0" encoding="utf-8"?>
+        encoding_specification, xml_body = xml_representation.split('\n', 1)
+        return xml_body
+
+    def open_in_browser(self):
+        """ Opens the OpenML web page corresponding to this object in your default browser. """
+        webbrowser.open(self.openml_url)
+
+    def push_tag(self, tag: str):
+        """Annotates this entity with a tag on the server.
+
+        Parameters
+        ----------
+        tag : str
+            Tag to attach to the flow.
+        """
+        _tag_openml_base(self, tag)
+
+    def remove_tag(self, tag: str):
+        """Removes a tag from this entity on the server.
+
+        Parameters
+        ----------
+        tag : str
+            Tag to attach to the flow.
+        """
+        _tag_openml_base(self, tag, untag=True)

openml/config.py

@@ -28,7 +28,8 @@
 
 # Default values are actually added here in the _setup() function which is
 # called at the end of this module
-server = _defaults['server']
+server = str(_defaults['server'])  # so mypy knows it is a string
+server_base_url = server[:-len('/api/v1/xml')]
 apikey = _defaults['apikey']
 # The current cache directory (without the server name)
 cache_directory = _defaults['cachedir']

openml/datasets/dataset.py

@@ -15,15 +15,15 @@
 from warnings import warn
 
 import openml._api_calls
+from openml.base import OpenMLBase
 from .data_feature import OpenMLDataFeature
 from ..exceptions import PyOpenMLError
-from ..utils import _tag_entity
 
 
 logger = logging.getLogger(__name__)
 
 
-class OpenMLDataset(object):
+class OpenMLDataset(OpenMLBase):
     """Dataset object.
 
     Allows fetching and uploading datasets to OpenML.
@@ -184,11 +184,12 @@ def __init__(self, name, description, format=None,
         else:
             self.data_pickle_file = None
 
-    def __repr__(self):
-        header = "OpenML Dataset"
-        header = '{}\n{}\n'.format(header, '=' * len(header))
+    @property
+    def id(self) -> Optional[int]:
+        return self.dataset_id
 
-        base_url = "{}".format(openml.config.server[:-len('api/v1/xml')])
+    def _get_repr_body_fields(self) -> List[Tuple[str, Union[str, int, List[str]]]]:
+        """ Collect all information to display in the __repr__ body. """
         fields = {"Name": self.name,
                   "Version": self.version,
                   "Format": self.format,
@@ -201,19 +202,14 @@ def __repr__(self):
         if self.upload_date is not None:
             fields["Upload Date"] = self.upload_date.replace('T', ' ')
         if self.dataset_id is not None:
-            fields["OpenML URL"] = "{}d/{}".format(base_url, self.dataset_id)
+            fields["OpenML URL"] = self.openml_url
         if self.qualities is not None and self.qualities['NumberOfInstances'] is not None:
             fields["# of instances"] = int(self.qualities['NumberOfInstances'])
 
         # determines the order in which the information will be printed
         order = ["Name", "Version", "Format", "Upload Date", "Licence", "Download URL",
                  "OpenML URL", "Data File", "Pickle File", "# of features", "# of instances"]
-        fields = [(key, fields[key]) for key in order if key in fields]
-
-        longest_field_name_length = max(len(name) for name, value in fields)
-        field_line_format = "{{:.<{}}}: {{}}".format(longest_field_name_length)
-        body = '\n'.join(field_line_format.format(name, value) for name, value in fields)
-        return header + body
+        return [(key, fields[key]) for key in order if key in fields]
 
     def __eq__(self, other):
 
@@ -462,26 +458,6 @@ def _load_data(self):
 
         return data, categorical, attribute_names
 
-    def push_tag(self, tag):
-        """Annotates this data set with a tag on the server.
-
-        Parameters
-        ----------
-        tag : str
-            Tag to attach to the dataset.
-        """
-        _tag_entity('data', self.dataset_id, tag)
-
-    def remove_tag(self, tag):
-        """Removes a tag from this dataset on the server.
-
-        Parameters
-        ----------
-        tag : str
-            Tag to attach to the dataset.
-        """
-        _tag_entity('data', self.dataset_id, tag, untag=True)
-
     @staticmethod
     def _convert_array_format(data, array_format, attribute_names):
         """Convert a dataset to a given array format.
@@ -796,22 +772,16 @@ def publish(self):
         self.dataset_id = int(response['oml:upload_data_set']['oml:id'])
         return self.dataset_id
 
-    def _to_xml(self):
-        """ Serialize object to xml for upload
-
-        Returns
-        -------
-        xml_dataset : str
-            XML description of the data.
-        """
+    def _to_dict(self) -> 'OrderedDict[str, OrderedDict]':
+        """ Creates a dictionary representation of self. """
         props = ['id', 'name', 'version', 'description', 'format', 'creator',
                  'contributor', 'collection_date', 'upload_date', 'language',
                  'licence', 'url', 'default_target_attribute',
                  'row_id_attribute', 'ignore_attribute', 'version_label',
                  'citation', 'tag', 'visibility', 'original_data_url',
                  'paper_url', 'update_comment', 'md5_checksum']
 
-        data_container = OrderedDict()
+        data_container = OrderedDict()  # type: 'OrderedDict[str, OrderedDict]'
         data_dict = OrderedDict([('@xmlns:oml', 'http://openml.org/openml')])
         data_container['oml:data_set_description'] = data_dict
 
@@ -820,14 +790,7 @@ def _to_xml(self):
             if content is not None:
                 data_dict["oml:" + prop] = content
 
-        xml_string = xmltodict.unparse(
-            input_dict=data_container,
-            pretty=True,
-        )
-        # A flow may not be uploaded with the xml encoding specification:
-        # <?xml version="1.0" encoding="utf-8"?>
-        xml_string = xml_string.split('\n', 1)[-1]
-        return xml_string
+        return data_container
 
 
 def _check_qualities(qualities):

openml/evaluations/evaluation.py

@@ -61,18 +61,17 @@ def __repr__(self):
         header = "OpenML Evaluation"
         header = '{}\n{}\n'.format(header, '=' * len(header))
 
-        base_url = "{}".format(openml.config.server[:-len('api/v1/xml')])
         fields = {"Upload Date": self.upload_time,
                   "Run ID": self.run_id,
-                  "OpenML Run URL": "{}r/{}".format(base_url, self.run_id),
+                  "OpenML Run URL": openml.runs.OpenMLRun.url_for_id(self.run_id),
                   "Task ID": self.task_id,
-                  "OpenML Task URL": "{}t/{}".format(base_url, self.task_id),
+                  "OpenML Task URL": openml.tasks.OpenMLTask.url_for_id(self.task_id),
                   "Flow ID": self.flow_id,
-                  "OpenML Flow URL": "{}f/{}".format(base_url, self.flow_id),
+                  "OpenML Flow URL": openml.flows.OpenMLFlow.url_for_id(self.flow_id),
                   "Setup ID": self.setup_id,
                   "Data ID": self.data_id,
                   "Data Name": self.data_name,
-                  "OpenML Data URL": "{}d/{}".format(base_url, self.data_id),
+                  "OpenML Data URL": openml.datasets.OpenMLDataset.url_for_id(self.data_id),
                   "Metric Used": self.function,
                   "Result": self.value}