Merge pull request #673 from openml/improve_extension_interface

Improve extension interface

Author: PGijsbers

openml/_api_calls.py

@@ -80,7 +80,7 @@ def _read_url_files(url, data=None, file_elements=None):
         files=file_elements,
     )
     if response.status_code != 200:
-        raise _parse_server_exception(response, url=url)
+        raise _parse_server_exception(response, url)
     if 'Content-Encoding' not in response.headers or \
             response.headers['Content-Encoding'] != 'gzip':
         warnings.warn('Received uncompressed content from OpenML for {}.'
@@ -95,7 +95,7 @@ def _read_url(url, request_method, data=None):
 
     response = send_request(request_method=request_method, url=url, data=data)
     if response.status_code != 200:
-        raise _parse_server_exception(response, url=url)
+        raise _parse_server_exception(response, url)
     if 'Content-Encoding' not in response.headers or \
             response.headers['Content-Encoding'] != 'gzip':
         warnings.warn('Received uncompressed content from OpenML for {}.'
@@ -137,15 +137,15 @@ def send_request(
     return response
 
 
-def _parse_server_exception(response, url=None):
+def _parse_server_exception(response, url):
     # OpenML has a sophisticated error system
     # where information about failures is provided. try to parse this
     try:
         server_exception = xmltodict.parse(response.text)
     except Exception:
         raise OpenMLServerError(
-            'Unexpected server error. Please contact the developers!\n'
-            'Status code: {}\n{}'.format(response.status_code, response.text))
+            'Unexpected server error when calling {}. Please contact the developers!\n'
+            'Status code: {}\n{}'.format(url, response.status_code, response.text))
 
     server_error = server_exception['oml:error']
     code = int(server_error['oml:code'])

openml/extensions/extension_interface.py

@@ -1,12 +1,15 @@
 from abc import ABC, abstractmethod
 from collections import OrderedDict  # noqa: F401
-from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING
+from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING, Union
+
+import numpy as np
+import scipy.sparse
 
 # Avoid import cycles: https://mypy.readthedocs.io/en/latest/common_issues.html#import-cycles
 if TYPE_CHECKING:
     from openml.flows import OpenMLFlow
     from openml.tasks.task import OpenMLTask
-    from openml.runs.trace import OpenMLRunTrace, OpenMLTraceIteration
+    from openml.runs.trace import OpenMLRunTrace, OpenMLTraceIteration  # noqa F401
 
 
 class Extension(ABC):
@@ -147,47 +150,46 @@ def _run_model_on_fold(
         self,
         model: Any,
         task: 'OpenMLTask',
+        X_train: Union[np.ndarray, scipy.sparse.spmatrix],
         rep_no: int,
         fold_no: int,
-        sample_no: int,
-        add_local_measures: bool,
-    ) -> Tuple[List[List], List[List], 'OrderedDict[str, float]', Any]:
+        y_train: Optional[np.ndarray] = None,
+        X_test: Optional[Union[np.ndarray, scipy.sparse.spmatrix]] = None,
+    ) -> Tuple[np.ndarray, np.ndarray, 'OrderedDict[str, float]', Optional['OpenMLRunTrace']]:
         """Run a model on a repeat,fold,subsample triplet of the task and return prediction information.
 
         Returns the data that is necessary to construct the OpenML Run object. Is used by
-        run_task_get_arff_content.
+        :func:`openml.runs.run_flow_on_task`.
 
         Parameters
         ----------
         model : Any
             The UNTRAINED model to run. The model instance will be copied and not altered.
         task : OpenMLTask
             The task to run the model on.
+        X_train : array-like
+            Training data for the given repetition and fold.
         rep_no : int
             The repeat of the experiment (0-based; in case of 1 time CV, always 0)
         fold_no : int
             The fold nr of the experiment (0-based; in case of holdout, always 0)
-        sample_no : int
-            In case of learning curves, the index of the subsample (0-based; in case of no
-            learning curve, always 0)
-        add_local_measures : bool
-            Determines whether to calculate a set of measures (i.e., predictive accuracy) locally,
-            to later verify server behaviour.
+        y_train : Optional[np.ndarray] (default=None)
+            Target attributes for supervised tasks. In case of classification, these are integer
+            indices to the potential classes specified by dataset.
+        X_test : Optional, array-like (default=None)
+            Test attributes to test for generalization in supervised tasks.
 
         Returns
         -------
-        arff_datacontent : List[List]
-            Arff representation (list of lists) of the predictions that were
-            generated by this fold (required to populate predictions.arff)
-        arff_tracecontent :  List[List]
-            Arff representation (list of lists) of the trace data that was generated by this fold
-            (will be used to populate trace.arff, leave it empty if the model did not perform any
-            hyperparameter optimization).
+        predictions : np.ndarray
+            Model predictions.
+        probabilities :  Optional, np.ndarray
+            Predicted probabilities (only applicable for supervised classification tasks).
         user_defined_measures : OrderedDict[str, float]
             User defined measures that were generated on this fold
-        model : Any
-            The model trained on this repeat,fold,subsample triple. Will be used to generate trace
-            information later on (in ``obtain_arff_trace``).
+        trace : Optional, OpenMLRunTrace
+            Hyperparameter optimization trace (only applicable for supervised tasks with
+            hyperparameter optimization).
         """
 
     @abstractmethod
@@ -222,21 +224,6 @@ def obtain_parameter_values(
     ################################################################################################
     # Abstract methods for hyperparameter optimization
 
-    def is_hpo_class(self, model: Any) -> bool:
-        """Check whether the model performs hyperparameter optimization.
-
-        Used to check whether an optimization trace can be extracted from the model after running
-        it.
-
-        Parameters
-        ----------
-        model : Any
-
-        Returns
-        -------
-        bool
-        """
-
     @abstractmethod
     def instantiate_model_from_hpo_class(
         self,
@@ -258,25 +245,3 @@ def instantiate_model_from_hpo_class(
         Any
         """
         # TODO a trace belongs to a run and therefore a flow -> simplify this part of the interface!
-
-    @abstractmethod
-    def obtain_arff_trace(
-        self,
-        model: Any,
-        trace_content: List[List],
-    ) -> 'OpenMLRunTrace':
-        """Create arff trace object from a fitted model and the trace content obtained by
-        repeatedly calling ``run_model_on_task``.
-
-        Parameters
-        ----------
-        model : Any
-            A fitted hyperparameter optimization model.
-
-        trace_content : List[List]
-            Trace content obtained by ``openml.runs.run_flow_on_task``.
-
-        Returns
-        -------
-        OpenMLRunTrace
-        """