incorporate pieter's feedback

Author: mfeurer

openml/extensions/sklearn/extension.py

@@ -1270,8 +1270,12 @@ def _prediction_to_probabilities(y: np.ndarray, classes: List[Any]) -> np.ndarra
                 # Remap the probabilities in case there was a class missing at training time
                 # By default, the classification targets are mapped to be zero-based indices to the
                 # actual classes. Therefore, the model_classes contain the correct indices to the
-                # correct probability array (the actually array might be incorrect if there are
-                # some classes not present during train time).
+                # correct probability array. Example:
+                # classes in the dataset: 0, 1, 2, 3, 4, 5
+                # classes in the training set: 0, 1, 2, 4, 5
+                # then we need to add a column full of zeros into the probabilities for class 3
+                # (because the rest of the library expects that the probabilities are ordered the
+                # same way as the classes are ordered).
                 proba_y_new = np.zeros((proba_y.shape[0], len(task.class_labels)))
                 for idx, model_class in enumerate(model_classes):
                     proba_y_new[:, model_class] = proba_y[:, idx]

openml/runs/functions.py

@@ -1,5 +1,6 @@
 from collections import OrderedDict
 import io
+import itertools
 import os
 from typing import Any, List, Optional, Set, Tuple, Union, TYPE_CHECKING  # noqa F401
 import warnings
@@ -395,125 +396,119 @@ def _run_task_get_arffcontent(
     # TODO use different iterator to only provide a single iterator (less
     # methods, less maintenance, less confusion)
     num_reps, num_folds, num_samples = task.get_split_dimensions()
-    classes = None
-
-    n_fit = 0
-    for rep_no in range(num_reps):
-        for fold_no in range(num_folds):
-            for sample_no in range(num_samples):
-                n_fit += 1
-
-                train_indices, test_indices = task.get_train_test_split_indices(
-                    repeat=rep_no, fold=fold_no, sample=sample_no)
-                if isinstance(task, OpenMLSupervisedTask):
-                    x, y = task.get_X_and_y(dataset_format='array')
-                    train_x = x[train_indices]
-                    train_y = y[train_indices]
-                    test_x = x[test_indices]
-                    test_y = y[test_indices]
-                    if isinstance(task, (OpenMLClassificationTask, OpenMLClassificationTask)):
-                        classes = task.class_labels
-                elif isinstance(task, OpenMLClusteringTask):
-                    x = task.get_X(dataset_format='array')
-                    train_x = train_indices
-                    train_y = None
-                    test_x = test_indices
-                    test_y = None
-                else:
-                    raise NotImplementedError(task.task_type)
-
-                config.logger.info(
-                    "Going to execute flow '%s' on task %d for repeat %d fold %d sample %d.",
-                    flow.name, task.task_id, rep_no, fold_no, sample_no,
-                )
 
-                (
-                    pred_y,
-                    proba_y,
-                    user_defined_measures_fold,
-                    trace,
-                ) = extension._run_model_on_fold(
-                    model=model,
-                    task=task,
-                    X_train=train_x,
-                    y_train=train_y,
-                    rep_no=rep_no,
-                    fold_no=fold_no,
-                    X_test=test_x,
+    for n_fit, (rep_no, fold_no, sample_no) in enumerate(itertools.product(
+        range(num_reps),
+        range(num_folds),
+        range(num_samples),
+    )):
+
+        train_indices, test_indices = task.get_train_test_split_indices(
+            repeat=rep_no, fold=fold_no, sample=sample_no)
+        if isinstance(task, OpenMLSupervisedTask):
+            x, y = task.get_X_and_y(dataset_format='array')
+            train_x = x[train_indices]
+            train_y = y[train_indices]
+            test_x = x[test_indices]
+            test_y = y[test_indices]
+        elif isinstance(task, OpenMLClusteringTask):
+            x = task.get_X(dataset_format='array')
+            train_x = x[train_indices]
+            train_y = None
+            test_x = None
+            test_y = None
+        else:
+            raise NotImplementedError(task.task_type)
+
+        config.logger.info(
+            "Going to execute flow '%s' on task %d for repeat %d fold %d sample %d.",
+            flow.name, task.task_id, rep_no, fold_no, sample_no,
+        )
+
+        (
+            pred_y,
+            proba_y,
+            user_defined_measures_fold,
+            trace,
+        ) = extension._run_model_on_fold(
+            model=model,
+            task=task,
+            X_train=train_x,
+            y_train=train_y,
+            rep_no=rep_no,
+            fold_no=fold_no,
+            X_test=test_x,
+        )
+        if trace is not None:
+            traces.append(trace)
+
+        # add client-side calculated metrics. These is used on the server as
+        # consistency check, only useful for supervised tasks
+        def _calculate_local_measure(sklearn_fn, openml_name):
+            user_defined_measures_fold[openml_name] = sklearn_fn(test_y, pred_y)
+
+        if isinstance(task, (OpenMLClassificationTask, OpenMLLearningCurveTask)):
+
+            for i in range(0, len(test_indices)):
+
+                arff_line = [rep_no, fold_no, sample_no, i]  # type: List[Any]
+                for j, class_label in enumerate(task.class_labels):
+                    arff_line.append(proba_y[i][j])
+
+                arff_line.append(task.class_labels[pred_y[i]])
+                arff_line.append(task.class_labels[test_y[i]])
+
+                arff_datacontent.append(arff_line)
+
+            if add_local_measures:
+                _calculate_local_measure(
+                    sklearn.metrics.accuracy_score,
+                    'predictive_accuracy',
                 )
 
-                arff_datacontent_fold = []  # type: List[List]
-                if trace is not None:
-                    traces.append(trace)
-
-                # add client-side calculated metrics. These is used on the server as
-                # consistency check, only useful for supervised tasks
-                def _calculate_local_measure(sklearn_fn, openml_name):
-                    user_defined_measures_fold[openml_name] = sklearn_fn(test_y, pred_y)
-
-                if isinstance(task, (OpenMLClassificationTask, OpenMLLearningCurveTask)):
-
-                    for i in range(0, len(test_indices)):
-
-                        arff_line = [rep_no, fold_no, sample_no, i]  # type: List[Any]
-                        for j, class_label in enumerate(task.class_labels):
-                            arff_line.append(proba_y[i][j])
-
-                        arff_line.append(task.class_labels[pred_y[i]])
-                        arff_line.append(task.class_labels[test_y[i]])
+        elif isinstance(task, OpenMLRegressionTask):
 
-                        arff_datacontent.append(arff_line)
+            for i in range(0, len(test_indices)):
+                arff_line = [rep_no, fold_no, test_indices[i], pred_y[i], test_y[i]]
+                arff_datacontent.append(arff_line)
 
-                    if add_local_measures:
-                        _calculate_local_measure(
-                            sklearn.metrics.accuracy_score,
-                            'predictive_accuracy',
-                        )
-
-                elif isinstance(task, OpenMLRegressionTask):
-
-                    for i in range(0, len(test_indices)):
-                        arff_line = [rep_no, fold_no, test_indices[i], pred_y[i], test_y[i]]
-                        arff_datacontent.append(arff_line)
-
-                    if add_local_measures:
-                        _calculate_local_measure(
-                            sklearn.metrics.mean_absolute_error,
-                            'mean_absolute_error',
-                        )
+            if add_local_measures:
+                _calculate_local_measure(
+                    sklearn.metrics.mean_absolute_error,
+                    'mean_absolute_error',
+                )
 
-                elif isinstance(task, OpenMLClusteringTask):
-                    for i in range(0, len(test_indices)):
-                        arff_line = [test_indices[i], pred_y[i]]  # row_id, cluster ID
-                        arff_datacontent.append(arff_line)
+        elif isinstance(task, OpenMLClusteringTask):
+            for i in range(0, len(test_indices)):
+                arff_line = [test_indices[i], pred_y[i]]  # row_id, cluster ID
+                arff_datacontent.append(arff_line)
 
-                else:
-                    raise TypeError(type(task))
-
-                arff_datacontent.extend(arff_datacontent_fold)
+        else:
+            raise TypeError(type(task))
 
-                for measure in user_defined_measures_fold:
+        for measure in user_defined_measures_fold:
 
-                    if measure not in user_defined_measures_per_fold:
-                        user_defined_measures_per_fold[measure] = OrderedDict()
-                    if rep_no not in user_defined_measures_per_fold[measure]:
-                        user_defined_measures_per_fold[measure][rep_no] = OrderedDict()
+            if measure not in user_defined_measures_per_fold:
+                user_defined_measures_per_fold[measure] = OrderedDict()
+            if rep_no not in user_defined_measures_per_fold[measure]:
+                user_defined_measures_per_fold[measure][rep_no] = OrderedDict()
 
-                    if measure not in user_defined_measures_per_sample:
-                        user_defined_measures_per_sample[measure] = OrderedDict()
-                    if rep_no not in user_defined_measures_per_sample[measure]:
-                        user_defined_measures_per_sample[measure][rep_no] = OrderedDict()
-                    if fold_no not in user_defined_measures_per_sample[
-                            measure][rep_no]:
-                        user_defined_measures_per_sample[measure][rep_no][fold_no] = OrderedDict()
+            if measure not in user_defined_measures_per_sample:
+                user_defined_measures_per_sample[measure] = OrderedDict()
+            if rep_no not in user_defined_measures_per_sample[measure]:
+                user_defined_measures_per_sample[measure][rep_no] = OrderedDict()
+            if fold_no not in user_defined_measures_per_sample[measure][rep_no]:
+                user_defined_measures_per_sample[measure][rep_no][fold_no] = OrderedDict()
 
-                    user_defined_measures_per_fold[measure][rep_no][
-                        fold_no] = user_defined_measures_fold[measure]
-                    user_defined_measures_per_sample[measure][rep_no][fold_no][
-                        sample_no] = user_defined_measures_fold[measure]
+            user_defined_measures_per_fold[measure][rep_no][fold_no] = (
+                user_defined_measures_fold[measure]
+            )
+            user_defined_measures_per_sample[measure][rep_no][fold_no][sample_no] = (
+                user_defined_measures_fold[measure]
+            )
 
     if len(traces) > 0:
-        if len(traces) != n_fit:
+        if len(traces) != n_fit + 1:
             raise ValueError(
                 'Did not find enough traces (expected {}, found {})'.format(n_fit, len(traces))
             )

openml/runs/trace.py

@@ -283,7 +283,7 @@ def _trace_from_arff_struct(cls, attributes, content, error_message):
                 setup_string=None,
                 evaluation=evaluation,
                 selected=selected,
-                paramaters=parameters,
+                parameters=parameters,
             )
             trace[(repeat, fold, iteration)] = current
 
@@ -350,15 +350,27 @@ def trace_from_xml(cls, xml):
 
     @classmethod
     def merge_traces(cls, traces: List['OpenMLRunTrace']) -> 'OpenMLRunTrace':
-        for i in range(1, len(traces)):
-            if traces[i] != traces[i - 1]:
-                raise ValueError('Cannot merge traces!')
 
         merged_trace = OrderedDict()  # type: OrderedDict[Tuple[int, int, int], OpenMLTraceIteration]  # noqa E501
 
+        previous_iteration = None
         for trace in traces:
             for iteration in trace:
-                merged_trace[(iteration.repeat, iteration.fold, iteration.iteration)] = iteration
+                key = (iteration.repeat, iteration.fold, iteration.iteration)
+                if previous_iteration is not None:
+                    if (
+                        list(merged_trace[previous_iteration].parameters.keys())
+                        != list(iteration.parameters.keys())
+                    ):
+                        raise ValueError(
+                            'Cannot merge traces because the parameters are not equal: {} vs {}'.
+                            format(
+                                list(merged_trace[previous_iteration].parameters.keys()),
+                                list(iteration.parameters.keys()),
+                            )
+                        )
+                merged_trace[key] = iteration
+                previous_iteration = key
 
         return cls(None, merged_trace)
 
@@ -410,25 +422,25 @@ def __init__(
         setup_string,
         evaluation,
         selected,
-        paramaters=None,
+        parameters=None,
     ):
 
         if not isinstance(selected, bool):
             raise TypeError(type(selected))
-        if setup_string and paramaters:
+        if setup_string and parameters:
             raise ValueError(
                 'Can only be instantiated with either '
                 'setup_string or parameters argument.'
             )
-        elif not setup_string and not paramaters:
+        elif not setup_string and not parameters:
             raise ValueError(
                 'Either setup_string or parameters needs to be passed as '
                 'argument.'
             )
-        if paramaters is not None and not isinstance(paramaters, OrderedDict):
+        if parameters is not None and not isinstance(parameters, OrderedDict):
             raise TypeError(
                 'argument parameters is not an instance of OrderedDict, but %s'
-                % str(type(paramaters))
+                % str(type(parameters))
             )
 
         self.repeat = repeat
@@ -437,7 +449,7 @@ def __init__(
         self.setup_string = setup_string
         self.evaluation = evaluation
         self.selected = selected
-        self.parameters = paramaters
+        self.parameters = parameters
 
     def get_parameters(self):
         result = {}
@@ -464,14 +476,3 @@ def __str__(self):
             self.evaluation,
             self.selected,
         )
-
-    def __eq__(self, other):
-        if not isinstance(other, OpenMLTraceIteration):
-            return False
-        attributes = [
-            'repeat', 'fold', 'iteration', 'setup_string', 'evaluation', 'selected', 'paramaters',
-        ]
-        for attr in attributes:
-            if getattr(self, attr) != getattr(other, attr):
-                return False
-        return True