Merge

Author: zardaloop

tests/.travis.yml .travis.ymltests/.travis.yml renamed to .travis.yml

@@ -26,11 +26,7 @@ install:
   - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION numpy scipy==0.13.3 matplotlib setuptools nose
   - source activate test-environment
   - python setup.py install
-  - pip install xmltodict
-  - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
-      pip install mock
-    fi
-  - python setup.py install
+  - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then pip install mock; fi
 
 # command to run tests, e.g. python setup.py test
 script:  python setup.py test

openml/apiconnector.py

@@ -23,13 +23,13 @@
 import xmltodict
 
 from .entities.dataset import OpenMLDataset
-from .entities.task import Task
+from .entities.task import OpenMLTask
 from .entities.split import OpenMLSplit
 from .util import is_string
 
 logger = logging.getLogger(__name__)
 
-OPENML_URL = "http://www.openml.org"
+OPENML_URL = "http://api_new.openml.org/v1/"
 
 
 class OpenMLStatusChange(Warning):
@@ -52,10 +52,6 @@ def __init__(self, message):
         super(OpenMLServerError, self).__init__(message)
 
 
-class AuthentificationError(PyOpenMLError):
-    def __init__(self, message):
-        super(AuthentificationError, self).__init__(message)
-
 class OpenMLCacheException(PyOpenMLError):
     def __init__(self, message):
         super(OpenMLCacheException, self).__init__(message)
@@ -79,14 +75,9 @@ class APIConnector(object):
         directory '.openml/cache' in the users home directory will be used.
         If either directory does not exist, it will be created.
 
-    username : string, optional (default=None)
-        Your username.
-
-    password : string, optional (default=None)
-        Your passwort. This will not be stored! Instead, the md5 hash is
-        calculated and used to authenticate to the OpenML server,
-        which returns a session key. This session key is the only credential
-        which is stored.
+    apikey : string, optional (default=None)
+        Your OpenML API key which will be used to authenticate you at the OpenML
+        server.
 
     server : string, optional (default=None)
         The OpenML server to connect to.
@@ -112,24 +103,18 @@ class APIConnector(object):
     Raises
     ------
     ValueError
-        If one of the following variables is neither specified in the
-        configuration file nor when creating the APIconnector class:
-        username, password
-    AuthentificationError
-        If authentification at the OpenML server does not work.
+        If apikey is neither specified in the config nor given as an argument.
     OpenMLServerError
         If the OpenML server returns an unexptected response.
 
     Testing the API calls in Firefox
     --------------------------------
-    With the Firefox AddOn HTTPRequestor, one can check the OpenML API calls
-    which need authentification (by providing such). First, create the md5
-    hash of your OpenML password and add it to
+    With the Firefox AddOn HTTPRequestor, one can check the OpenML API calls.
 
     """
-    def __init__(self, cache_directory=None, username=None, password=None,
+    def __init__(self, cache_directory=None, apikey=None,
                  server=None, verbosity=None, configure_logger=True,
-                 authenticate=True, private_directory=None):
+                 private_directory=None):
         # The .openml directory is necessary, just try to create it (EAFP)
         try:
             os.mkdir(os.path.expanduser('~/.openml'))
@@ -141,10 +126,8 @@ def __init__(self, cache_directory=None, username=None, password=None,
         self.config = self._parse_config()
         if cache_directory is not None:
             self.config.set('FAKE_SECTION', 'cachedir', cache_directory)
-        if username is not None:
-            self.config.set('FAKE_SECTION', 'username', username)
-        if password is not None:
-            self.config.set('FAKE_SECTION', 'password', password)
+        if apikey is not None:
+            self.config.set('FAKE_SECTION', 'apikey', apikey)
         if server is not None:
             self.config.set('FAKE_SECTION', 'server', server)
         if verbosity is not None:
@@ -164,11 +147,6 @@ def __init__(self, cache_directory=None, username=None, password=None,
                 format='[%(levelname)s] [%(asctime)s:%(name)s] %('
                        'message)s', datefmt='%H:%M:%S', level=level)
 
-        if authenticate:
-            self._session_hash = self._authenticate(
-                self.config.get('FAKE_SECTION', 'username'),
-                self.config.get('FAKE_SECTION', 'password'))
-
         # Set up the cache directories
         self.cache_dir = self.config.get('FAKE_SECTION', 'cachedir')
         self.dataset_cache_dir = os.path.join(self.cache_dir, "datasets")
@@ -189,50 +167,8 @@ def __init__(self, cache_directory=None, username=None, password=None,
             if not os.path.exists(dir_) and not os.path.isdir(dir_):
                 os.mkdir(dir_)
 
-    def _authenticate(self, username, password):
-        # Check the username
-        if username is None:
-            raise ValueError("No username specified.")
-        elif not is_string(username):
-            raise ValueError("Username must be of type string.")
-        elif not username:
-            raise ValueError("No value for argument username specified.")
-
-        # Check the password
-        if password is None:
-            raise ValueError("No password specified.")
-        elif not is_string(password):
-            raise ValueError("Password must be of type string.")
-        elif not password:
-            raise ValueError("No value for argument password specified.")
-
-        m = hashlib.md5()
-        m.update(password.encode('utf-8'))
-        md5 = m.hexdigest()
-
-        # TODO: catch possible exceptions
-        data = {'username': username, 'password': md5}
-        return_code, xml_string = self._perform_api_call(
-            "openml.authenticate", data=data, add_authentication=False)
-
-        xml_dict = xmltodict.parse(xml_string)
-        if xml_dict.get('oml:authenticate'):
-            session_hash = xml_dict['oml:authenticate']['oml:session_hash']
-            return session_hash
-        elif xml_dict.get('oml:error'):
-            error_code = xml_dict['oml:error']['oml:code']
-            if error_code == '252':
-                raise AuthentificationError(
-                    "Authentication failed. The username and password did not "
-                    "match any record in the database.")
-            else:
-                OpenMLServerError(
-                    "Unexpected server response code %d with response"
-                    " message %s" % (return_code, xml_string))
-
     def _parse_config(self):
-        defaults = {'username': '',
-                    'password': '',
+        defaults = {'apikey': '',
                     'server': OPENML_URL,
                     'verbosity': 0,
                     'cachedir': os.path.expanduser('~/.openml/cache'),
@@ -438,7 +374,7 @@ def get_dataset_list(self):
             these are also returned.
         """
         # TODO add proper error handling here!
-        return_code, xml_string = self._perform_api_call("openml.data")
+        return_code, xml_string = self._perform_api_call("data/list/")
         datasets_dict = xmltodict.parse(xml_string)
 
         # Minimalistic check if the XML is useful
@@ -553,7 +489,7 @@ def download_dataset_description(self, did):
         except (OpenMLCacheException):
             try:
                 return_code, dataset_xml = self._perform_api_call(
-                    "openml.data.description", data_id=did)
+                    "data/%d" % did)
             except (URLError, UnicodeEncodeError) as e:
                 # TODO logger.debug
                 self._remove_dataset_chache_dir(did)
@@ -568,7 +504,7 @@ def download_dataset_description(self, did):
                 "oml:data_set_description"]
         except Exception as e:
             # TODO logger.debug
-            self._remove_dataset_chache_dir()
+            self._remove_dataset_chache_dir(did)
             print("Dataset ID", did)
             raise e
 
@@ -613,7 +549,7 @@ def download_dataset_features(self, did):
         except (OSError, IOError):
             try:
                 return_code, features_xml = self._perform_api_call(
-                    "openml.data.features", data_id=did)
+                    "data/features/%d" % did)
             except (URLError, UnicodeEncodeError) as e:
                 # TODO logger.debug
                 print(e)
@@ -637,7 +573,7 @@ def download_dataset_qualities(self, did):
         qualities_file = os.path.join(did_cache_dir, "qualities.xml")
         try:
             return_code, qualities_xml = self._perform_api_call(
-                "openml.data.qualities", data_id=did)
+                "data/qualities/%d" % did)
         except (URLError, UnicodeEncodeError) as e:
             # TODO logger.debug
             print(e)
@@ -725,7 +661,7 @@ def get_task_list(self, task_type_id=1):
                              "cast to an Integer.")
 
         return_code, xml_string = self._perform_api_call(
-            "openml.tasks", task_type_id=task_type_id)
+            "task/list/%d" % task_type_id)
         tasks_dict = xmltodict.parse(xml_string)
         # Minimalistic check if the XML is useful
         assert tasks_dict['oml:tasks']['@xmlns:oml'] == \
@@ -775,7 +711,7 @@ def download_task(self, task_id):
 
             try:
                 return_code, task_xml = self._perform_api_call(
-                    "openml.task.search", task_id=task_id)
+                    "task/%d" % task_id)
             except (URLError, UnicodeEncodeError) as e:
                 print(e)
                 raise e
@@ -819,7 +755,7 @@ def _create_task_from_xml(self, xml):
             text = parameter.get("#text", "")
             estimation_parameters[name] = text
 
-        return Task(
+        return OpenMLTask(
             dic["oml:task_id"], dic["oml:task_type"],
             inputs["source_data"]["oml:data_set"]["oml:data_set_id"],
             inputs["source_data"]["oml:data_set"]["oml:target_feature"],
@@ -878,21 +814,38 @@ def _create_task_cache_dir(self, task_id):
         return task_cache_dir
 
     def _perform_api_call(self, call, data=None, filePath=None, add_authentication=True,
-                          **kwargs):
-        # TODO: do input validation!
-        url = self.config.get("FAKE_SECTION", "server") + "/api/?f="
-        url += "" + call
-        if kwargs:
-            for key in kwargs:
-                url += "&" + key + "=" + str(kwargs[key])
-        # TODO logger.debug(url)
+        """Perform an API call at the OpenML server.
         return self._read_url(url, data=data, filePath=filePath,
-                              add_authentication=add_authentication)
 
     def _read_url(self, url, add_authentication=False, data=None, filePath=None):
+
+        Parameters
+        ----------
+        call : str
+            The API call. For example data/list
+        data : dict (default=None)
+            Dictionary containing data which will be sent to the OpenML
+            server via a POST request.
+        **kwargs
+            Further arguments which are appended as GET arguments.
+
+        Returns
+        -------
+        return_code : int
+            HTTP return code
+        return_value : str
+            Return value of the OpenML server
+        """
+        url = self.config.get("FAKE_SECTION", "server")
+        if not url.endswith("/"):
+            url += "/"
+        url += call
+        return self._read_url(url, data=data)
+
+    def _read_url(self, url, data=None):
         if data is None:
             data = {}
-        if add_authentication:
+        data['session_hash'] = self.config.get('FAKE_SECTION', 'apikey')
             data['session_hash'] = self._session_hash
 
         if filePath is not None:

openml/entities/task.py

@@ -9,7 +9,7 @@
         import pickle
 
 
-class Task(object):
+class OpenMLTask(object):
     def __init__(self, task_id, task_type, data_set_id, target_feature,
                  estimation_procedure_type, data_splits_url,
                  estimation_parameters, evaluation_measure,cost_matrix, api_connector):

requirements.txt

@@ -0,0 +1,3 @@
+numpy>=1.6.2
+scipy>=0.13.3
+numpydoc

setup.py

@@ -11,11 +11,12 @@
                  version="0.0.1dev",
                  packages=setuptools.find_packages(),
                  package_data={'': ['*.txt', '*.md']},
-                 install_requires=["liac-arff>=2.1.0",
+                 install_requires=["liac-arff>=2.1.1dev",
                                    "numpy>=1.6.2",
                                    "scipy>=0.13.3",
                                    "xmltodict",
-                                   "nose"],
+                                   "nose",
+                                   "numpydoc"],
                  test_suite="nose.collector",
                  classifiers=['Intended Audience :: Science/Research',
                               'Intended Audience :: Developers',
@@ -31,4 +32,7 @@
                               'Programming Language :: Python :: 3',
                               'Programming Language :: Python :: 3.3',
                               'Programming Language :: Python :: 3.4',
-                 ])
+                 ],
+                 dependency_links=[
+                     "http://github.com/mfeurer/liac-arff/archive/master.zip"
+                     "#egg=liac-arff-2.1.1dev"])

source/api.rst

@@ -5,4 +5,10 @@
 APIs
 ****
 
-.. autoclass:: openml.apiconnector.APIConnector
+.. autoclass:: openml.apiconnector.APIConnector
+
+.. autoclass:: openml.entities.dataset.OpenMLDataset
+
+.. autoclass:: openml.entities.task.OpenMLTask
+
+.. autoclass:: openml.entities.split.OpenMLSplit

source/conf.py

@@ -12,8 +12,25 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-import sys
 import os
+import sys
+
+if sys.version_info[0] >= 3:
+    from unittest.mock import MagicMock
+else:
+    from mock import MagicMock
+
+
+class Mock(MagicMock):
+    @classmethod
+    def __getattr__(cls, name):
+        return Mock()
+
+
+MOCK_MODULES = ['pygtk', 'gtk', 'gobject', 'argparse', 'numpy', 'pandas',
+                'scipy', 'scipy.sparse', 'scipy.io', 'scipy.io.arff']
+sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
+
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
@@ -33,10 +50,12 @@
 # ones.
 extensions = [
     'sphinx.ext.autodoc',
+    'sphinx.ext.autosummary',
     'sphinx.ext.doctest',
     'sphinx.ext.coverage',
     'sphinx.ext.mathjax',
     'sphinx.ext.ifconfig',
+    'numpydoc'
 ]
 
 # Add any paths that contain templates here, relative to this directory.