Skip to content

Commit 9a00067

Browse files
zardaloopzardaloop
committed
Merge branch 'feature/AddingFlowFunction' into develop
* feature/AddingFlowFunction: updated the source progress Implemented posting run. Added upload flow function
2 parents 36ef910 + 61f54da commit 9a00067

3 files changed

Lines changed: 79 additions & 52 deletions

File tree

‎openml/apiconnector.py‎

Lines changed: 43 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
from collections import OrderedDict
2+
import hashlib
23
import logging
34
import os
45
import re
@@ -816,6 +817,7 @@ def _create_task_cache_dir(self, task_id):
816817
pass
817818
return task_cache_dir
818819

820+
def _perform_api_call(self, call, data=None, file_dictionary=None, add_authentication=True):
819821
############################################################################
820822
# Runs
821823
def get_runs_list(self, task_id=None, flow_id=None, setup_id=None):
@@ -1025,33 +1027,37 @@ def _read_url(self, url, add_authentication=False, data=None, filePath=None):
10251027
if not url.endswith("/"):
10261028
url += "/"
10271029
url += call
1028-
return self._read_url(url, data=data, file_path=file_path)
1030+
return self._read_url(url, data=data, file_dictionary=file_dictionary)
10291031

1030-
def _read_url(self, url, data=None, file_path=None):
1032+
def _read_url(self, url, data=None, file_dictionary=None):
10311033
if data is None:
10321034
data = {}
10331035
data['api_key'] = self.config.get('FAKE_SECTION', 'apikey')
10341036

1035-
if file_path is not None:
1036-
if os.path.isabs(file_path):
1037-
try:
1038-
decoder = arff.ArffDecoder()
1039-
except:
1040-
raise "The file you provided is not a valid arff file"
1041-
1042-
fileElement={'dataset': open(file_path, 'rb')}
1043-
data['description']= data.get('description')
1044-
data.pop('dataset', None)
1045-
1046-
try:
1047-
response = requests.post(url, data=data, files=fileElement)
1037+
if file_dictionary is not None:
1038+
file_elements = {}
1039+
for key, path in file_dictionary.items():
1040+
if os.path.isabs(path) and os.path.exists(path):
1041+
try:
1042+
if key is 'dataset':
1043+
decoder = arff.ArffDecoder()
1044+
with open(path) as fh:
1045+
decoder.decode(fh, encode_nominal=True)
1046+
except:
1047+
raise ValueError("The file you have provided is not a valid arff file")
1048+
1049+
file_elements[key] = open(path, 'rb')
10481050
except URLError as error:
10491051
print(error)
10501052

1053+
else:
1054+
raise ValueError("File doesn't exist")
1055+
1056+
response = requests.post(url, data=data, files=file_elements)
10511057
return response.status_code, response
1052-
else:
1053-
raise "File doesn't exists"
10541058

1059+
except URLError as error:
1060+
print(error)
10551061
else:
10561062
data = urlencode(data)
10571063
data = data.encode('utf-8')
@@ -1096,38 +1102,41 @@ def _read_url(self, url, data=None, file_path=None):
10961102
def upload_dataset(self, description, file_path=None):
10971103
try:
10981104
data = {'description': description}
1099-
return_code, dataset_xml = self._perform_api_call(
1100-
"/data/", data=data, file_path=file_path)
1105+
if file_path is not None:
1106+
return_code, dataset_xml = self._perform_api_call("/data/",data=data, file_dictionary={'dataset': file_path})
11011107

11021108
except URLError as e:
11031109
# TODO logger.debug
11041110
print(e)
11051111
raise e
11061112
return return_code, dataset_xml
11071113

1108-
def upload_flow(self, description, binary, source):
1114+
def upload_flow(self, description, file_path=None):
11091115
try:
1110-
data = {'description': description, 'binary': binary, 'source': source}
1111-
return_code, dataset_xml = self._perform_api_call(
1112-
"openml.implementation.upload", data=data)
1116+
data = {'description': description}
1117+
return_code, dataset_xml = self._perform_api_call("/flow/", data=data, file_dictionary={'source': file_path})
11131118

11141119
except URLError as e:
11151120
# TODO logger.debug
11161121
print(e)
11171122
raise e
11181123
return return_code, dataset_xml
11191124

1120-
def upload_run(self, description, files):
1121-
try:
1122-
data ={'description': description}
1123-
for key, value in files:
1124-
data[key] = value
1125+
def upload_run(self, files):
1126+
file_dictionary = {}
1127+
if 'predictions' in files:
1128+
try:
1129+
for key, value in files.items():
1130+
file_dictionary[key] = value
11251131

1126-
return_code, dataset_xml = self._perform_api_call("openml.run.upload", data=data)
1132+
return_code, dataset_xml = self._perform_api_call("/run/", file_dictionary=file_dictionary)
1133+
1134+
except URLError as e:
1135+
# TODO logger.debug
1136+
print(e)
1137+
raise e
1138+
return return_code, dataset_xml
1139+
else:
1140+
raise ValueError("prediction files doesn't exist")
11271141

1128-
except URLError as e:
1129-
# TODO logger.debug
1130-
print(e)
1131-
raise e
1132-
return return_code, dataset_xml
11331142

‎source/progress.rst‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ API call implemented tested properly test
1616
/data/list/tag/{tag}
1717
/data/{data_id} yes yes
1818
/data/delete/
19-
/data/upload/
19+
/data/upload/ yes yes
2020
/data/features/{data_id} yes yes
2121
/data/features/upload/
2222
/data/qualities/{data_id} yes yes
@@ -36,12 +36,12 @@ API call implemented tested properly test
3636
/flow/tag
3737
/flow/untag
3838
/flow/{flow_id}
39-
/flow/
39+
/flow/ yes yes
4040
/flow/exists/{name,ext_version}
4141
/flow/owned
4242
/run/list yes yes
4343
/run/{run_id} yes yes
44-
/run
44+
/run yes yes
4545
/run/tag
4646
/run/untag
4747
/run/evaluate

‎tests/test_apiconnector.py‎

Lines changed: 33 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,17 @@
55
import shutil
66
import sys
77

8+
89
if sys.version_info[0] >= 3:
910
from unittest import mock
11+
from urllib.request import urlopen
12+
from urllib.parse import urlencode
13+
from urllib.error import URLError
1014
else:
1115
import mock
16+
from urllib import urlencode, urlopen
17+
from urllib2 import URLError, urlopen
18+
1219

1320
from openml.util import is_string
1421

@@ -39,13 +46,11 @@ def setUp(self):
3946
os.chdir(self.workdir)
4047

4148
self.cached = True
42-
49+
self.connector = APIConnector(cache_directory=self.workdir)
4350
try:
4451
apikey = os.environ['OPENMLAPIKEY']
4552
except:
4653
apikey = None
47-
self.connector = APIConnector(cache_directory=self.workdir,
48-
apikey=apikey)
4954

5055
def tearDown(self):
5156
os.chdir(self.cwd)
@@ -269,7 +274,7 @@ def check_flow(flow):
269274
def test_upload_dataset(self):
270275

271276
dataset = self.connector.download_dataset(3)
272-
filePath = os.path.join(self.connector.dataset_cache_dir, "3", "dataset.arff")
277+
file_path = os.path.join(self.connector.dataset_cache_dir, "3", "dataset.arff")
273278

274279
description = """ <oml:data_set_description xmlns:oml="http://openml.org/openml">
275280
<oml:name>anneal</oml:name>
@@ -281,7 +286,7 @@ def test_upload_dataset(self):
281286
<oml:md5_checksum></oml:md5_checksum>
282287
</oml:data_set_description>
283288
"""
284-
return_code, dataset_xml = self.connector.upload_dataset(description, filePath)
289+
return_code, dataset_xml = self.connector.upload_dataset(description, file_path)
285290
self.assertEqual(return_code, 200)
286291

287292
def test_upload_dataset_with_url(self):
@@ -294,20 +299,33 @@ def test_upload_dataset_with_url(self):
294299
<oml:url>http://expdb.cs.kuleuven.be/expdb/data/uci/nominal/iris.arff</oml:url>
295300
</oml:data_set_description>
296301
"""
297-
return_code, dataset_xml = self.connector.upload_dataset (description)
302+
return_code, dataset_xml = self.connector.upload_dataset(description)
298303
self.assertEqual(return_code, 200)
299304

300305
def test_upload_flow(self):
306+
file_path = os.path.join(self.connector.dataset_cache_dir,"uploadflow.txt")
307+
file = open(file_path, "w")
308+
file.write("Testing upload flow")
309+
file.close()
310+
description = '''<oml:flow xmlns:oml="http://openml.org/openml"><oml:name>Test</oml:name><oml:description>description</oml:description> </oml:flow>'''
311+
return_code, dataset_xml = self.connector.upload_flow(description, file_path)
312+
self.assertEqual(return_code, 200)
301313

302-
description = """ <oml:data_set_description xmlns:oml="http://openml.org/openml">
303-
<oml:name>UploadTestWithURL</oml:name>
304-
<oml:version>1</oml:version>
305-
<oml:description>test</oml:description>
306-
<oml:format>ARFF</oml:format>
307-
<oml:url>http://expdb.cs.kuleuven.be/expdb/data/uci/nominal/iris.arff</oml:url>
308-
</oml:data_set_description>
309-
"""
310-
return_code, dataset_xml = self.connector.upload_dataset (description)
314+
def test_upload_run(self):
315+
file = urlopen("http://www.openml.org/data/download/224/weka_generated_predictions1977525485999711307.arff")
316+
file_text = file.read()
317+
file_path = os.path.join(self.connector.dataset_cache_dir, "weka_generated_predictions1977525485999711307.arff")
318+
with open(file_path, "wb") as prediction_file:
319+
prediction_file.write(file_text)
320+
321+
description_text = '''<oml:run xmlns:oml="http://openml.org/openml"><oml:task_id>59</oml:task_id><oml:flow_id>67</oml:flow_id></oml:run>'''
322+
description_path = os.path.join(self.connector.dataset_cache_dir, "description.xml")
323+
with open(description_path, "w") as description_file:
324+
description_file.write(description_text)
325+
326+
file_dictionary = {'predictions': file_path, 'description': description_path}
327+
328+
return_code, dataset_xml = self.connector.upload_run(file_dictionary)
311329
self.assertEqual(return_code, 200)
312330

313331

0 commit comments

Comments
 (0)