Added own implementation

Author: tk-king

src/edgeml/Dataset.py

@@ -0,0 +1,58 @@
+import tempfile
+import h5py
+import numpy as np
+from edgeml.TimeSeries import TimeSeries
+from edgeml.Labeling import Labeling, Label
+from functools import reduce
+import pandas as pd 
+
+class Dataset():
+    def __init__(self, backendURL, readKey=None, writeKey=None):
+        self._backendURL = backendURL
+        self._readKey = readKey
+        self._writeKey = writeKey
+        
+        self._id = None
+        self.name = None
+        self.metaData = None
+        self.timeSeries = None
+        self.labelings = None
+
+    def parse(self, data, labelings):
+        self._id = data["_id"]
+        self.name = data["name"]
+        self.metaData = data["metaData"]
+        self.timeSeries = []
+        for ts in data["timeSeries"]:
+            tmp_timeSeries = TimeSeries(self._backendURL, self._id, self._readKey, self._writeKey)
+            tmp_timeSeries.parse(ts)
+            self.timeSeries.append(tmp_timeSeries)
+
+        self.labelings = []
+        label_name_map =  {label['_id']: label['name'] for entry in labelings for label in entry.get('labels', [])}
+
+        for labeling in data["labelings"]:
+            labeling["name"] = next(x["name"] for x in labelings if x["_id"] == labeling["labelingId"])
+            for label in labeling["labels"]:
+                label["name"] = label_name_map[label["type"]]
+            temp_labeling = Labeling()
+            temp_labeling.parse(labeling)
+            self.labelings.append(temp_labeling)
+
+    @property
+    def data(self):
+        df = reduce(lambda x,y: pd.merge(x,y, on='time', how='outer'), [x.data for x in self.timeSeries])
+        for labeling in self.labelings:
+            for label in labeling.labels:
+                if labeling.name not in df.columns:
+                    df[labeling.name] = ""
+                df.loc[(df['time'] >= label.start) & (df['time'] <= label.end), labeling.name] = label.name
+                
+
+
+
+        return df
+
+    def loadData(self):
+        for ts in self.timeSeries:
+            ts.loadData()

src/edgeml/Labeling.py

@@ -0,0 +1,42 @@
+class Labeling:
+    def __init__(self):
+        self._id = None
+        self.labels = None
+        self.name = None
+
+    def parse(self, data):
+        self._id = data["labelingId"]
+        self.name = data["name"]
+        self.labels = []
+        for label in data["labels"]:
+            tmp_label = Label()
+            tmp_label.parse(label)
+            self.labels.append(tmp_label)
+
+    def __str__(self):
+        return f"Labeling(_id={self._id}, labels={self.labels}, name={self.name})"
+
+    def __repr__(self):
+        return str(self)
+
+
+class Label:
+    def __init__(self):
+        self._id = None
+        self.start = None
+        self.end = None
+        self.type = None
+        self.name = None
+
+    def parse(self, data):
+        self._id = data["_id"]
+        self.start = data["start"]
+        self.end = data["end"]
+        self.type = data["type"]
+        self.name = data["name"]
+
+    def __str__(self):
+        return f"Label(_id={self._id}, start={self.start}, end={self.end}, type={self.type}, name={self.name})"
+
+    def __repr__(self):
+        return str(self)

src/edgeml/Project.py

@@ -0,0 +1,28 @@
+import requests as req
+from edgeml.consts import getProjectEndpoint
+from edgeml.Dataset import Dataset
+
+
+class Project():
+
+    def __init__(self, backendURL, readKey=None, writeKey=None):
+        self.backendURL = backendURL
+        self._readKey=readKey
+        self._writeKey=writeKey
+        res = req.get(backendURL + getProjectEndpoint + readKey)
+        if res.status_code == 403:
+            raise RuntimeError("Invalid key")
+        elif res.status_code >= 300:
+            raise RuntimeError(res.reason)
+        self.datasets = []
+        res_data = res.json()
+        datasets = res_data["datasets"]
+        self.labeligns = res_data["labelings"]
+        for d in datasets:
+            tmp_dataset = Dataset(backendURL, self._readKey, self._writeKey)
+            tmp_dataset.parse(d, self.labeligns)
+            self.datasets.append(tmp_dataset)
+
+    def loadData(self):
+        for d in self.datasets:
+            d.loadData()

src/edgeml/TimeSeries.py

@@ -0,0 +1,55 @@
+import tempfile
+import h5py
+import numpy as np
+from edgeml.consts import getProjectEndpoint
+import requests as req
+import pandas as pd
+
+class SamplingRate:
+    def __init__(self, mean, var):
+        self.mean = mean
+        self.var = var
+
+
+class TimeSeries:
+    def __init__(self, backendURL, datasetId, readKey=None, writeKey=None):
+        self._backendURL = backendURL
+        self._datasetId = datasetId
+        self._readKey = readKey
+        self._writeKey = writeKey
+        self._id = None
+        self.name = None
+        self.start = None
+        self.end = None
+        self.unit = None
+        self._data = None
+        self.samplingRate = None
+        self.length = None
+
+    def parse(self, data):
+        self._id = data["_id"]
+        self.name = data["name"]
+        self.start = data["start"]
+        self.end = data["end"]
+        self.unit = data["unit"]
+        self.samplingRate = SamplingRate(data["samplingRate"]["mean"], data["samplingRate"]["var"])
+        self.length = data["length"]
+
+    @property
+    def data(self):
+        if self._data is None:
+            raise Exception("You need to load the data first. Call loadData on the project, dataset, or time-series level.")
+        return self._data
+
+    @data.setter
+    def data(self, value):
+        self._data = value
+
+    def loadData(self) -> pd.DataFrame:
+        res = req.get(self._backendURL + getProjectEndpoint + self._readKey + "/" + self._datasetId + "/" + self._id)
+        with tempfile.NamedTemporaryFile(suffix=".h5", delete=False) as temp_file:
+            temp_file.write(res.content)
+            with h5py.File(temp_file.name, "r") as hf:
+                time_array = np.array(hf["time"])
+                data_array = np.array(hf["data"])
+                self.data = pd.DataFrame({"time": time_array, self.name: data_array})

src/edgeml/consts.py

@@ -0,0 +1,5 @@
+
+uploadDataset = "/api/deviceapi/uploadDataset",
+initDatasetIncrement = "/ds/api/dataset/init/",
+addDatasetIncrement = "/ds/api/dataset/append/"
+getProjectEndpoint = "/ds/api/project/"

src/edgeml/edgeml.py

@@ -1,114 +1,32 @@
-from typing import List
 import requests as req
-import pandas as pd
-import time as timelib
-from functools import reduce
-
-# TODO add typing
-
-uploadDataset = "/api/deviceApi/uploadDataset"
-initDatasetIncrement = "/api/deviceApi/initDatasetIncrement"
-addDatasetIncrement = "/api/deviceApi/addDatasetIncrement"
-addDatasetIncrementBatch = "/api/deviceApi/addDatasetIncrementBatch"
-getProjectEndpoint = "/api/deviceApi/getProject"
-
-#
-#  Uploads a whole dataset to a specific project
-#  @param {string} url - The url of the backend server
-#  @param {string} key - The Device-Api-Key
-#  @param {object} dataset - The dataset to upload
-#  @returns A Promise indicating success or failure
-#
-
-def sendDataset(url: str, key: str, dataset: dict):
-    try:
-        res = req.post(url + uploadDataset, json = {"key": key, "payload": dataset})
-    except req.exceptions.RequestException:
-        raise "error" #TODO
-
-#
-# Returns the all datasets and labels belonging to a project
-# Can be used for further processing
-# @param {string} url - The url of the backend server
-# @param {string} key - The Device-Api-Key
-#
-
-def getProject(url: str, key: str):
-    print('fetching project...')
-    res = req.post(url + getProjectEndpoint, json = {"key": key})
-    if res.ok:
-        return res.json()
-    if res.status_code == 403:
-        raise RuntimeError("Invalid key")
-    raise RuntimeError(res.reason)
-
-def __extractLabels(dataset, labeling: str=None):
-    labelingSets = dataset['labels']
-    matchedSet = None
-    for labelingSet in labelingSets:
-        if labelingSet and labelingSet[0] and (labelingSet[0]['labelingName'] == labeling or labeling == None):
-            labeling = labelingSet[0]['labelingName']
-            matchedSet = labelingSet
-            break
-    if matchedSet == None:
-        return (None, None, None)
-    labelSet = {} # stores different start and end times (intervals) belonging to a label
-    labelIds = {} # assing distinct ids to labels, required for training with data
-    labelId = 0
-    for label in labelingSet:
-        name = label['name']
-        start = label['start']
-        end = label['end']
-        if not name in labelSet:
-            labelSet[name] = []
-            labelIds[name] = labelId        # assign id to the label
-            labelId = labelId + 1
-        labelSet[name].append((start, end)) # add interval to the label
-    return (labeling, labelSet, labelIds)
-
-#
-# Returns a list of Pandas.DataFrames generated from the datasets in the project
-# Each dataframe corresponds to a single dataset in the project
-# For each dataset only with the given labeling labeled parts are included in the dataframes
-# If no labeling is provided, first labeling with a valid label on part of the dataset will be used for that dataset
-# In this case different datasets may have different labelings as a result in the returned list
-# @param {string} url - The url of the backend server
-# @param {string} key - The Device-Api-Key
-# @param {string} labeling - Labeling used to generate the dataframes
-
-def getDataFrames(url: str, key: str, labeling: str=None) -> List[pd.DataFrame]:
-    datasets = getProject(url, key)['datasets']
-    df_project: List[pd.DataFrame] = []
-    for dataset in datasets:
-        (labeling, labelSet, labelIds) = __extractLabels(dataset, labeling)
-        if labelSet == None: # dataset is not labeled
-            continue
-        sensors = dataset['sensors']
-        df_dataset = []
-        for sensor in sensors:
-            sensorName = sensor['name']
-            data = sensor['data']
-            df_sensor = {'timestamp': [], 'label': [], sensorName: []}
-            for dataPoint in data:
-                timestamp = dataPoint['timestamp']
-                value = dataPoint['datapoint']
-                for label, intervals in labelSet.items():
-                    for start, end in intervals:
-                        if timestamp >= start and timestamp <= end:
-                            df_sensor['timestamp'].append(timestamp)
-                            df_sensor[sensorName].append(value)
-                            df_sensor['label'].append(label)
-                            # can break here if it is ensured that labels are not overlapping
-            df_sensor = pd.DataFrame(df_sensor)
-            df_dataset.append(df_sensor)
-        if not df_dataset:
-            continue
-        df_dataset = reduce(
-            lambda left, right: pd.merge(
-                left, right, on=['timestamp', 'label'], how='outer'), df_dataset
-            ).sort_values('timestamp').reset_index(drop=True)
-        df_project.append(df_dataset)
-    return df_project
+from edgeml.consts import getProjectEndpoint
+from edgeml.Dataset import Dataset
+import timelib
+
+
+class edgeml:
+
+    def __init__(self, backendURL, readKey=None, writeKey=None):
+        self.backendURL = backendURL
+        self._readKey=readKey
+        self._writeKey=writeKey
+        res = req.get(backendURL + getProjectEndpoint + readKey)
+        if res.status_code == 403:
+            raise RuntimeError("Invalid key")
+        elif res.status_code >= 300:
+            raise RuntimeError(res.reason)
+        self.datasets = []
+        res_data = res.json()
+        datasets = res_data["datasets"]
+        self.labeligns = res_data["labelings"]
+        for d in datasets:
+            tmp_dataset = Dataset(backendURL, self._readKey, self._writeKey)
+            tmp_dataset.parse(d, self.labeligns)
+            self.datasets.append(tmp_dataset)
+
+    def loadData(self):
+        for d in self.datasets:
+            d.loadData()
 
 #
 #  @param {string} url - The url of the backend server