added getRemainingLimit()

Author: joeri-vv

README.md

@@ -45,6 +45,15 @@ This is the python wrapper for the Bitvavo API. This project can be used to buil
 pip install python-bitvavo-api
 ```
 
+## Rate Limiting
+
+Bitvavo uses a weight based rate limiting system, with an allowed limit of 1000 per IP or API key each minute. Please inspect each endpoint in the [documentation](https://docs.bitvavo.com/) to see the weight. Failure to respect the rate limit will result in an IP or API key ban.
+Since the remaining limit is returned in the header on each REST request, the remaining limit is tracked locally and can be requested through:
+```
+limit = bitvavo.getRemainingLimit()
+```
+The websocket functions however do not return a remaining limit, therefore the limit is only updated locally once a ban has been issued.
+
 ## REST requests
 
 The general convention used in all functions (both REST and websockets), is that all optional parameters are passed as an dictionary, while required parameters are passed as separate values. Only when [placing orders](https://github.com/bitvavo/python-bitvavo-api#place-order) some of the optional parameters are required, since a limit order requires more information than a market order. The returned responses are all converted to a dictionary as well, such that `response['<key>'] = '<value>'`.
@@ -1060,7 +1069,7 @@ All requests which can be done through REST requests can also be performed over
 
 ### Getting started
 
-The websocket object should be intialised through the `newWebsocket()` function. After which a callback for the errors should be set. After this any desired function can be called. Finally the main thread should be kept alive for as long as you want the socket to stay open. This can be achieved through a simple `while(True)` loop.
+The websocket object should be intialised through the `newWebsocket()` function. After which a callback for the errors should be set. After this any desired function can be called. Finally the main thread should be kept alive for as long as you want the socket to stay open. This can be achieved through a simple `while()` loop, where the remaining limit is checked. This is in case a ban has been issued, otherwise the websocket object will keep trying to reconnect, while our servers keep closing the connection.
 
 ```python
 def errorCallback(error):
@@ -1075,9 +1084,11 @@ websocket.setErrorCallback(errorCallback)
 # Call functions here, like:
 # websocket.time(ownCallback)
 
+limit = bitvavo.getRemainingLimit()
 try:
-  while(True):
-    time.sleep(2)
+  while(limit > 0):
+    time.sleep(0.5)
+    limit = bitvavo.getRemainingLimit()
 except KeyboardInterrupt:
   websocket.closeSocket()
 ```

pythonBitvavoApi/bitvavo.py

@@ -89,6 +89,25 @@ def processLocalBook(ws, message):
 
   ws.callbacks['subscriptionBookUser'][market](ws.localBook[market])
 
+class rateLimitThread (threading.Thread):
+  def __init__(self, reset, bitvavo):
+    self.timeToWait = reset
+    self.bitvavo = bitvavo
+    threading.Thread.__init__(self)
+
+  def waitForReset(self, waitTime):
+    time.sleep(waitTime)
+    if (time.time() < self.bitvavo.rateLimitReset):
+      self.bitvavo.rateLimitRemaining = 1000
+      debugToConsole('Ban should have been lifted, resetting rate limit to 1000.')
+    else:
+      timeToWait = (self.bitvavo.rateLimitReset / 1000) - time.time()
+      debugToConsole('Ban took longer than expected, sleeping again for', timeToWait, 'seconds.')
+      self.waitForReset(timeToWait)
+
+  def run(self):
+    self.waitForReset(self.timeToWait)
+
 
 class receiveThread (threading.Thread):
   def __init__(self, ws, wsObject):
@@ -103,8 +122,8 @@ def run(self):
         self.wsObject.reconnect = True
         self.wsObject.authenticated = False
         time.sleep(self.wsObject.reconnectTimer)
-        self.wsObject.reconnectTimer = self.wsObject.reconnectTimer * 2
         debugToConsole("we have just set reconnect to true and have waited for " + str(self.wsObject.reconnectTimer))
+        self.wsObject.reconnectTimer = self.wsObject.reconnectTimer * 2
     except KeyboardInterrupt:
       debugToConsole("We caught keyboard interrupt in the websocket thread.")
 
@@ -114,6 +133,8 @@ def __init__(self, options = {}):
     self.ACCESSWINDOW = None
     self.APIKEY = ''
     self.APISECRET = ''
+    self.rateLimitRemaining = 1000
+    self.rateLimitReset = 0
     global debugging
     debugging = False
     for key in options:
@@ -129,9 +150,38 @@ def __init__(self, options = {}):
       self.ACCESSWINDOW = 10000
     self.base = 'https://api.bitvavo.com/v2'
 
+  def getRemainingLimit(self):
+    return self.rateLimitRemaining
+
+  def updateRateLimit(self, response):
+    if 'errorCode' in response:
+      if (response['errorCode'] == 105):
+        self.rateLimitRemaining = 0
+        self.rateLimitReset = int(response['error'].split(' at ')[1].split('.')[0])
+        timeToWait = (self.rateLimitReset / 1000) - time.time()
+        if(not hasattr(self, 'rateLimitThread')):
+          self.rateLimitThread = rateLimitThread(timeToWait, self)
+          self.rateLimitThread.daemon = True
+          self.rateLimitThread.start()
+      # setTimeout(checkLimit, timeToWait)
+    if ('bitvavo-ratelimit-remaining' in response):
+      self.rateLimitRemaining = int(response['bitvavo-ratelimit-remaining'])
+    if ('bitvavo-ratelimit-resetat' in response):
+      self.rateLimitReset = int(response['bitvavo-ratelimit-resetat'])
+      timeToWait = (self.rateLimitReset / 1000) - time.time()
+      if(not hasattr(self, 'rateLimitThread')):
+          self.rateLimitThread = rateLimitThread(timeToWait, self)
+          self.rateLimitThread.daemon = True
+          self.rateLimitThread.start()
+
+
   def publicRequest(self, url):
     debugToConsole("REQUEST: " + url)
     r = requests.get(url)
+    if('error' in r.json()):
+      self.updateRateLimit(r.json())
+    else:
+      self.updateRateLimit(r.headers)
     return r.json()
 
   def privateRequest(self, endpoint, postfix, body = {}, method = 'GET'):
@@ -147,20 +197,17 @@ def privateRequest(self, endpoint, postfix, body = {}, method = 'GET'):
     debugToConsole("REQUEST: " + url)
     if(method == 'GET'):
       r = requests.get(url, headers = headers)
-      print(r)
-      return r.json()
     elif(method == 'DELETE'):
       r = requests.delete(url, headers = headers)
-      print(r)
-      return r.json()
     elif(method == 'POST'):
       r = requests.post(url, headers = headers, json = body)
-      print(r)
-      return r.json()
     elif(method == 'PUT'):
       r = requests.put(url, headers = headers, json = body)
-      print(r)
-      return r.json()
+    if('error' in r.json()):
+      self.updateRateLimit(r.json())
+    else:
+      self.updateRateLimit(r.headers)
+    return r.json()
 
   def time(self):
     return self.publicRequest((self.base + '/time'))
@@ -278,10 +325,10 @@ def withdrawalHistory(self, options):
     return self.privateRequest('/withdrawalHistory', postfix, {}, 'GET')
 
   def newWebsocket(self):
-    return Bitvavo.websocket(self.APIKEY, self.APISECRET, self.ACCESSWINDOW)
+    return Bitvavo.websocket(self.APIKEY, self.APISECRET, self.ACCESSWINDOW, self)
 
   class websocket:
-    def __init__(self, APIKEY, APISECRET, ACCESSWINDOW):
+    def __init__(self, APIKEY, APISECRET, ACCESSWINDOW, bitvavo):
       self.APIKEY = APIKEY
       self.APISECRET = APISECRET
       self.ACCESSWINDOW = ACCESSWINDOW
@@ -290,6 +337,7 @@ def __init__(self, APIKEY, APISECRET, ACCESSWINDOW):
       self.keepAlive = True
       self.reconnect = False
       self.reconnectTimer = 0.1
+      self.bitvavo = bitvavo
 
       self.subscribe()
 
@@ -336,6 +384,8 @@ def on_message(ws, msg):
       callbacks = ws.callbacks
 
       if('error' in msg):
+        if (msg['errorCode'] == 105):
+          ws.bitvavo.updateRateLimit(msg)
         if('error' in callbacks):
           callbacks['error'](msg)
         else:
@@ -455,7 +505,7 @@ def checkReconnect(self):
     def on_open(self):
       now = int(time.time()*1000)
       self.open = True
-      self.reconnectTimer = 0.1
+      self.reconnectTimer = 0.5
       if(self.APIKEY != ''):
         self.doSend(self.ws, json.dumps({ 'window':str(self.ACCESSWINDOW), 'action': 'authenticate', 'key': self.APIKEY, 'signature': createSignature(now, 'GET', '/websocket', {}, self.APISECRET), 'timestamp': now }))
       if self.reconnect:

testApi.py

@@ -1,4 +1,4 @@
-from pythonBitvavoApi.bitvavo import Bitvavo
+from bitvavo import Bitvavo
 import sys
 import signal
 import time
@@ -24,8 +24,11 @@ def main():
   testWebsockets(bitvavo)
 
 def testREST(bitvavo):
+  limit = bitvavo.getRemainingLimit()
+  print('Remaining ratelimit is', limit)
+
   response = bitvavo.time()
-  # print('Current time:', response['time'])
+  print(response)
 
   # response = bitvavo.markets({})
   # for market in response:
@@ -150,9 +153,11 @@ def testWebsockets(bitvavo):
   # websocket.subscriptionBookUpdate('BTC-EUR', callback)
 
   # websocket.subscriptionBook('BTC-EUR', callback)
+  limit = bitvavo.getRemainingLimit()
   try:
-    while(True):
-      time.sleep(2)
+    while(limit > 0):
+      time.sleep(0.5)
+      limit = bitvavo.getRemainingLimit()
   except KeyboardInterrupt:
     websocket.closeSocket()