From 64033f16a5fbe7da1b881b23931b36107d61e210 Mon Sep 17 00:00:00 2001 From: Rener Baffa da Silva Date: Fri, 5 Jul 2013 08:47:03 -0300 Subject: [PATCH 01/22] Adding CouchIPC dependences --- pox/ext/rfproxy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pox/ext/rfproxy.py b/pox/ext/rfproxy.py index 8c0388e3..9542af21 100644 --- a/pox/ext/rfproxy.py +++ b/pox/ext/rfproxy.py @@ -9,6 +9,7 @@ import rflib.ipc.IPC as IPC import rflib.ipc.MongoIPC as MongoIPC +import rflib.ipc.CouchIPC as CouchIPC from rflib.ipc.RFProtocol import * from rflib.ipc.RFProtocolFactory import RFProtocolFactory from rflib.defs import * From d94fb22a2bbcb02c55214cf0bcfbde7046d356a7 Mon Sep 17 00:00:00 2001 From: Rener Baffa da Silva Date: Fri, 5 Jul 2013 14:11:19 -0300 Subject: [PATCH 02/22] Creating README file with Couchbase instructions Creating README file with Couchbase instruction (first in portuguese) --- Danilo_Denis_Eloisa_Rener_Vinicius.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 Danilo_Denis_Eloisa_Rener_Vinicius.txt diff --git a/Danilo_Denis_Eloisa_Rener_Vinicius.txt b/Danilo_Denis_Eloisa_Rener_Vinicius.txt new file mode 100644 index 00000000..776cee87 --- /dev/null +++ b/Danilo_Denis_Eloisa_Rener_Vinicius.txt @@ -0,0 +1,10 @@ +1 - Instalar o CouchBase na máquina principal e criar o bucket com as configurações desejadas. + Available in: + +2 - Instalar o CouchBase nos servidores escolhendo a opção "Join Cluster Now", adicionando o usuário, senha e IP do servidor principal. + Available in: + +3 - Acessando o CouchBase pela URL (porta 8091), na aba "Server Nodes" clicar em "Rebalance", para balancear o uso de cada nó. + Available in: + +4 - Lembrar de configurar o arquivo *******************, no campo host, com a lista de todos os servidores no modo cluster. From be448f2079f3be6422b3a7901e1445be4ce79ffb Mon Sep 17 00:00:00 2001 From: Rener Baffa da Silva Date: Fri, 5 Jul 2013 14:13:48 -0300 Subject: [PATCH 03/22] Adding couchbase dependences Adding couchbase dependences --- rflib/ipc/CouchIPC.py | 131 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 rflib/ipc/CouchIPC.py diff --git a/rflib/ipc/CouchIPC.py b/rflib/ipc/CouchIPC.py new file mode 100644 index 00000000..4fecec68 --- /dev/null +++ b/rflib/ipc/CouchIPC.py @@ -0,0 +1,131 @@ +from couchbase import Couchbase + +import rflib.ipc.IPC as IPC + +FROM_FIELD = "from" +TO_FIELD = "to" +TYPE_FIELD = "type" +READ_FIELD = "read" +CONTENT_FIELD = "content" + +# 1 MB for the capped collection +CC_SIZE = 1048576 + +def put_in_envelope(from_, to, msg): + envelope = {} + + envelope[FROM_FIELD] = from_ + envelope[TO_FIELD] = to + envelope[READ_FIELD] = False + envelope[TYPE_FIELD] = msg.get_type() + + envelope[CONTENT_FIELD] = {} + for (k, v) in msg.to_dict().items(): + envelope[CONTENT_FIELD][k] = v + + return envelope + +def take_from_envelope(envelope, factory): + msg = factory.build_for_type(envelope[TYPE_FIELD]); + msg.from_dict(envelope[CONTENT_FIELD]); + return msg; + +def format_address(address): + try: + tmp = address.split(":") + if len(tmp) == 2: + return (tmp[0], int(tmp[1])) + elif len(tmp) == 1: + return (tmp[0],) + except: + raise ValueError, "Invalid address: " + str(address) + +class MongoIPCMessageService(IPC.IPCMessageService): + def __init__(self, address, db, id_, thread_constructor, sleep_function): + """Construct an IPCMessageService + + Args: + address: designates where the MongoDB instance is running. + db: is the database name to connect to on MongoDB. + id_: is an identifier to allow messages to be directed to the + appropriate recipient. + thread_constructor: function that takes 'target' and 'args' + parameters for the function to run and arguments to pass, and + return an object that has start() and join() functions. + sleep_function: function that takes a float and delays processing + for the specified period. + """ + self._db = db + self.address = format_address(address) + self._id = id_ + self._producer_connection = mongo.Connection(*self.address) + self._threading = thread_constructor + self._sleep = sleep_function + + def listen(self, channel_id, factory, processor, block=True): + worker = self._threading(target=self._listen_worker, + args=(channel_id, factory, processor)) + worker.start() + if block: + worker.join() + + def send(self, channel_id, to, msg): + self._create_channel(self._producer_connection, channel_id) + collection = self._producer_connection[self._db][channel_id] + collection.insert(put_in_envelope(self.get_id(), to, msg)) + return True + + def _listen_worker(self, channel_id, factory, processor): + connection = mongo.Connection(*self.address) + self._create_channel(connection, channel_id) + + collection = connection[self._db][channel_id] + cursor = collection.find({TO_FIELD: self.get_id(), READ_FIELD: False}, sort=[("_id", mongo.ASCENDING)]) + + while True: + for envelope in cursor: + msg = take_from_envelope(envelope, factory) + processor.process(envelope[FROM_FIELD], envelope[TO_FIELD], channel_id, msg); + collection.update({"_id": envelope["_id"]}, {"$set": {READ_FIELD: True}}) + self._sleep(0.05) + cursor = collection.find({TO_FIELD: self.get_id(), READ_FIELD: False}, sort=[("_id", mongo.ASCENDING)]) + + def _create_channel(self, connection, name): + db = connection[self._db] + try: + collection = mongo.collection.Collection(db, name, None, True, capped=True, size=CC_SIZE) + collection.ensure_index([("_id", mongo.ASCENDING)]) + collection.ensure_index([(TO_FIELD, mongo.ASCENDING)]) + # TODO: improve this catch. It should be more specific, but pymongo + # behavior doesn't match its documentation, so we are being dirty. + except: + pass + +class MongoIPCMessage(dict, IPC.IPCMessage): + def __init__(self, type_, **kwargs): + dict.__init__(self) + self.from_dict(kwargs) + self._type = type_ + + def get_type(self): + return self._type + + def from_dict(self, data): + for (k, v) in data.items(): + self[k] = v + + def from_bson(self, data): + data = bson.BSON.decode(data) + self.from_dict(data) + + def to_bson(self): + return bson.BSON.encode(self) + + def str(self): + string = "" + for (k, v) in self.items(): + string += str(k) + ": " + str(v) + "\n" + return string + + def __str__(self): + return IPC.IPCMessage.__str__(self) From 753b48bc1c08fd72e8a47f2e0f350743ae209617 Mon Sep 17 00:00:00 2001 From: Rener Baffa da Silva Date: Fri, 5 Jul 2013 16:51:07 -0300 Subject: [PATCH 04/22] Connection data changed. Connection data changed to new DB. --- rflib/ipc/CouchIPC.py | 45 +++++++++++++++++-------------------------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/rflib/ipc/CouchIPC.py b/rflib/ipc/CouchIPC.py index 4fecec68..9c024b0f 100644 --- a/rflib/ipc/CouchIPC.py +++ b/rflib/ipc/CouchIPC.py @@ -8,9 +8,6 @@ READ_FIELD = "read" CONTENT_FIELD = "content" -# 1 MB for the capped collection -CC_SIZE = 1048576 - def put_in_envelope(from_, to, msg): envelope = {} @@ -29,52 +26,44 @@ def take_from_envelope(envelope, factory): msg = factory.build_for_type(envelope[TYPE_FIELD]); msg.from_dict(envelope[CONTENT_FIELD]); return msg; - -def format_address(address): - try: - tmp = address.split(":") - if len(tmp) == 2: - return (tmp[0], int(tmp[1])) - elif len(tmp) == 1: - return (tmp[0],) - except: - raise ValueError, "Invalid address: " + str(address) -class MongoIPCMessageService(IPC.IPCMessageService): - def __init__(self, address, db, id_, thread_constructor, sleep_function): +class CouchIPCMessageService(IPC.IPCMessageService): + def __init__(self, bucket, hosts, thread_constructor, sleep_function): """Construct an IPCMessageService Args: - address: designates where the MongoDB instance is running. - db: is the database name to connect to on MongoDB. - id_: is an identifier to allow messages to be directed to the - appropriate recipient. + bucket: Bucket name of CouchBase. + hosts: list of hosts in the CouchBase cluster. thread_constructor: function that takes 'target' and 'args' parameters for the function to run and arguments to pass, and return an object that has start() and join() functions. sleep_function: function that takes a float and delays processing for the specified period. """ - self._db = db - self.address = format_address(address) - self._id = id_ - self._producer_connection = mongo.Connection(*self.address) + self._bucket = bucket + self._hosts = format_address(address) + self._producer_connection = Couchbase.connect( + bucket=_bucket, + hosts=_hosts + ) self._threading = thread_constructor self._sleep = sleep_function - + + #mudar... def listen(self, channel_id, factory, processor, block=True): worker = self._threading(target=self._listen_worker, args=(channel_id, factory, processor)) worker.start() if block: worker.join() - - def send(self, channel_id, to, msg): + + def send(self, to, msg): self._create_channel(self._producer_connection, channel_id) collection = self._producer_connection[self._db][channel_id] collection.insert(put_in_envelope(self.get_id(), to, msg)) return True + #mudar... def _listen_worker(self, channel_id, factory, processor): connection = mongo.Connection(*self.address) self._create_channel(connection, channel_id) @@ -89,7 +78,8 @@ def _listen_worker(self, channel_id, factory, processor): collection.update({"_id": envelope["_id"]}, {"$set": {READ_FIELD: True}}) self._sleep(0.05) cursor = collection.find({TO_FIELD: self.get_id(), READ_FIELD: False}, sort=[("_id", mongo.ASCENDING)]) - + + #mudar... def _create_channel(self, connection, name): db = connection[self._db] try: @@ -101,6 +91,7 @@ def _create_channel(self, connection, name): except: pass +#mudar... class MongoIPCMessage(dict, IPC.IPCMessage): def __init__(self, type_, **kwargs): dict.__init__(self) From a1e1c3bddfc14ef82ecdc17e8dccc6017e4896b3 Mon Sep 17 00:00:00 2001 From: Denis Wilson de Souza Oliveira Date: Fri, 5 Jul 2013 16:55:58 -0300 Subject: [PATCH 05/22] Add new Instructions --- Danilo_Denis_Eloisa_Rener_Vinicius.txt | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Danilo_Denis_Eloisa_Rener_Vinicius.txt b/Danilo_Denis_Eloisa_Rener_Vinicius.txt index 776cee87..e0508584 100644 --- a/Danilo_Denis_Eloisa_Rener_Vinicius.txt +++ b/Danilo_Denis_Eloisa_Rener_Vinicius.txt @@ -8,3 +8,20 @@ Available in: 4 - Lembrar de configurar o arquivo *******************, no campo host, com a lista de todos os servidores no modo cluster. + +OBS: + +Scripts +Instalação do CouchBase Server -> +Desenvolvimento -> dev.sh + +CouchBase + +Criar uma view e publicar no CouchBase + +CouchIPC.py +Alterações no para criar a conexão + + + + From cb4c3713d095c7e163ae553082427d0f8563d1bd Mon Sep 17 00:00:00 2001 From: Rener Baffa da Silva Date: Fri, 5 Jul 2013 17:47:12 -0300 Subject: [PATCH 06/22] Inserting data on CouchBase Changing rfproxy to insert data on Couchbase --- pox/ext/rfproxy.py | 6 ++---- rflib/defs.py | 4 ++++ rflib/ipc/CouchIPC.py | 10 +++++----- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pox/ext/rfproxy.py b/pox/ext/rfproxy.py index 9542af21..e31d87e9 100644 --- a/pox/ext/rfproxy.py +++ b/pox/ext/rfproxy.py @@ -63,10 +63,8 @@ def delete_dp(self, dp_id): netmask_prefix = lambda a: sum([bin(int(x)).count("1") for x in a.split(".", 4)]) -# TODO: add proper support for ID -ID = 0 -ipc = MongoIPC.MongoIPCMessageService(MONGO_ADDRESS, MONGO_DB_NAME, str(ID), - threading.Thread, time.sleep) +#ipc = MongoIPC.MongoIPCMessageService(MONGO_ADDRESS, MONGO_DB_NAME, str(ID), threading.Thread, time.sleep) +ipc = CouchIPC.CouchIPCMessageService ( COUCH_HOSTS, COUCH_BUCKET, threading.Thread, time.sleep) table = Table() # Logging diff --git a/rflib/defs.py b/rflib/defs.py index df310f1e..810c6a34 100644 --- a/rflib/defs.py +++ b/rflib/defs.py @@ -1,6 +1,10 @@ MONGO_ADDRESS = "192.169.1.1:27017" MONGO_DB_NAME = "db" +COUCH_HOSTS = ["rener-notebook.local", "elantra-CQ43.local", "rener-IPMH67.local"] +COUCH_BUCKET = "default" +COUCH_INITIAL_VALUE = 0 + RFCLIENT_RFSERVER_CHANNEL = "rfclient<->rfserver" RFSERVER_RFPROXY_CHANNEL = "rfserver<->rfproxy" diff --git a/rflib/ipc/CouchIPC.py b/rflib/ipc/CouchIPC.py index 9c024b0f..e24f3695 100644 --- a/rflib/ipc/CouchIPC.py +++ b/rflib/ipc/CouchIPC.py @@ -28,7 +28,7 @@ def take_from_envelope(envelope, factory): return msg; class CouchIPCMessageService(IPC.IPCMessageService): - def __init__(self, bucket, hosts, thread_constructor, sleep_function): + def __init__(self, hosts, bucket, thread_constructor, sleep_function): """Construct an IPCMessageService Args: @@ -40,14 +40,15 @@ def __init__(self, bucket, hosts, thread_constructor, sleep_function): sleep_function: function that takes a float and delays processing for the specified period. """ - self._bucket = bucket self._hosts = format_address(address) + self._bucket = bucket self._producer_connection = Couchbase.connect( bucket=_bucket, hosts=_hosts ) self._threading = thread_constructor self._sleep = sleep_function + self.set("key", COUCH_INITIAL_VALUE) #mudar... def listen(self, channel_id, factory, processor, block=True): @@ -58,9 +59,8 @@ def listen(self, channel_id, factory, processor, block=True): worker.join() def send(self, to, msg): - self._create_channel(self._producer_connection, channel_id) - collection = self._producer_connection[self._db][channel_id] - collection.insert(put_in_envelope(self.get_id(), to, msg)) + key = self.incr("key") + self.set( key, put_in_envelope(self.get_id(), to, msg) ) return True #mudar... From f9fa76cfa6c57ec36b782a9667a0aca2d8df3fed Mon Sep 17 00:00:00 2001 From: Rener Baffa da Silva Date: Fri, 5 Jul 2013 18:41:11 -0300 Subject: [PATCH 07/22] Change CouchIPC send. Change CouchIP send method to support connection with new db. --- pox/ext/rfproxy.py | 2 +- rflib/defs.py | 2 +- rflib/ipc/CouchIPC.py | 19 ++++++++++--------- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/pox/ext/rfproxy.py b/pox/ext/rfproxy.py index e31d87e9..d300878e 100644 --- a/pox/ext/rfproxy.py +++ b/pox/ext/rfproxy.py @@ -64,7 +64,7 @@ def delete_dp(self, dp_id): netmask_prefix = lambda a: sum([bin(int(x)).count("1") for x in a.split(".", 4)]) #ipc = MongoIPC.MongoIPCMessageService(MONGO_ADDRESS, MONGO_DB_NAME, str(ID), threading.Thread, time.sleep) -ipc = CouchIPC.CouchIPCMessageService ( COUCH_HOSTS, COUCH_BUCKET, threading.Thread, time.sleep) +ipc = CouchIPC.CouchIPCMessageService ( COUCH_HOST, COUCH_BUCKET, threading.Thread, time.sleep) table = Table() # Logging diff --git a/rflib/defs.py b/rflib/defs.py index 810c6a34..bb72feaa 100644 --- a/rflib/defs.py +++ b/rflib/defs.py @@ -1,7 +1,7 @@ MONGO_ADDRESS = "192.169.1.1:27017" MONGO_DB_NAME = "db" -COUCH_HOSTS = ["rener-notebook.local", "elantra-CQ43.local", "rener-IPMH67.local"] +COUCH_HOST = "rener-IPMH67.local" COUCH_BUCKET = "default" COUCH_INITIAL_VALUE = 0 diff --git a/rflib/ipc/CouchIPC.py b/rflib/ipc/CouchIPC.py index e24f3695..583c6812 100644 --- a/rflib/ipc/CouchIPC.py +++ b/rflib/ipc/CouchIPC.py @@ -1,6 +1,7 @@ from couchbase import Couchbase import rflib.ipc.IPC as IPC +from rflib.defs import * FROM_FIELD = "from" TO_FIELD = "to" @@ -28,27 +29,27 @@ def take_from_envelope(envelope, factory): return msg; class CouchIPCMessageService(IPC.IPCMessageService): - def __init__(self, hosts, bucket, thread_constructor, sleep_function): + def __init__(self, host, bucket, thread_constructor, sleep_function): """Construct an IPCMessageService Args: bucket: Bucket name of CouchBase. - hosts: list of hosts in the CouchBase cluster. + host: list of hosts in the CouchBase cluster. thread_constructor: function that takes 'target' and 'args' parameters for the function to run and arguments to pass, and return an object that has start() and join() functions. sleep_function: function that takes a float and delays processing for the specified period. """ - self._hosts = format_address(address) + self._host = host self._bucket = bucket - self._producer_connection = Couchbase.connect( - bucket=_bucket, - hosts=_hosts + self._connection = Couchbase.connect( + bucket=self._bucket, + host=self._hosts ) self._threading = thread_constructor self._sleep = sleep_function - self.set("key", COUCH_INITIAL_VALUE) + self._connection.set("key", COUCH_INITIAL_VALUE) #mudar... def listen(self, channel_id, factory, processor, block=True): @@ -59,8 +60,8 @@ def listen(self, channel_id, factory, processor, block=True): worker.join() def send(self, to, msg): - key = self.incr("key") - self.set( key, put_in_envelope(self.get_id(), to, msg) ) + key = self._connection.incr("key") + self._connection.set( key, put_in_envelope(self.get_id(), to, msg) ) return True #mudar... From 0ae08476c122a3073618969b64493676244f9bb8 Mon Sep 17 00:00:00 2001 From: Denis Wilson de Souza Oliveira Date: Sat, 6 Jul 2013 09:48:12 -0300 Subject: [PATCH 08/22] More bugs Fix Tab problens in: * CouchIPC * rfproxy * rfserver Change ipc in: rfproxy rfserver --- pox/ext/rfproxy.py | 3 ++- rflib/defs.py | 3 ++- rflib/ipc/CouchIPC.py | 48 +++++++++++++++++++++++++++---------------- rfserver/rfserver.py | 5 +++++ 4 files changed, 39 insertions(+), 20 deletions(-) diff --git a/pox/ext/rfproxy.py b/pox/ext/rfproxy.py index e31d87e9..61c0f246 100644 --- a/pox/ext/rfproxy.py +++ b/pox/ext/rfproxy.py @@ -17,6 +17,7 @@ FAILURE = 0 SUCCESS = 1 +ID = 0 # Association table class Table: @@ -64,7 +65,7 @@ def delete_dp(self, dp_id): netmask_prefix = lambda a: sum([bin(int(x)).count("1") for x in a.split(".", 4)]) #ipc = MongoIPC.MongoIPCMessageService(MONGO_ADDRESS, MONGO_DB_NAME, str(ID), threading.Thread, time.sleep) -ipc = CouchIPC.CouchIPCMessageService ( COUCH_HOSTS, COUCH_BUCKET, threading.Thread, time.sleep) +ipc = CouchIPC.CouchIPCMessageService ( COUCH_HOST, COUCH_BUCKET, str(ID), threading.Thread, time.sleep) table = Table() # Logging diff --git a/rflib/defs.py b/rflib/defs.py index 810c6a34..9cf33293 100644 --- a/rflib/defs.py +++ b/rflib/defs.py @@ -1,7 +1,8 @@ MONGO_ADDRESS = "192.169.1.1:27017" MONGO_DB_NAME = "db" -COUCH_HOSTS = ["rener-notebook.local", "elantra-CQ43.local", "rener-IPMH67.local"] +#COUCH_HOST = ["rener-notebook.local", "elantra-CQ43.local"] +COUCH_HOST = ["192.168.70.128", "192.168.70.129"] COUCH_BUCKET = "default" COUCH_INITIAL_VALUE = 0 diff --git a/rflib/ipc/CouchIPC.py b/rflib/ipc/CouchIPC.py index e24f3695..dab5ad5c 100644 --- a/rflib/ipc/CouchIPC.py +++ b/rflib/ipc/CouchIPC.py @@ -1,6 +1,10 @@ from couchbase import Couchbase +import pymongo as mongo +import bson + import rflib.ipc.IPC as IPC +from rflib.defs import * FROM_FIELD = "from" TO_FIELD = "to" @@ -28,7 +32,7 @@ def take_from_envelope(envelope, factory): return msg; class CouchIPCMessageService(IPC.IPCMessageService): - def __init__(self, hosts, bucket, thread_constructor, sleep_function): + def __init__(self, host, bucket, id_, thread_constructor, sleep_function): """Construct an IPCMessageService Args: @@ -40,15 +44,16 @@ def __init__(self, hosts, bucket, thread_constructor, sleep_function): sleep_function: function that takes a float and delays processing for the specified period. """ - self._hosts = format_address(address) + self._host = host self._bucket = bucket - self._producer_connection = Couchbase.connect( - bucket=_bucket, - hosts=_hosts + self._connection = Couchbase.connect( + bucket=self._bucket, + host=self._host ) + self._id = id_ self._threading = thread_constructor self._sleep = sleep_function - self.set("key", COUCH_INITIAL_VALUE) + self._connection.set("key", COUCH_INITIAL_VALUE) #mudar... def listen(self, channel_id, factory, processor, block=True): @@ -58,22 +63,31 @@ def listen(self, channel_id, factory, processor, block=True): if block: worker.join() - def send(self, to, msg): - key = self.incr("key") - self.set( key, put_in_envelope(self.get_id(), to, msg) ) + def send(self, channel_id, to, msg): + key = self._connection.incr("key") + envelope = put_in_envelope(self.get_id(), to, msg) + key = "{0}-{1}-{2}".format( + envelope[CONTENT_FIELD]['dp_id'], + envelope[CONTENT_FIELD]['ct_id'], + envelope[CONTENT_FIELD]['dp_port']) + self._connection.set( key, put_in_envelope(self.get_id(), to, msg) ) return True + # Denis alterou #mudar... def _listen_worker(self, channel_id, factory, processor): - connection = mongo.Connection(*self.address) + # collection = self._connection + # cursor = collection.query("all", "default_all" {TO_FIELD: self.get_id(), READ_FIELD: False}, sort=[("_id", mongo.ASCENDING)]) + connection = mongo.Connection(MONGO_ADDRESS) self._create_channel(connection, channel_id) - - collection = connection[self._db][channel_id] + collection = connection[MONGO_DB_NAME][channel_id] cursor = collection.find({TO_FIELD: self.get_id(), READ_FIELD: False}, sort=[("_id", mongo.ASCENDING)]) - while True: for envelope in cursor: msg = take_from_envelope(envelope, factory) + print "\n\n\n\n\n\n\n\n\n\n" + print msg + print "\n\n\n\n\n\n\n\n\n\n" processor.process(envelope[FROM_FIELD], envelope[TO_FIELD], channel_id, msg); collection.update({"_id": envelope["_id"]}, {"$set": {READ_FIELD: True}}) self._sleep(0.05) @@ -81,7 +95,7 @@ def _listen_worker(self, channel_id, factory, processor): #mudar... def _create_channel(self, connection, name): - db = connection[self._db] + db = connection[MONGO_DB_NAME] try: collection = mongo.collection.Collection(db, name, None, True, capped=True, size=CC_SIZE) collection.ensure_index([("_id", mongo.ASCENDING)]) @@ -91,8 +105,7 @@ def _create_channel(self, connection, name): except: pass -#mudar... -class MongoIPCMessage(dict, IPC.IPCMessage): +class CouchIPCMessage(dict, IPC.IPCMessage): def __init__(self, type_, **kwargs): dict.__init__(self) self.from_dict(kwargs) @@ -106,11 +119,10 @@ def from_dict(self, data): self[k] = v def from_bson(self, data): - data = bson.BSON.decode(data) self.from_dict(data) def to_bson(self): - return bson.BSON.encode(self) + return self def str(self): string = "" diff --git a/rfserver/rfserver.py b/rfserver/rfserver.py index 9b7a2db3..1f207e78 100755 --- a/rfserver/rfserver.py +++ b/rfserver/rfserver.py @@ -41,12 +41,17 @@ def __init__(self, configfile, islconffile): ch.setFormatter(logging.Formatter(logging.BASIC_FORMAT)) self.log.addHandler(ch) + self.ipc = CouchIPC.CouchIPCMessageService( COUCH_HOST, COUCH_BUCKET, RFSERVER_ID, threading.Thread, time.sleep ) + ''' self.ipc = MongoIPC.MongoIPCMessageService(MONGO_ADDRESS, MONGO_DB_NAME, RFSERVER_ID, threading.Thread, time.sleep) + ''' + # alterar funcoes no listen self.ipc.listen(RFCLIENT_RFSERVER_CHANNEL, self, self, False) + # alterar funcoes no listen self.ipc.listen(RFSERVER_RFPROXY_CHANNEL, self, self, True) def process(self, from_, to, channel, msg): From 2a27e731eebaad5f4ca6e31c07f79e27c3727e58 Mon Sep 17 00:00:00 2001 From: Rener Baffa da Silva Date: Sat, 6 Jul 2013 10:06:59 -0300 Subject: [PATCH 09/22] Solving merge problems --- pox/ext/rfproxy.py | 4 ---- rflib/defs.py | 4 ---- rflib/ipc/CouchIPC.py | 13 ------------- 3 files changed, 21 deletions(-) diff --git a/pox/ext/rfproxy.py b/pox/ext/rfproxy.py index fd32b40b..61c0f246 100644 --- a/pox/ext/rfproxy.py +++ b/pox/ext/rfproxy.py @@ -65,11 +65,7 @@ def delete_dp(self, dp_id): netmask_prefix = lambda a: sum([bin(int(x)).count("1") for x in a.split(".", 4)]) #ipc = MongoIPC.MongoIPCMessageService(MONGO_ADDRESS, MONGO_DB_NAME, str(ID), threading.Thread, time.sleep) -<<<<<<< HEAD ipc = CouchIPC.CouchIPCMessageService ( COUCH_HOST, COUCH_BUCKET, str(ID), threading.Thread, time.sleep) -======= -ipc = CouchIPC.CouchIPCMessageService ( COUCH_HOST, COUCH_BUCKET, threading.Thread, time.sleep) ->>>>>>> f9fa76cfa6c57ec36b782a9667a0aca2d8df3fed table = Table() # Logging diff --git a/rflib/defs.py b/rflib/defs.py index 43e01cd8..9cf33293 100644 --- a/rflib/defs.py +++ b/rflib/defs.py @@ -1,12 +1,8 @@ MONGO_ADDRESS = "192.169.1.1:27017" MONGO_DB_NAME = "db" -<<<<<<< HEAD #COUCH_HOST = ["rener-notebook.local", "elantra-CQ43.local"] COUCH_HOST = ["192.168.70.128", "192.168.70.129"] -======= -COUCH_HOST = "rener-IPMH67.local" ->>>>>>> f9fa76cfa6c57ec36b782a9667a0aca2d8df3fed COUCH_BUCKET = "default" COUCH_INITIAL_VALUE = 0 diff --git a/rflib/ipc/CouchIPC.py b/rflib/ipc/CouchIPC.py index cd96055d..cca6940a 100644 --- a/rflib/ipc/CouchIPC.py +++ b/rflib/ipc/CouchIPC.py @@ -32,11 +32,7 @@ def take_from_envelope(envelope, factory): return msg; class CouchIPCMessageService(IPC.IPCMessageService): -<<<<<<< HEAD def __init__(self, host, bucket, id_, thread_constructor, sleep_function): -======= - def __init__(self, host, bucket, thread_constructor, sleep_function): ->>>>>>> f9fa76cfa6c57ec36b782a9667a0aca2d8df3fed """Construct an IPCMessageService Args: @@ -52,11 +48,7 @@ def __init__(self, host, bucket, thread_constructor, sleep_function): self._bucket = bucket self._connection = Couchbase.connect( bucket=self._bucket, -<<<<<<< HEAD host=self._host -======= - host=self._hosts ->>>>>>> f9fa76cfa6c57ec36b782a9667a0aca2d8df3fed ) self._id = id_ self._threading = thread_constructor @@ -71,7 +63,6 @@ def listen(self, channel_id, factory, processor, block=True): if block: worker.join() -<<<<<<< HEAD def send(self, channel_id, to, msg): key = self._connection.incr("key") envelope = put_in_envelope(self.get_id(), to, msg) @@ -79,10 +70,6 @@ def send(self, channel_id, to, msg): envelope[CONTENT_FIELD]['dp_id'], envelope[CONTENT_FIELD]['ct_id'], envelope[CONTENT_FIELD]['dp_port']) -======= - def send(self, to, msg): - key = self._connection.incr("key") ->>>>>>> f9fa76cfa6c57ec36b782a9667a0aca2d8df3fed self._connection.set( key, put_in_envelope(self.get_id(), to, msg) ) return True From ef516742f39bcc19b95cc0eb93799e733966b2e1 Mon Sep 17 00:00:00 2001 From: Denis Wilson de Souza Oliveira Date: Sat, 6 Jul 2013 15:01:10 -0300 Subject: [PATCH 10/22] Fix Bugs rftest2 worker --- pox/ext/rfproxy.py | 3 --- rflib/ipc/CouchIPC.py | 55 ++++++++++++++----------------------------- rfserver/rfserver.py | 5 ++-- 3 files changed, 21 insertions(+), 42 deletions(-) diff --git a/pox/ext/rfproxy.py b/pox/ext/rfproxy.py index 61c0f246..a1b01661 100644 --- a/pox/ext/rfproxy.py +++ b/pox/ext/rfproxy.py @@ -5,10 +5,8 @@ from pox.core import core from pox.openflow.libopenflow_01 import * -import pymongo as mongo import rflib.ipc.IPC as IPC -import rflib.ipc.MongoIPC as MongoIPC import rflib.ipc.CouchIPC as CouchIPC from rflib.ipc.RFProtocol import * from rflib.ipc.RFProtocolFactory import RFProtocolFactory @@ -64,7 +62,6 @@ def delete_dp(self, dp_id): netmask_prefix = lambda a: sum([bin(int(x)).count("1") for x in a.split(".", 4)]) -#ipc = MongoIPC.MongoIPCMessageService(MONGO_ADDRESS, MONGO_DB_NAME, str(ID), threading.Thread, time.sleep) ipc = CouchIPC.CouchIPCMessageService ( COUCH_HOST, COUCH_BUCKET, str(ID), threading.Thread, time.sleep) table = Table() diff --git a/rflib/ipc/CouchIPC.py b/rflib/ipc/CouchIPC.py index cca6940a..e997b53b 100644 --- a/rflib/ipc/CouchIPC.py +++ b/rflib/ipc/CouchIPC.py @@ -1,7 +1,5 @@ -from couchbase import Couchbase - -import pymongo as mongo -import bson +from couchbase import Couchbase, LOCKMODE_WAIT +import couchbase.views import rflib.ipc.IPC as IPC from rflib.defs import * @@ -37,7 +35,7 @@ def __init__(self, host, bucket, id_, thread_constructor, sleep_function): Args: bucket: Bucket name of CouchBase. - host: list of hosts in the CouchBase cluster. + hosts: list of hosts in the CouchBase cluster. thread_constructor: function that takes 'target' and 'args' parameters for the function to run and arguments to pass, and return an object that has start() and join() functions. @@ -64,46 +62,29 @@ def listen(self, channel_id, factory, processor, block=True): worker.join() def send(self, channel_id, to, msg): - key = self._connection.incr("key") + key = self._connection.incr("key").value envelope = put_in_envelope(self.get_id(), to, msg) - key = "{0}-{1}-{2}".format( - envelope[CONTENT_FIELD]['dp_id'], - envelope[CONTENT_FIELD]['ct_id'], - envelope[CONTENT_FIELD]['dp_port']) - self._connection.set( key, put_in_envelope(self.get_id(), to, msg) ) + key = "{0}-{1}".format( key, to) + self._connection.set( str(key), envelope ) return True # Denis alterou #mudar... def _listen_worker(self, channel_id, factory, processor): - # collection = self._connection - # cursor = collection.query("all", "default_all" {TO_FIELD: self.get_id(), READ_FIELD: False}, sort=[("_id", mongo.ASCENDING)]) - connection = mongo.Connection(MONGO_ADDRESS) - self._create_channel(connection, channel_id) - collection = connection[MONGO_DB_NAME][channel_id] - cursor = collection.find({TO_FIELD: self.get_id(), READ_FIELD: False}, sort=[("_id", mongo.ASCENDING)]) + connection = Couchbase.connect( + bucket=self._bucket, + host=self._host, + ) while True: + cursor = connection.query("all", "filter") for envelope in cursor: - msg = take_from_envelope(envelope, factory) - print "\n\n\n\n\n\n\n\n\n\n" - print msg - print "\n\n\n\n\n\n\n\n\n\n" - processor.process(envelope[FROM_FIELD], envelope[TO_FIELD], channel_id, msg); - collection.update({"_id": envelope["_id"]}, {"$set": {READ_FIELD: True}}) - self._sleep(0.05) - cursor = collection.find({TO_FIELD: self.get_id(), READ_FIELD: False}, sort=[("_id", mongo.ASCENDING)]) - - #mudar... - def _create_channel(self, connection, name): - db = connection[MONGO_DB_NAME] - try: - collection = mongo.collection.Collection(db, name, None, True, capped=True, size=CC_SIZE) - collection.ensure_index([("_id", mongo.ASCENDING)]) - collection.ensure_index([(TO_FIELD, mongo.ASCENDING)]) - # TODO: improve this catch. It should be more specific, but pymongo - # behavior doesn't match its documentation, so we are being dirty. - except: - pass + if envelope.value[TO_FIELD] == self.get_id(): + msg = take_from_envelope(envelope.value, factory) + processor.process(envelope.value[FROM_FIELD], envelope.value[TO_FIELD], channel_id, msg); + envelope.value[READ_FIELD] = True + self._connection.set(envelope.docid, envelope.value) + # work is better without with CouchBase + #self._sleep(0.05) class CouchIPCMessage(dict, IPC.IPCMessage): def __init__(self, type_, **kwargs): diff --git a/rfserver/rfserver.py b/rfserver/rfserver.py index 1f207e78..27a83d60 100755 --- a/rfserver/rfserver.py +++ b/rfserver/rfserver.py @@ -8,10 +8,11 @@ import time import argparse -from bson.binary import Binary +#from bson.binary import Binary import rflib.ipc.IPC as IPC -import rflib.ipc.MongoIPC as MongoIPC +#import rflib.ipc.MongoIPC as MongoIPC +import rflib.ipc.CouchIPC as CouchIPC from rflib.ipc.RFProtocol import * from rflib.ipc.RFProtocolFactory import RFProtocolFactory from rflib.defs import * From 108c7cf84af9f9643c125dff78913b08959d3e8a Mon Sep 17 00:00:00 2001 From: Denis Wilson de Souza Oliveira Date: Sat, 6 Jul 2013 17:38:59 -0300 Subject: [PATCH 11/22] Instalation Server and Dev Libraries Scripts couchbase-server.sh -> install server couchbase-dev.sh -> install dev --- couchbase-dev.sh | 16 ++++++++++++++++ couchbase-server.sh | 13 +++++++++++++ 2 files changed, 29 insertions(+) create mode 100755 couchbase-dev.sh create mode 100755 couchbase-server.sh diff --git a/couchbase-dev.sh b/couchbase-dev.sh new file mode 100755 index 00000000..8e2c1377 --- /dev/null +++ b/couchbase-dev.sh @@ -0,0 +1,16 @@ +#!/bin/bash + + user=`whoami` + + if [ "$user" == "root" ] + then + wget -O /etc/apt/sources.list.d/couchbase.list http://packages.couchbase.com/ubuntu/couchbase-ubuntu1204.list + wget -O- http://packages.couchbase.com/ubuntu/couchbase.key | apt-key add - + apt-get update + apt-get install libcouchbase2 libcouchbase-dev python-pip python-dev + pip install couchbase + else + echo "Precisa ser root." + fi + +exit 0 diff --git a/couchbase-server.sh b/couchbase-server.sh new file mode 100755 index 00000000..910a3940 --- /dev/null +++ b/couchbase-server.sh @@ -0,0 +1,13 @@ +#!/bin/bash + + user=`whoami` + + if [ "$whoami" == root] + then + wget http://packages.couchbase.com/releases/2.0.1/couchbase-server-enterprise_x86_2.0.1.deb + dpkg -i http://packages.couchbase.com/releases/2.0.1/couchbase-server-enterprise_x86_2.0.1.deb + else + echo "Precisa ser root." + fi + +exit 0 From 9ec89aca53b5008bcbc98d89e7e3339f01974d26 Mon Sep 17 00:00:00 2001 From: viniciuskol Date: Sat, 6 Jul 2013 17:58:40 -0300 Subject: [PATCH 12/22] Update couchbase-dev.sh --- couchbase-dev.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchbase-dev.sh b/couchbase-dev.sh index 8e2c1377..f5797b13 100755 --- a/couchbase-dev.sh +++ b/couchbase-dev.sh @@ -10,7 +10,7 @@ apt-get install libcouchbase2 libcouchbase-dev python-pip python-dev pip install couchbase else - echo "Precisa ser root." + echo "You must be a root user." fi exit 0 From a0619b76d36fd0a6faee727faa5eaa96f554c26f Mon Sep 17 00:00:00 2001 From: viniciuskol Date: Sat, 6 Jul 2013 18:00:15 -0300 Subject: [PATCH 13/22] Messages were translated to english --- couchbase-dev.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchbase-dev.sh b/couchbase-dev.sh index f5797b13..f05b4040 100755 --- a/couchbase-dev.sh +++ b/couchbase-dev.sh @@ -10,7 +10,7 @@ apt-get install libcouchbase2 libcouchbase-dev python-pip python-dev pip install couchbase else - echo "You must be a root user." + echo "You must be a root user." fi exit 0 From 47fadfe4e8db9da3e384a98577d02b2f0b9ece1f Mon Sep 17 00:00:00 2001 From: viniciuskol Date: Sat, 6 Jul 2013 18:01:52 -0300 Subject: [PATCH 14/22] Messages were translated to english --- couchbase-server.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchbase-server.sh b/couchbase-server.sh index 910a3940..f49fa3fc 100755 --- a/couchbase-server.sh +++ b/couchbase-server.sh @@ -7,7 +7,7 @@ wget http://packages.couchbase.com/releases/2.0.1/couchbase-server-enterprise_x86_2.0.1.deb dpkg -i http://packages.couchbase.com/releases/2.0.1/couchbase-server-enterprise_x86_2.0.1.deb else - echo "Precisa ser root." + echo "You must be a root user." fi exit 0 From ab1c04a8e7adeebc5b9016b6a7d9c872a0fbe6f4 Mon Sep 17 00:00:00 2001 From: viniciuskol Date: Sat, 6 Jul 2013 18:05:41 -0300 Subject: [PATCH 15/22] Create README-RouteFlow.md --- README-RouteFlow.md | 154 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 README-RouteFlow.md diff --git a/README-RouteFlow.md b/README-RouteFlow.md new file mode 100644 index 00000000..76bac1d1 --- /dev/null +++ b/README-RouteFlow.md @@ -0,0 +1,154 @@ +# Welcome +This version of RouteFlow is a beta developers' release intended to evaluate RouteFlow for providing virtualized IP routing services on one or more OpenFlow switches. + +You can learn more about RouteFlow in our [main page in GitHub](http://cpqd.github.com/RouteFlow/) and in our [website](https://sites.google.com/site/routeflow/). + +Please be aware of POX, OpenFlow, Open vSwitch, Quagga, MongoDB, jQuery, JIT and RouteFlow licenses and terms. + +# Distribution overview +RouteFlow is a distribution composed by three basic applications: RFClient, RFServer and RFProxy. + +* RFClient runs as a daemon in the Virtual Machine (VM), detecting changes in the Linux ARP and routing tables. Routing information is sent to the RFServer when there's an update. + +* RFServer is a standalone application that manages the VMs running the RFClient daemons. The RFServer keeps the mapping between the RFClient VM instances and interfaces and the corresponding switches and ports. It connects to RFProxy to instruct it about when to configure flows and also to configure the Open vSwitch to maintain the connectivity in the virtual environment formed by the set of VMs. + +* RFProxy is an application (for POX and other controllers) responsible for the interactions with the OpenFlow switches (identified by datapaths) via the OpenFlow protocol. It listens to instructions from the RFServer and notifies it about events in the network. We recommend running POX when you are experimenting and testing your network. Other implementations in different controllers will be available soon. + +There is also a library of common functions (rflib). It has implementations of the IPC, utilities like custom types for IP and MAC addresses manipulation and OpenFlow message creation. + +Additionally, there's `rfweb`, an extra module that provides an web interface for RouteFlow. + +``` +The RouteFlow Architecture + ++--------VM---------+ +| Quagga | RFClient | ++-------------------+ + \ +M:1 \ RFProtocol + \ ++-------------------+ +| RFServer | ++-------------------+ + \ +1:1 \ RFProtocol + \ ++-------------------+ +| RFProxy | +|-------------------| +| Controller | ++-------------------+ + \ +1:N \ OpenFlow Protocol + \ ++-------------------+ +| OpenFlow Switch | ++-------------------+ +``` + +# Building + +RouteFlow runs on Ubuntu 12.04. + +1. Install the dependencies: +``` +sudo apt-get install build-essential git libboost-dev \ + libboost-program-options-dev libboost-thread-dev \ + libboost-filesystem-dev iproute-dev openvswitch-switch \ + mongodb python-pymongo +``` + +2. Clone RouteFlow's repository on GitHub: +``` +$ git clone git://github.com/CPqD/RouteFlow.git +``` + +3. Build `rfclient` +``` +make rfclient +``` + +That's it! Now you can run tests 1 and 2. The setup to run them is described in the "Running" section. + +# Running +The folder rftest contains all that is needed to create and run two test cases. + +## Virtual environment +First, create the default LXC containers that will run as virtual machines: +``` +$ cd rftest +$ sudo ./create +``` +The containers will have a default ubuntu/ubuntu user/password combination. **You should change that if you plan to deploy RouteFlow**. + +By default, the tests below will use the LXC containers created by the `create` script. You can use other virtualization technologies. If you have experience with or questions about setting up RouteFlow on a particular technology, contact us! See the "Support" section. + +## Test cases + +Default configuration files are provided for these tests in the `rftest` directory (you don't need to change anything). +You can stops them at any time by pressing CTRL+C. + +### rftest1 +1. Run: +``` +$ sudo ./rftest1 +``` + +2. You can then log in to the LXC container b1 and try to ping b2: +``` +$ sudo lxc-console -n b1 +``` + +3. Inside b1, run: +``` +# ping 172.31.2.2 +``` + +For more details on this test, see its [tutorial](https://github.com/CPqD/RouteFlow/wiki/Tutorial-1:-rftest1). + +### rftest2 +This test should be run with a [Mininet](http://mininet.org/) simulated network. +In the steps below, replace [guest address] with the IP address you use to access your Mininet VM. +The same applies to [host address], that should be the address to access the host from inside the VM. + +1. Run: +``` +$ sudo ./rftest2 +``` + +2. Once you have a Mininet VM up and running, copy the network topology files in rftest to the VM: +``` +$ scp topo-4sw-4host.py mininet@[guest address]:/home/mininet/mininet/custom +$ scp ipconf mininet@[guest address]:/home/mininet +``` + +3. Then start the network: +``` +$ sudo mn --custom mininet/custom/topo-4sw-4host.py --topo=rftest2 --controller=remote,ip=[host address],port=6633 --pre=ipconf +``` + +Wait for the network to converge (it should take a few seconds), and try to ping: +``` +mininet> pingall +... +mininet> h2 ping h3 +``` + +For more details on this test, see its [tutorial](https://github.com/CPqD/RouteFlow/wiki/Tutorial-2:-rftest2). + + +# Now what? +If you want to use the web interface to inspect RouteFlow behavior, see the wiki page on [rfweb](https://github.com/CPqD/RouteFlow/wiki/The-web-interface). + +If you want to create your custom configurations schemes for a given setup, check out the [configuration section of the first tutorial](https://github.com/CPqD/RouteFlow/wiki/Tutorial-1:-rftest1#configuration-file) and the guide on [how to create your virtual environment](https://github.com/CPqD/RouteFlow/wiki/Virtual-environment-creation). + +If you're developing for RouteFlow, there are more advanced options and components that can be installed through the `build.sh` script. See its [source](https://github.com/CPqD/RouteFlow/blob/master/build.sh) for more information. + + +# Support +If you want to know more or need to contact us regarding the project for anything (questions, suggestions, bug reports, discussions about RouteFlow and SDN in general) you can use the following resources: +* RouteFlow repository [wiki](https://github.com/CPqD/RouteFlow/wiki) and [issues](https://github.com/CPqD/RouteFlow/issues) in GitHub + +* Google Groups [mailing list](http://groups.google.com/group/routeflow-discuss?hl=en_US) + +_RouteFlow - Copyright (c) 2012 CPqD_ From fe109e7d60ee4cc6b6eddfce157af6a402e76d0c Mon Sep 17 00:00:00 2001 From: viniciuskol Date: Sat, 6 Jul 2013 18:06:40 -0300 Subject: [PATCH 16/22] Update README.md --- README.md | 179 +++++++++++++++--------------------------------------- 1 file changed, 50 insertions(+), 129 deletions(-) diff --git a/README.md b/README.md index 76bac1d1..2e03e1f3 100644 --- a/README.md +++ b/README.md @@ -1,154 +1,75 @@ -# Welcome -This version of RouteFlow is a beta developers' release intended to evaluate RouteFlow for providing virtualized IP routing services on one or more OpenFlow switches. - -You can learn more about RouteFlow in our [main page in GitHub](http://cpqd.github.com/RouteFlow/) and in our [website](https://sites.google.com/site/routeflow/). - -Please be aware of POX, OpenFlow, Open vSwitch, Quagga, MongoDB, jQuery, JIT and RouteFlow licenses and terms. - -# Distribution overview -RouteFlow is a distribution composed by three basic applications: RFClient, RFServer and RFProxy. - -* RFClient runs as a daemon in the Virtual Machine (VM), detecting changes in the Linux ARP and routing tables. Routing information is sent to the RFServer when there's an update. - -* RFServer is a standalone application that manages the VMs running the RFClient daemons. The RFServer keeps the mapping between the RFClient VM instances and interfaces and the corresponding switches and ports. It connects to RFProxy to instruct it about when to configure flows and also to configure the Open vSwitch to maintain the connectivity in the virtual environment formed by the set of VMs. - -* RFProxy is an application (for POX and other controllers) responsible for the interactions with the OpenFlow switches (identified by datapaths) via the OpenFlow protocol. It listens to instructions from the RFServer and notifies it about events in the network. We recommend running POX when you are experimenting and testing your network. Other implementations in different controllers will be available soon. - -There is also a library of common functions (rflib). It has implementations of the IPC, utilities like custom types for IP and MAC addresses manipulation and OpenFlow message creation. - -Additionally, there's `rfweb`, an extra module that provides an web interface for RouteFlow. - -``` -The RouteFlow Architecture - -+--------VM---------+ -| Quagga | RFClient | -+-------------------+ - \ -M:1 \ RFProtocol - \ -+-------------------+ -| RFServer | -+-------------------+ - \ -1:1 \ RFProtocol - \ -+-------------------+ -| RFProxy | -|-------------------| -| Controller | -+-------------------+ - \ -1:N \ OpenFlow Protocol - \ -+-------------------+ -| OpenFlow Switch | -+-------------------+ -``` +# RouteFlow with Database in cluster mode -# Building +- Introduction +- What's new +- Installation +- Known Issues or Bugs +- Licensing/legal +- Feedback -RouteFlow runs on Ubuntu 12.04. -1. Install the dependencies: -``` -sudo apt-get install build-essential git libboost-dev \ - libboost-program-options-dev libboost-thread-dev \ - libboost-filesystem-dev iproute-dev openvswitch-switch \ - mongodb python-pymongo -``` +# Introduction +This is a version of RouteFlow project managed by CPqD with better support to scalability through a DB in cluster mode. RouteFlow is an open source project to provide virtualized IP routing services over OpenFlow enabled hardware. -2. Clone RouteFlow's repository on GitHub: -``` -$ git clone git://github.com/CPqD/RouteFlow.git -``` +Minimum Requirements: +* CouchBase 2.0.1 or newer -3. Build `rfclient` -``` -make rfclient -``` +Recommended Requirements: +* CouchBase 2.0.1 -That's it! Now you can run tests 1 and 2. The setup to run them is described in the "Running" section. +Source: https://github.com/renerbaffa/RouteFlow -# Running -The folder rftest contains all that is needed to create and run two test cases. -## Virtual environment -First, create the default LXC containers that will run as virtual machines: -``` -$ cd rftest -$ sudo ./create -``` -The containers will have a default ubuntu/ubuntu user/password combination. **You should change that if you plan to deploy RouteFlow**. +# What's New -By default, the tests below will use the LXC containers created by the `create` script. You can use other virtualization technologies. If you have experience with or questions about setting up RouteFlow on a particular technology, contact us! See the "Support" section. +This release sees some significant advances in the RouteFlow project. +Notable changes are: -## Test cases +- Couchbase as new DBMS. +- Support for use RouteFlow with the BD in cluster mode. +- Improved scalability of RouteFlow. -Default configuration files are provided for these tests in the `rftest` directory (you don't need to change anything). -You can stops them at any time by pressing CTRL+C. -### rftest1 -1. Run: -``` -$ sudo ./rftest1 -``` +# Installation -2. You can then log in to the LXC container b1 and try to ping b2: -``` -$ sudo lxc-console -n b1 -``` +## Manual Installation -3. Inside b1, run: -``` -# ping 172.31.2.2 -``` +1. Download and install Couchbase Server on main server. -For more details on this test, see its [tutorial](https://github.com/CPqD/RouteFlow/wiki/Tutorial-1:-rftest1). +2. Start Couchbase Server, create a bucket with desired configurations. -### rftest2 -This test should be run with a [Mininet](http://mininet.org/) simulated network. -In the steps below, replace [guest address] with the IP address you use to access your Mininet VM. -The same applies to [host address], that should be the address to access the host from inside the VM. - -1. Run: -``` -$ sudo ./rftest2 -``` - -2. Once you have a Mininet VM up and running, copy the network topology files in rftest to the VM: -``` -$ scp topo-4sw-4host.py mininet@[guest address]:/home/mininet/mininet/custom -$ scp ipconf mininet@[guest address]:/home/mininet -``` - -3. Then start the network: -``` -$ sudo mn --custom mininet/custom/topo-4sw-4host.py --topo=rftest2 --controller=remote,ip=[host address],port=6633 --pre=ipconf -``` - -Wait for the network to converge (it should take a few seconds), and try to ping: -``` -mininet> pingall -... -mininet> h2 ping h3 -``` +3. Install Couchbase Server on others servers and choose [Join Cluster Now]. Set the following data: [user], [password] and [host ip address]. If you have any questions, you can follow this manual: -For more details on this test, see its [tutorial](https://github.com/CPqD/RouteFlow/wiki/Tutorial-2:-rftest2). +4. Access Couchbase’s administration, go to [Server Nodes] and click [Rebalance] to optimize all servers. More information are available in: <> +5. Setup the file [***] with following data: ip host, with the list of all servers in cluster mode. -# Now what? -If you want to use the web interface to inspect RouteFlow behavior, see the wiki page on [rfweb](https://github.com/CPqD/RouteFlow/wiki/The-web-interface). -If you want to create your custom configurations schemes for a given setup, check out the [configuration section of the first tutorial](https://github.com/CPqD/RouteFlow/wiki/Tutorial-1:-rftest1#configuration-file) and the guide on [how to create your virtual environment](https://github.com/CPqD/RouteFlow/wiki/Virtual-environment-creation). +## Auto Installation -If you're developing for RouteFlow, there are more advanced options and components that can be installed through the `build.sh` script. See its [source](https://github.com/CPqD/RouteFlow/blob/master/build.sh) for more information. +1. Run script [couchbase-server.sh] to install Couchbase server. +2. Run script [couchbase-dev.sh] to install Development libraries. -# Support -If you want to know more or need to contact us regarding the project for anything (questions, suggestions, bug reports, discussions about RouteFlow and SDN in general) you can use the following resources: -* RouteFlow repository [wiki](https://github.com/CPqD/RouteFlow/wiki) and [issues](https://github.com/CPqD/RouteFlow/issues) in GitHub -* Google Groups [mailing list](http://groups.google.com/group/routeflow-discuss?hl=en_US) +# Known Issues or Bugs -_RouteFlow - Copyright (c) 2012 CPqD_ + + + +# Licensing/Legal + +This version is released under the Apache License, Version 2.0. +The License can be found here: http://www.apache.org/licenses/LICENSE-2.0 + + +# Feedback + +If you find a problem, obsolete or improper code or such, please let us know by contacting us. +Suggestions and corrections are welcome. Below are contacts of the authors of this project. + +- Danilo Pinto da Silva ( *danilo.silva (at) dc.ufscar.br* ) +- Denis Wilson de Souza Oliveira ( *denis.oliveira (at) dc.ufscar.br* ) +- Eloisa Cristina Silva Santos ( *eloisa.santos (at) dc.ufscar.br* ) +- Rener Baffa da Silva ( *rener.silva (at) dc.ufscar.br* ) +- Vinícius Afonso Raimundo Ferreira ( *vinicius.ferreira (at) dc.ufscar.br* ) From 32059eae144056559c6b3efbd4362129581cbb5b Mon Sep 17 00:00:00 2001 From: viniciuskol Date: Mon, 8 Jul 2013 18:02:28 -0300 Subject: [PATCH 17/22] Update README.md --- README.md | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 2e03e1f3..2a63d2ba 100644 --- a/README.md +++ b/README.md @@ -12,10 +12,10 @@ This is a version of RouteFlow project managed by CPqD with better support to scalability through a DB in cluster mode. RouteFlow is an open source project to provide virtualized IP routing services over OpenFlow enabled hardware. Minimum Requirements: -* CouchBase 2.0.1 or newer +- CouchBase 2.0.1 or newer Recommended Requirements: -* CouchBase 2.0.1 +- CouchBase 2.0.1 Source: https://github.com/renerbaffa/RouteFlow @@ -25,8 +25,8 @@ Source: https://github.com/renerbaffa/RouteFlow This release sees some significant advances in the RouteFlow project. Notable changes are: -- Couchbase as new DBMS. -- Support for use RouteFlow with the BD in cluster mode. +- Couchbase as new Database Management System. +- Support for use RouteFlow with DB in cluster mode. - Improved scalability of RouteFlow. @@ -34,22 +34,17 @@ Notable changes are: ## Manual Installation -1. Download and install Couchbase Server on main server. +1. Download and install Couchbase Server on main server. Also, you can run script ```couchbase-server.sh``` to download and install Couchbase server. 2. Start Couchbase Server, create a bucket with desired configurations. -3. Install Couchbase Server on others servers and choose [Join Cluster Now]. Set the following data: [user], [password] and [host ip address]. If you have any questions, you can follow this manual: +3. Install Couchbase Server on others servers and choose ```Join Cluster Now```. Set the following data: ```user```, ```password``` and ```host ip address```. If you have any questions, you can follow this manual: -4. Access Couchbase’s administration, go to [Server Nodes] and click [Rebalance] to optimize all servers. More information are available in: <> +4. Access Couchbase’s administration, go to ```Server Nodes``` and click ```Rebalance``` to optimize all servers. More information are available in: 5. Setup the file [***] with following data: ip host, with the list of all servers in cluster mode. - -## Auto Installation - -1. Run script [couchbase-server.sh] to install Couchbase server. - -2. Run script [couchbase-dev.sh] to install Development libraries. +6. Run script ```couchbase-dev.sh``` to install Development libraries. # Known Issues or Bugs From 02a15b2947b3d1bcbc0cc7c96dfd75319cea8284 Mon Sep 17 00:00:00 2001 From: denis-oliveira Date: Tue, 9 Jul 2013 10:09:24 -0300 Subject: [PATCH 18/22] Update CouchIPC.py Improve working --- rflib/ipc/CouchIPC.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rflib/ipc/CouchIPC.py b/rflib/ipc/CouchIPC.py index e997b53b..d7ce3d74 100644 --- a/rflib/ipc/CouchIPC.py +++ b/rflib/ipc/CouchIPC.py @@ -46,7 +46,8 @@ def __init__(self, host, bucket, id_, thread_constructor, sleep_function): self._bucket = bucket self._connection = Couchbase.connect( bucket=self._bucket, - host=self._host + host=self._host, + unlock_gil=False ) self._id = id_ self._threading = thread_constructor @@ -55,8 +56,7 @@ def __init__(self, host, bucket, id_, thread_constructor, sleep_function): #mudar... def listen(self, channel_id, factory, processor, block=True): - worker = self._threading(target=self._listen_worker, - args=(channel_id, factory, processor)) + worker = self._threading(target=self._listen_worker, args=(channel_id, factory, processor)) worker.start() if block: worker.join() @@ -74,6 +74,7 @@ def _listen_worker(self, channel_id, factory, processor): connection = Couchbase.connect( bucket=self._bucket, host=self._host, + unlock_gil=False ) while True: cursor = connection.query("all", "filter") From f4c3f9663cc6cb2bd3f4936865c47bcaa1ea2872 Mon Sep 17 00:00:00 2001 From: Rener Baffa da Silva Date: Tue, 9 Jul 2013 11:22:05 -0300 Subject: [PATCH 19/22] =?UTF-8?q?Resumo=20das=20altera=C3=A7=C3=B5es=20fei?= =?UTF-8?q?tas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No arquivo está o resumo das alterações feitas pelo grupo de estudos. --- Danilo_Denis_Eloisa_Rener_Vinicius.txt | 31 +++++++++++--------------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/Danilo_Denis_Eloisa_Rener_Vinicius.txt b/Danilo_Denis_Eloisa_Rener_Vinicius.txt index e0508584..6564bcb5 100644 --- a/Danilo_Denis_Eloisa_Rener_Vinicius.txt +++ b/Danilo_Denis_Eloisa_Rener_Vinicius.txt @@ -1,27 +1,22 @@ -1 - Instalar o CouchBase na máquina principal e criar o bucket com as configurações desejadas. - Available in: +RESUMO DE ATIVIDADES -2 - Instalar o CouchBase nos servidores escolhendo a opção "Join Cluster Now", adicionando o usuário, senha e IP do servidor principal. - Available in: +- Criamos um script para instalação do Couchbase (é o arquivo 'couchbase-server.sh'). +- Criamos um script para instalação da biblioteca e dependências para desenvolver com o Couchbase (é o arquivo 'couchbase-dev.sh'). +- Criamos a classe CouchIPC.py (responsável pela conexão com o novo banco de dados e que substitui o MongoIPC.py). +- Retiramos as conversões de dados para/de bson para json (necessário no Mongo e não mais necessário). -3 - Acessando o CouchBase pela URL (porta 8091), na aba "Server Nodes" clicar em "Rebalance", para balancear o uso de cada nó. - Available in: -4 - Lembrar de configurar o arquivo *******************, no campo host, com a lista de todos os servidores no modo cluster. -OBS: -Scripts -Instalação do CouchBase Server -> -Desenvolvimento -> dev.sh - -CouchBase - -Criar uma view e publicar no CouchBase - -CouchIPC.py -Alterações no para criar a conexão +Alteramos os arquivos necessários para fazer o script rftest2 funcionar com o novo BD. +Para funcionar, é necessário: +1 - Instalar o Couchbase (mais informações em ). +2 - Criar um bucket com o nome 'routeflow' (pode ser outro nome. Mais instruções no passo 5). +3 - Instalar o Couchbase nos demais servidores que farão parte do cluster através da opção 'Join cluster now' (mais informações em http://www.couchbase.com/docs/couchbase-manual-1.8/couchbase-getting-started-setup.html>), adicionando usuário, senha e IP do servidor principal (aquele dos passos 1 e 2). +4 - O Couchbase possui uma interface (HTTP) para configuração e manipulação dos dados. Basta acessar o servidor local na porta 8091. Na aba 'Server Nodes', clicar em 'Rebalance' para balancear o uso de cada nó (mais informações em ). +5 - O arquivo rflib/defs.py possui os arquivos de configurações para conexão com o banco de dados. Lembre-se de colocar os IPs das máquinas em cluster na variável COUCH_HOST (exemplo: ["192.168.70.128", "192.168.70.129"]), o nome do Bucket criado na variável COUCH_BUCKET e o valor 0 na variável COUCH_INITIAL_VALUE (que é o valor inicial da chave do conteúdo que será inserido no BD). +6 - FALAR SOBRE A VIEW ****************************************** From 1a15941abfca4a022e849e517a857b9887797374 Mon Sep 17 00:00:00 2001 From: denis-oliveira Date: Tue, 9 Jul 2013 11:46:47 -0300 Subject: [PATCH 20/22] Update Danilo_Denis_Eloisa_Rener_Vinicius.txt --- Danilo_Denis_Eloisa_Rener_Vinicius.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Danilo_Denis_Eloisa_Rener_Vinicius.txt b/Danilo_Denis_Eloisa_Rener_Vinicius.txt index 6564bcb5..74fa6f27 100644 --- a/Danilo_Denis_Eloisa_Rener_Vinicius.txt +++ b/Danilo_Denis_Eloisa_Rener_Vinicius.txt @@ -4,7 +4,7 @@ RESUMO DE ATIVIDADES - Criamos um script para instalação da biblioteca e dependências para desenvolver com o Couchbase (é o arquivo 'couchbase-dev.sh'). - Criamos a classe CouchIPC.py (responsável pela conexão com o novo banco de dados e que substitui o MongoIPC.py). - Retiramos as conversões de dados para/de bson para json (necessário no Mongo e não mais necessário). - +- Alteramos as referências do rfserver e o rfproxy para utilizar o CouchIPC @@ -18,5 +18,7 @@ Para funcionar, é necessário: 4 - O Couchbase possui uma interface (HTTP) para configuração e manipulação dos dados. Basta acessar o servidor local na porta 8091. Na aba 'Server Nodes', clicar em 'Rebalance' para balancear o uso de cada nó (mais informações em ). 5 - O arquivo rflib/defs.py possui os arquivos de configurações para conexão com o banco de dados. Lembre-se de colocar os IPs das máquinas em cluster na variável COUCH_HOST (exemplo: ["192.168.70.128", "192.168.70.129"]), o nome do Bucket criado na variável COUCH_BUCKET e o valor 0 na variável COUCH_INITIAL_VALUE (que é o valor inicial da chave do conteúdo que será inserido no BD). -6 - FALAR SOBRE A VIEW ****************************************** +6 - View +Para as pesquisas foram criadas views que consistem em funções de javascript para map dos dados dentro do CouchBase, as views são criadas dentro da interface web do CouchBase. Pode ser encontrado um guia para criação de view em http://www.couchbase.com/docs/couchbase-manual-2.0/couchbase-views-types.html + From df2f9d1d003afe9b958cdc07912732d45d0ed852 Mon Sep 17 00:00:00 2001 From: viniciuskol Date: Tue, 9 Jul 2013 13:15:16 -0300 Subject: [PATCH 21/22] Update README.md --- README.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 2a63d2ba..9d6d63c3 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@ - Introduction - What's new - Installation -- Known Issues or Bugs - Licensing/legal - Feedback @@ -28,6 +27,11 @@ Notable changes are: - Couchbase as new Database Management System. - Support for use RouteFlow with DB in cluster mode. - Improved scalability of RouteFlow. +- Scripts to install Couchbase and development libraries. +- MongoIPC class was replaced for CouchIPC class. CouchIPC class is responsable for management of connection with Database. +- Conversion from BSon (Binary JSON) to JSON was removed, because this conversion it isn't necessary in Couchbase. +- References for MongoIPC class were changed to CouchIPC class. +- Script rftest2 was updated to use CouchIPC. # Installation @@ -36,20 +40,17 @@ Notable changes are: 1. Download and install Couchbase Server on main server. Also, you can run script ```couchbase-server.sh``` to download and install Couchbase server. -2. Start Couchbase Server, create a bucket with desired configurations. +2. Start Couchbase Server, create a bucket with name 'routeflow' and desired configurations. -3. Install Couchbase Server on others servers and choose ```Join Cluster Now```. Set the following data: ```user```, ```password``` and ```host ip address```. If you have any questions, you can follow this manual: +3. Install Couchbase Server on others servers and choose ```Join Cluster Now```. Set the following data: ```user```, ```password``` and ```main server ip address```. If you have any questions, you can follow this manual: -4. Access Couchbase’s administration, go to ```Server Nodes``` and click ```Rebalance``` to optimize all servers. More information are available in: +4. Access Couchbase's administration, go to ```Server Nodes``` and click ```Rebalance``` to optimize all servers. More information are available in: -5. Setup the file [***] with following data: ip host, with the list of all servers in cluster mode. +5. Setup the file ```rflib/defs.py``` with following data: bucket's name (variable: ```COUCH_BUCKET```); list of all servers in cluster mode (variable: ```COUCH_HOST```, example: ```["192.168.70.128", "192.168.70.129"]```); set variable ```COUCH_INITIAL_VALUE``` with ```0```. 6. Run script ```couchbase-dev.sh``` to install Development libraries. - -# Known Issues or Bugs - - +7. Create views in Couchbase, for more information: # Licensing/Legal From 2c8470cd421917d524d474f8826f4769d1bff102 Mon Sep 17 00:00:00 2001 From: Rener Baffa da SIlva Date: Tue, 9 Jul 2013 19:08:11 -0300 Subject: [PATCH 22/22] =?UTF-8?q?Adi=C3=A7=C3=A3o=20das=20instru=C3=A7?= =?UTF-8?q?=C3=B5es=20das=20views?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adicionando as instruções dasviews criadas para total funcionamento das alterações no projeto. --- Danilo_Denis_Eloisa_Rener_Vinicius.txt | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/Danilo_Denis_Eloisa_Rener_Vinicius.txt b/Danilo_Denis_Eloisa_Rener_Vinicius.txt index 74fa6f27..54a0e7ed 100644 --- a/Danilo_Denis_Eloisa_Rener_Vinicius.txt +++ b/Danilo_Denis_Eloisa_Rener_Vinicius.txt @@ -17,8 +17,24 @@ Para funcionar, é necessário: 3 - Instalar o Couchbase nos demais servidores que farão parte do cluster através da opção 'Join cluster now' (mais informações em http://www.couchbase.com/docs/couchbase-manual-1.8/couchbase-getting-started-setup.html>), adicionando usuário, senha e IP do servidor principal (aquele dos passos 1 e 2). 4 - O Couchbase possui uma interface (HTTP) para configuração e manipulação dos dados. Basta acessar o servidor local na porta 8091. Na aba 'Server Nodes', clicar em 'Rebalance' para balancear o uso de cada nó (mais informações em ). 5 - O arquivo rflib/defs.py possui os arquivos de configurações para conexão com o banco de dados. Lembre-se de colocar os IPs das máquinas em cluster na variável COUCH_HOST (exemplo: ["192.168.70.128", "192.168.70.129"]), o nome do Bucket criado na variável COUCH_BUCKET e o valor 0 na variável COUCH_INITIAL_VALUE (que é o valor inicial da chave do conteúdo que será inserido no BD). - 6 - View -Para as pesquisas foram criadas views que consistem em funções de javascript para map dos dados dentro do CouchBase, as views são criadas dentro da interface web do CouchBase. Pode ser encontrado um guia para criação de view em http://www.couchbase.com/docs/couchbase-manual-2.0/couchbase-views-types.html - - + Para as pesquisas foram criadas views que consistem em funções de javascript para map dos dados dentro do CouchBase, as views são criadas dentro da interface web do CouchBase. Pode ser encontrado um guia para criação de view em + Foram criadas 2 views: + - com o nome "filter": + function (doc, meta) { + if (meta.type == "base64") { + emit("base64", doc) + } + else { + if (doc.read == false) { + emit(meta.id, doc); + } + } + } + + - com o nome "all": + function (doc, meta) { + emit(meta.id, doc) + } + + É necessário publicá-las (clicando em publish) para que o código tenha acesso à elas e funcione corretamente.