Skip to content

Commit 1faa7b9

Browse files
psavpsav
committed
Refactor
1 parent 8fea231 commit 1faa7b9

8 files changed

Lines changed: 303 additions & 210 deletions

File tree

‎.github/workflows/tester.yml‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,13 @@ jobs:
2222
- name: Perform perf test 1 - Random
2323
run: |
2424
python ./test/perf/node_utils.py file ./test/perf/topology-random.yaml&
25-
python ./test/perf/node_utils.py dot-compare graph_controller.dot last-topology_graph.dot --wait 40
25+
python ./test/perf/node_utils.py dot-compare graph_controller.dot .last-topology-graph.dot --wait 40
2626
python ./test/perf/node_utils.py ping ./test/perf/topology-random.yaml --count 100 --validate 0.1
27-
python ./test/perf/node_utils.py check-stats last-topology.yaml
27+
python ./test/perf/node_utils.py check-stats .last-topology.yaml
2828
kill `pidof python`
2929
rm -Rf /tmp/receptor
3030
rm graph_controller.dot
31-
rm last-topology_graph.dot
31+
rm .last-topology-graph.dot
3232
- name: Upload artifact for Perf - Random
3333
uses: actions/upload-artifact@v1
3434
with:
@@ -37,13 +37,13 @@ jobs:
3737
- name: Perform perf test 2 - Flat
3838
run: |
3939
python ./test/perf/node_utils.py file ./test/perf/topology-flat.yaml&
40-
python ./test/perf/node_utils.py dot-compare graph_controller.dot last-topology_graph.dot --wait 40
40+
python ./test/perf/node_utils.py dot-compare graph_controller.dot .last-topology-graph.dot --wait 40
4141
python ./test/perf/node_utils.py ping ./test/perf/topology-flat.yaml --count 100 --validate 0.1
42-
python ./test/perf/node_utils.py check-stats last-topology.yaml
42+
python ./test/perf/node_utils.py check-stats .last-topology.yaml
4343
kill `pidof python`
4444
rm -Rf /tmp/receptor
4545
rm graph_controller.dot
46-
rm last-topology_graph.dot
46+
rm .last-topology-graph.dot
4747
- name: Upload artifact for Perf - Flat
4848
uses: actions/upload-artifact@v1
4949
with:
@@ -52,13 +52,13 @@ jobs:
5252
- name: Perform perf test 3 - Tree
5353
run: |
5454
python ./test/perf/node_utils.py file ./test/perf/topology-tree.yaml&
55-
python ./test/perf/node_utils.py dot-compare graph_controller.dot last-topology_graph.dot --wait 40
55+
python ./test/perf/node_utils.py dot-compare graph_controller.dot .last-topology-graph.dot --wait 40
5656
python ./test/perf/node_utils.py ping ./test/perf/topology-tree.yaml --count 100 --validate 0.1
57-
python ./test/perf/node_utils.py check-stats last-topology.yaml
57+
python ./test/perf/node_utils.py check-stats .last-topology.yaml
5858
kill `pidof python`
5959
rm -Rf /tmp/receptor
6060
rm graph_controller.dot
61-
rm last-topology_graph.dot
61+
rm .last-topology-graph.dot
6262
- name: Upload artifact for Perf - Tree
6363
uses: actions/upload-artifact@v1
6464
with:

‎test/perf/affinity.py‎

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
import random
2+
from collections import defaultdict
3+
import subprocess
4+
import attr
5+
import atexit
6+
from utils import random_port
7+
import uuid
8+
import yaml
9+
10+
procs = {}
11+
12+
13+
def shut_all_procs():
14+
for proc in procs.values():
15+
proc.kill()
16+
17+
18+
atexit.register(shut_all_procs)
19+
20+
21+
@attr.s
22+
class Node:
23+
name = attr.ib()
24+
controller = attr.ib(default=False)
25+
listen_port = attr.ib(factory=random_port)
26+
connections = attr.ib(factory=list)
27+
stats_enable = attr.ib(default=False)
28+
stats_port = attr.ib(default=None)
29+
profile = attr.ib(default=False)
30+
socket_path = attr.ib(default=None)
31+
data_path = attr.ib(default=None)
32+
topology = attr.ib(init=False, default=None)
33+
uuid = attr.ib(init=False, factory=uuid.uuid4)
34+
35+
def __attrs_post_init__(self):
36+
if not self.socket_path:
37+
self.socket_path = f"/tmp/receptor/{str(self.uuid)}/receptor.sock"
38+
if not self.data_path:
39+
self.data_path = f"/tmp/receptor/{str(self.uuid)}"
40+
41+
@staticmethod
42+
def create_from_config(config):
43+
return Node(
44+
name=config["name"],
45+
controller=config.get("controller", False),
46+
listen_port=config.get("listen_port", None) or random_port(),
47+
connections=config.get("connections", []) or [],
48+
stats_enable=config.get("stats_enable", False),
49+
stats_port=config.get("stats_port", None) or random_port(),
50+
profile=config.get("profile", False),
51+
socket_path=config.get("socket_path", None),
52+
data_path=config.get("data_path", None),
53+
)
54+
55+
def _construct_run_command(self):
56+
if self.profile:
57+
st = ["python", "-m", "cProfile", "-o", f"{self.name}.prof", "-m", "receptor.__main__"]
58+
else:
59+
st = ["receptor"]
60+
61+
if self.controller:
62+
st.extend(["--debug", "-d", self.data_path, "--node-id", self.name, "controller"])
63+
st.extend([f"--socket-path={self.socket_path}"])
64+
st.extend([f"--listen-port={self.listen_port}"])
65+
else:
66+
peer_string = " ".join(
67+
[
68+
f"--peer=localhost:{self.topology.nodes[pnode].listen_port}"
69+
for pnode in self.connections
70+
]
71+
)
72+
st.extend(["-d", self.data_path, "--node-id", self.name, "node"])
73+
st.extend([f"--listen-port={self.listen_port}", peer_string])
74+
75+
if self.stats_enable:
76+
st.extend(["--stats-enable", f"--stats-port={self.stats_port}"])
77+
78+
return st
79+
80+
def start(self):
81+
op = subprocess.Popen(" ".join(self._construct_run_command()), shell=True)
82+
procs[self.uuid] = op
83+
84+
def stop(self):
85+
print(f"killing {self.name}({self.uuid})")
86+
procs[self.uuid].kill()
87+
88+
89+
@attr.s
90+
class Topology:
91+
nodes = attr.ib(init=False, factory=dict)
92+
93+
def add_node(self, node):
94+
if node.name not in self.nodes:
95+
self.nodes[node.name] = node
96+
node.topology = self
97+
else:
98+
raise Exception("Topology already has a node by the same name")
99+
100+
def remove_node(self, node_or_name):
101+
if isinstance(node_or_name, Node):
102+
node_name = node_or_name.name
103+
else:
104+
node_name = node_or_name
105+
if node_name not in self.nodes:
106+
raise Exception("Topology has no node by that name")
107+
else:
108+
self.nodes[node_name].topology = None
109+
del self.nodes[node_name]
110+
111+
@staticmethod
112+
def generate_mesh(
113+
controller_port, node_count, conn_method, profile=False, socket_path=None
114+
):
115+
topology = Topology()
116+
topology.add_node(
117+
Node(
118+
name="controller",
119+
controller=True,
120+
listen_port=controller_port,
121+
profile=profile,
122+
socket_path=socket_path,
123+
)
124+
)
125+
126+
for i in range(node_count):
127+
topology.add_node(
128+
Node(name=f"node{i}", controller=False, listen_port=random_port(), profile=profile)
129+
)
130+
131+
for k, node in topology.nodes.items():
132+
if node.controller:
133+
continue
134+
else:
135+
node.connections.extend(conn_method(topology, node))
136+
return topology
137+
138+
@staticmethod
139+
def generate_random_mesh(controller_port, node_count, max_conn_count, profile, socket_path):
140+
def peer_function(topology, cur_node):
141+
nconns = defaultdict(int)
142+
print(topology)
143+
for k, node in topology.nodes.items():
144+
for conn in node.connections:
145+
nconns[conn] += 1
146+
available_nodes = list(filter(lambda o: nconns[o] < max_conn_count, topology.nodes))
147+
print("------")
148+
print(nconns)
149+
print(available_nodes)
150+
print(cur_node.name)
151+
print(random.choices(available_nodes, k=int(random.random() * max_conn_count)))
152+
print("----")
153+
if cur_node.name not in available_nodes:
154+
return []
155+
else:
156+
return random.choices(available_nodes, k=int(random.random() * max_conn_count))
157+
158+
topology = Topology.generate_random_mesh(
159+
controller_port, node_count, peer_function, profile, socket_path
160+
)
161+
return topology
162+
163+
@staticmethod
164+
def generate_flat_mesh(controller_port, node_count, profile, socket_path):
165+
def peer_function(*args):
166+
return ["controller"]
167+
168+
topology = Topology.generate_random_mesh(
169+
controller_port, node_count, peer_function, profile, socket_path
170+
)
171+
return topology
172+
173+
174+
def dump_yaml(self, filename=".last-topology.yaml"):
175+
with open(filename, "w") as f:
176+
data = {"nodes": {}}
177+
for node, node_data in self.nodes.items():
178+
data["nodes"][node] = {
179+
"name": node_data.name,
180+
"listen_port": node_data.listen_port if node_data.controller else None,
181+
"controller": node_data.controller,
182+
"connections": node_data.connections,
183+
"stats_enable": node_data.stats_enable,
184+
"stats_port": node_data.stats_port,
185+
}
186+
if node_data.socket_path:
187+
data["nodes"][node]["socket_path"] = node_data.socket_path
188+
if node_data.data_path:
189+
data["nodes"][node]["data_path"] = node_data.data_path
190+
191+
yaml.dump(data, f)
192+
193+
def dump_dot(self, filename=".last-topology-graph.dot"):
194+
with open(filename, "w") as f:
195+
f.write("graph {")
196+
for node, node_data in self.nodes.items():
197+
for conn in node_data.connections:
198+
f.write(f"{node} -- {conn}; ")
199+
f.write("}")
200+
201+
def start(self):
202+
self.dump_yaml()
203+
self.dump_dot()
204+
205+
for k, node in self.nodes.items():
206+
node.start()
207+
208+
def stop(self):
209+
for k, node in self.nodes.items():
210+
node.stop()
211+
print("all killed")
212+
213+
@staticmethod
214+
def load_topology_from_file(filename):
215+
data = yaml.safe_load(filename)
216+
topology = Topology()
217+
for node_name, definition in data["nodes"].items():
218+
node = Node.create_from_config(definition)
219+
topology.add_node(node)
220+
221+
return topology
222+
223+
def find_controller(self):
224+
return list(filter(lambda o: o.controller, self.nodes.values()))

0 commit comments

Comments
 (0)