Skip to content

Commit 4dcd4af

Browse files
ehsan6shaehsan6sha
committed
added benchmark
1 parent c1bb3dc commit 4dcd4af

14 files changed

Lines changed: 594 additions & 3 deletions

‎benchmark_s3.py‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1354,6 +1354,14 @@ def _error_explanation(category: str) -> str:
13541354
# Main
13551355
# ---------------------------------------------------------------------------
13561356

1357+
# Default output goes to docs/website/ so the benchmark page can load it
1358+
# directly without any manual copying.
1359+
_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
1360+
_DEFAULT_OUTPUT = os.path.join(
1361+
_SCRIPT_DIR, "docs", "website", "assets", "benchmark", "benchmark_results.md"
1362+
)
1363+
1364+
13571365
def parse_args() -> argparse.Namespace:
13581366
parser = argparse.ArgumentParser(
13591367
description="S3-Compatible Benchmark for Fula API",
@@ -1380,8 +1388,11 @@ def parse_args() -> argparse.Namespace:
13801388
)
13811389
parser.add_argument(
13821390
"--output",
1383-
default="benchmark_results.md",
1384-
help="Output markdown report path (default: benchmark_results.md)",
1391+
default=_DEFAULT_OUTPUT,
1392+
help=(
1393+
"Output markdown report path "
1394+
"(default: docs/website/assets/benchmark/benchmark_results.md)"
1395+
),
13851396
)
13861397
return parser.parse_args()
13871398

@@ -1392,6 +1403,10 @@ async def async_main():
13921403
endpoint = args.endpoint.rstrip("/")
13931404
output = args.output
13941405

1406+
# Ensure the output directory exists
1407+
output_dir = os.path.dirname(os.path.abspath(output))
1408+
os.makedirs(output_dir, exist_ok=True)
1409+
13951410
print(
13961411
f"Fula S3 Benchmark\n"
13971412
f" Endpoint: {endpoint}\n"
@@ -1413,7 +1428,10 @@ async def async_main():
14131428
report.generate()
14141429

14151430
print(f"Report saved to {output}", file=sys.stderr, flush=True)
1416-
print("4 chart PNGs generated alongside report.", file=sys.stderr, flush=True)
1431+
print(
1432+
f"4 chart PNGs generated in {output_dir}/",
1433+
file=sys.stderr, flush=True,
1434+
)
14171435

14181436

14191437
def main():

‎docs/website/api.html‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ <h3>Documentation</h3>
5959
<li><a href="sdk.html">SDK Examples</a></li>
6060
<li><a href="platforms.html">Platform Guides</a></li>
6161
<li><a href="x402.html">x402 AI Agent Storage</a></li>
62+
<li><a href="benchmark.html">Benchmark</a></li>
6263
</ul>
6364
</div>
6465
<div class="nav-section">

benchmark_latency_distribution.png renamed to docs/website/assets/benchmark/benchmark_latency_distribution.png

File renamed without changes.

benchmark_results.md renamed to docs/website/assets/benchmark/benchmark_results.md

File renamed without changes.

benchmark_success_rate.png renamed to docs/website/assets/benchmark/benchmark_success_rate.png

File renamed without changes.

benchmark_throughput_comparison.png renamed to docs/website/assets/benchmark/benchmark_throughput_comparison.png

File renamed without changes.

benchmark_ttfb_comparison.png renamed to docs/website/assets/benchmark/benchmark_ttfb_comparison.png

File renamed without changes.

‎docs/website/benchmark.html‎

Lines changed: 249 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,249 @@
1+
<!DOCTYPE html>
2+
<html lang="en" data-theme="dark">
3+
<head>
4+
<meta charset="UTF-8">
5+
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=5.0, user-scalable=yes">
6+
<title>Benchmark - Fula API Documentation</title>
7+
<link rel="stylesheet" href="css/styles.css">
8+
<link rel="stylesheet" href="css/benchmark.css">
9+
<link rel="stylesheet" href="css/i18n.css">
10+
<script>
11+
(function() {
12+
const savedTheme = localStorage.getItem('fula-docs-theme');
13+
if (savedTheme) document.documentElement.setAttribute('data-theme', savedTheme);
14+
const savedLang = localStorage.getItem('fula-docs-language');
15+
if (savedLang && ['fa', 'ar'].includes(savedLang)) {
16+
document.documentElement.setAttribute('dir', 'rtl');
17+
document.documentElement.setAttribute('lang', savedLang);
18+
}
19+
})();
20+
</script>
21+
</head>
22+
<body>
23+
<!-- Mobile Header -->
24+
<header class="mobile-header">
25+
<button class="hamburger" aria-label="Toggle menu">
26+
<span></span>
27+
<span></span>
28+
<span></span>
29+
</button>
30+
<span class="logo-text">Fula API</span>
31+
<div class="mobile-header-actions">
32+
<button class="theme-toggle" aria-label="Toggle theme">
33+
<span class="icon-sun">☀️</span>
34+
<span class="icon-moon">🌙</span>
35+
</button>
36+
</div>
37+
</header>
38+
39+
<div class="sidebar-overlay"></div>
40+
41+
<nav class="sidebar">
42+
<div class="sidebar-header">
43+
<div class="logo">
44+
<h1>Fula API</h1>
45+
<span class="version">v0.1.0</span>
46+
</div>
47+
<button class="theme-toggle" aria-label="Toggle theme">
48+
<span class="icon-sun">☀️</span>
49+
<span class="icon-moon">🌙</span>
50+
</button>
51+
</div>
52+
<div class="nav-section">
53+
<h3>Documentation</h3>
54+
<ul>
55+
<li><a href="index.html">Introduction</a></li>
56+
<li><a href="security.html">Security & Encryption</a></li>
57+
<li><a href="api.html">API Reference</a></li>
58+
<li><a href="sdk.html">SDK Examples</a></li>
59+
<li><a href="platforms.html">Platform Guides</a></li>
60+
<li><a href="x402.html">x402 AI Agent Storage</a></li>
61+
<li><a href="benchmark.html" class="active">Benchmark</a></li>
62+
</ul>
63+
</div>
64+
<div class="nav-section">
65+
<h3>On This Page</h3>
66+
<ul>
67+
<li><a href="#about">About This Benchmark</a></li>
68+
<li><a href="#run-it">How to Run</a></li>
69+
<li><a href="#results">Benchmark Results</a></li>
70+
</ul>
71+
</div>
72+
</nav>
73+
74+
<main class="content">
75+
<!-- Hero -->
76+
<section class="benchmark-hero">
77+
<h1>S3 Gateway Performance Benchmark</h1>
78+
<p>
79+
Automated performance testing of the Fula decentralized S3-compatible storage gateway,
80+
compared against published industry baselines from AWS S3, IPFS, and Cloudflare R2.
81+
</p>
82+
</section>
83+
84+
<!-- About -->
85+
<section id="about" class="benchmark-section">
86+
<h2>About This Benchmark</h2>
87+
<p>
88+
The Fula S3 Gateway provides an S3-compatible API backed by decentralized IPFS storage.
89+
This benchmark measures real-world upload and download performance across different
90+
file sizes and concurrency levels, then compares the results against industry standards.
91+
</p>
92+
<h3>What is tested</h3>
93+
<ul>
94+
<li><strong>File sizes:</strong> 64 KB, 1 MB, and 10 MB objects</li>
95+
<li><strong>Concurrency:</strong> 1 (sequential), 10, and 50 parallel requests</li>
96+
<li><strong>Operations:</strong> Upload (PUT), Download (GET), and Mixed 80/20 read/write</li>
97+
<li><strong>Metrics:</strong> Time to First Byte (TTFB), throughput (MB/s), latency percentiles, success rate</li>
98+
</ul>
99+
<h3>Industry baselines compared</h3>
100+
<ul>
101+
<li><strong>AWS S3 Standard</strong> &mdash; same-region EC2, published benchmarks</li>
102+
<li><strong>IPFS Public Gateways</strong> &mdash; cached and uncached content retrieval</li>
103+
<li><strong>Cloudflare R2</strong> &mdash; S3-compatible API latency</li>
104+
</ul>
105+
</section>
106+
107+
<!-- How to run -->
108+
<section id="run-it" class="benchmark-section">
109+
<h2>How to Run the Benchmark</h2>
110+
<p>
111+
The benchmark is a self-contained Python script that tests any S3-compatible endpoint
112+
and generates a detailed markdown report with charts.
113+
</p>
114+
<h3>Prerequisites</h3>
115+
<pre><code>pip install aiohttp matplotlib
116+
# Python 3.9+ required</code></pre>
117+
<h3>Run the benchmark</h3>
118+
<pre><code>python benchmark_s3.py --token "YOUR_JWT_TOKEN" --endpoint https://s3.cloud.fx.land</code></pre>
119+
<h3>Options</h3>
120+
<ul>
121+
<li><code>--token</code> &mdash; (required) JWT Bearer token for authentication</li>
122+
<li><code>--endpoint</code> &mdash; S3 endpoint URL (default: <code>https://s3.cloud.fx.land</code>)</li>
123+
<li><code>--bucket</code> &mdash; Bucket name (default: <code>benchmark-{timestamp}</code>)</li>
124+
<li><code>--output</code> &mdash; Output report path (default: <code>benchmark_results.md</code>)</li>
125+
</ul>
126+
<h3>Output</h3>
127+
<p>
128+
The script produces a markdown report (<code>benchmark_results.md</code>) and
129+
4 chart images (<code>benchmark_*.png</code>) in the same directory.
130+
The report shown below is loaded directly from the latest generated results.
131+
</p>
132+
<a class="script-link"
133+
href="https://github.com/functionland/fula-api/blob/main/benchmark_s3.py"
134+
target="_blank" rel="noopener">
135+
View benchmark_s3.py on GitHub &rarr;
136+
</a>
137+
</section>
138+
139+
<!-- Results (loaded dynamically from benchmark_results.md) -->
140+
<section id="results" class="benchmark-section">
141+
<h2>Benchmark Results</h2>
142+
</section>
143+
144+
<div id="benchmark-report">
145+
<div class="benchmark-loading" id="benchmark-loading">
146+
<span class="spinner"></span>
147+
Loading benchmark results&hellip;
148+
</div>
149+
</div>
150+
151+
<footer>
152+
<p>Fula API Benchmark Documentation &bull; Built with decentralized storage in mind</p>
153+
<p class="footer-links">
154+
<a href="https://github.com/functionland/fula-api">GitHub</a> &bull;
155+
<a href="https://fx.land">Functionland</a>
156+
</p>
157+
</footer>
158+
</main>
159+
160+
<script src="js/i18n.js"></script>
161+
<script src="js/translations.js"></script>
162+
<script src="js/app.js"></script>
163+
164+
<!-- Marked.js for rendering markdown -->
165+
<script src="https://cdn.jsdelivr.net/npm/marked@15/marked.min.js"></script>
166+
<script>
167+
(function() {
168+
// Paths to try in order (local deploy, then GitHub raw)
169+
var REPORT_PATHS = [
170+
'assets/benchmark/benchmark_results.md',
171+
'https://raw.githubusercontent.com/functionland/fula-api/main/docs/website/assets/benchmark/benchmark_results.md'
172+
];
173+
174+
var IMAGE_BASES = [
175+
'assets/benchmark/',
176+
'https://raw.githubusercontent.com/functionland/fula-api/main/docs/website/assets/benchmark/'
177+
];
178+
179+
var container = document.getElementById('benchmark-report');
180+
var loadingEl = document.getElementById('benchmark-loading');
181+
182+
function rewriteImages(html, base) {
183+
// Rewrite relative image src paths to use the same base as the markdown
184+
return html.replace(
185+
/(<img\s+[^>]*src=")(?!https?:\/\/)([^"]+)(")/g,
186+
'$1' + base + '$2$3'
187+
);
188+
}
189+
190+
function renderMarkdown(md, imageBase) {
191+
if (typeof marked === 'undefined' || typeof marked.parse !== 'function') {
192+
container.innerHTML = '<div class="benchmark-placeholder"><p>Markdown renderer not available. ' +
193+
'<a href="https://github.com/functionland/fula-api/blob/main/benchmark_results.md" target="_blank">' +
194+
'View results on GitHub</a></p></div>';
195+
return;
196+
}
197+
var html = marked.parse(md);
198+
html = rewriteImages(html, imageBase);
199+
container.innerHTML = html;
200+
}
201+
202+
function showPlaceholder() {
203+
container.innerHTML =
204+
'<div class="benchmark-placeholder">' +
205+
'<h3>No benchmark results available yet</h3>' +
206+
'<p>Run the benchmark script to generate results:</p>' +
207+
'<pre><code>python benchmark_s3.py --token "YOUR_JWT"</code></pre>' +
208+
'<p>The generated <code>benchmark_results.md</code> and chart images will appear here automatically.</p>' +
209+
'</div>';
210+
}
211+
212+
function tryFetch(index) {
213+
if (index >= REPORT_PATHS.length) {
214+
showPlaceholder();
215+
return;
216+
}
217+
var url = REPORT_PATHS[index];
218+
var imageBase = IMAGE_BASES[index] || '';
219+
220+
fetch(url)
221+
.then(function(resp) {
222+
if (!resp.ok) throw new Error('HTTP ' + resp.status);
223+
return resp.text();
224+
})
225+
.then(function(md) {
226+
// Sanity check: must look like our report
227+
if (md.indexOf('Benchmark') === -1 && md.indexOf('benchmark') === -1) {
228+
throw new Error('Not a benchmark report');
229+
}
230+
renderMarkdown(md, imageBase);
231+
})
232+
.catch(function() {
233+
tryFetch(index + 1);
234+
});
235+
}
236+
237+
// Wait for marked.js to load, then fetch
238+
function init() {
239+
if (typeof marked !== 'undefined') {
240+
tryFetch(0);
241+
} else {
242+
setTimeout(init, 100);
243+
}
244+
}
245+
init();
246+
})();
247+
</script>
248+
</body>
249+
</html>

0 commit comments

Comments
 (0)