Skip to content

Commit 7692fa8

Browse files
authored
🪣 fix: S3 path-style URL support for MinIO, R2, and custom endpoints (danny-avila#11894)
* 🪣 fix: S3 path-style URL support for MinIO, R2, and custom endpoints `extractKeyFromS3Url` now uses `AWS_BUCKET_NAME` to automatically detect and strip the bucket prefix from path-style URLs, fixing `NoSuchKey` errors on URL refresh for any S3-compatible provider using a custom endpoint (MinIO, Cloudflare R2, Hetzner, Backblaze B2, etc.). No additional configuration required — the bucket name is already a required env var for S3 to function. `initializeS3` now passes `forcePathStyle: true` to the S3Client constructor when `AWS_FORCE_PATH_STYLE=true` is set. Required for providers whose SSL certificates do not support virtual-hosted-style bucket subdomains (e.g. Hetzner Object Storage), which previously caused 401 / SignatureDoesNotMatch on upload. Additional fixes: - Suppress error log noise in `extractKeyFromS3Url` catch path: plain S3 keys no longer log as errors, only inputs that start with http(s):// do - Fix test env var ordering so module-level constants pick up `AWS_BUCKET_NAME` and `S3_URL_EXPIRY_SECONDS` correctly before the module is required - Add missing `deleteRagFile` mock and assertion in `deleteFileFromS3` tests - Add `AWS_BUCKET_NAME` cleanup to `afterEach` to prevent cross-test pollution - Add `initializeS3` unit tests covering endpoint, forcePathStyle, credentials, singleton, and IRSA code paths - Document `AWS_FORCE_PATH_STYLE` in `.env.example`, `dotenv.mdx`, and `s3.mdx` * 🪣 fix: Enhance S3 URL key extraction for custom endpoints Updated `extractKeyFromS3Url` to support precise key extraction when using custom endpoints with path-style URLs. The logic now accounts for the `AWS_ENDPOINT_URL` and `AWS_FORCE_PATH_STYLE` environment variables, ensuring correct key handling for various S3-compatible providers. Added unit tests to verify the new functionality, including scenarios for endpoints with base paths. This improves compatibility and reduces potential errors when interacting with S3-like services.
1 parent b7bfdfa commit 7692fa8

5 files changed

Lines changed: 182 additions & 2 deletions

File tree

‎.env.example‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -658,6 +658,9 @@ AWS_ACCESS_KEY_ID=
658658
AWS_SECRET_ACCESS_KEY=
659659
AWS_REGION=
660660
AWS_BUCKET_NAME=
661+
# Required for path-style S3-compatible providers (MinIO, Hetzner, Backblaze B2, etc.)
662+
# that don't support virtual-hosted-style URLs (bucket.endpoint). Not needed for AWS S3.
663+
# AWS_FORCE_PATH_STYLE=false
661664

662665
#========================#
663666
# Azure Blob Storage #

‎api/server/services/Files/S3/crud.js‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ const fetch = require('node-fetch');
33
const { logger } = require('@librechat/data-schemas');
44
const { FileSources } = require('librechat-data-provider');
55
const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');
6-
const { initializeS3, deleteRagFile } = require('@librechat/api');
6+
const { initializeS3, deleteRagFile, isEnabled } = require('@librechat/api');
77
const {
88
PutObjectCommand,
99
GetObjectCommand,
@@ -13,6 +13,8 @@ const {
1313

1414
const bucketName = process.env.AWS_BUCKET_NAME;
1515
const defaultBasePath = 'images';
16+
const endpoint = process.env.AWS_ENDPOINT_URL;
17+
const forcePathStyle = isEnabled(process.env.AWS_FORCE_PATH_STYLE);
1618

1719
let s3UrlExpirySeconds = 2 * 60; // 2 minutes
1820
let s3RefreshExpiryMs = null;
@@ -255,6 +257,26 @@ function extractKeyFromS3Url(fileUrlOrKey) {
255257
const hostname = url.hostname;
256258
const pathname = url.pathname.substring(1); // Remove leading slash
257259

260+
// Explicit path-style with custom endpoint: use endpoint pathname for precise key extraction.
261+
// Handles endpoints with a base path (e.g. https://example.com/storage/).
262+
if (endpoint && forcePathStyle) {
263+
const endpointUrl = new URL(endpoint);
264+
const startPos =
265+
endpointUrl.pathname.length +
266+
(endpointUrl.pathname.endsWith('/') ? 0 : 1) +
267+
bucketName.length +
268+
1;
269+
const key = url.pathname.substring(startPos);
270+
if (!key) {
271+
logger.warn(
272+
`[extractKeyFromS3Url] Extracted key is empty for endpoint path-style URL: ${fileUrlOrKey}`,
273+
);
274+
} else {
275+
logger.debug(`[extractKeyFromS3Url] fileUrlOrKey: ${fileUrlOrKey}, Extracted key: ${key}`);
276+
}
277+
return key;
278+
}
279+
258280
if (
259281
hostname === 's3.amazonaws.com' ||
260282
hostname.match(/^s3[-.][a-z0-9-]+\.amazonaws\.com$/) ||

‎api/test/services/Files/S3/crud.test.js‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ jest.mock('@aws-sdk/client-s3');
1919
jest.mock('@librechat/api', () => ({
2020
initializeS3: jest.fn(),
2121
deleteRagFile: jest.fn().mockResolvedValue(undefined),
22+
isEnabled: jest.fn((val) => val === 'true'),
2223
}));
2324

2425
jest.mock('@librechat/data-schemas', () => ({
@@ -841,5 +842,35 @@ describe('S3 CRUD Operations', () => {
841842
),
842843
).toBe('images/user123/avatar.png');
843844
});
845+
846+
it('should use endpoint base path when AWS_ENDPOINT_URL and AWS_FORCE_PATH_STYLE are set', () => {
847+
process.env.AWS_BUCKET_NAME = 'test-bucket';
848+
process.env.AWS_ENDPOINT_URL = 'https://minio.example.com';
849+
process.env.AWS_FORCE_PATH_STYLE = 'true';
850+
jest.resetModules();
851+
const { extractKeyFromS3Url: fn } = require('~/server/services/Files/S3/crud');
852+
853+
expect(fn('https://minio.example.com/test-bucket/images/user123/file.jpg')).toBe(
854+
'images/user123/file.jpg',
855+
);
856+
857+
delete process.env.AWS_ENDPOINT_URL;
858+
delete process.env.AWS_FORCE_PATH_STYLE;
859+
});
860+
861+
it('should handle endpoint with a base path', () => {
862+
process.env.AWS_BUCKET_NAME = 'test-bucket';
863+
process.env.AWS_ENDPOINT_URL = 'https://example.com/storage/';
864+
process.env.AWS_FORCE_PATH_STYLE = 'true';
865+
jest.resetModules();
866+
const { extractKeyFromS3Url: fn } = require('~/server/services/Files/S3/crud');
867+
868+
expect(fn('https://example.com/storage/test-bucket/images/user123/file.jpg')).toBe(
869+
'images/user123/file.jpg',
870+
);
871+
872+
delete process.env.AWS_ENDPOINT_URL;
873+
delete process.env.AWS_FORCE_PATH_STYLE;
874+
});
844875
});
845876
});
Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
import type { S3Client } from '@aws-sdk/client-s3';
2+
3+
const mockLogger = { info: jest.fn(), error: jest.fn() };
4+
5+
jest.mock('@aws-sdk/client-s3', () => ({
6+
S3Client: jest.fn(),
7+
}));
8+
9+
jest.mock('@librechat/data-schemas', () => ({
10+
logger: mockLogger,
11+
}));
12+
13+
describe('initializeS3', () => {
14+
const REQUIRED_ENV = {
15+
AWS_REGION: 'us-east-1',
16+
AWS_BUCKET_NAME: 'test-bucket',
17+
AWS_ACCESS_KEY_ID: 'test-key-id',
18+
AWS_SECRET_ACCESS_KEY: 'test-secret',
19+
};
20+
21+
beforeEach(() => {
22+
jest.resetModules();
23+
jest.clearAllMocks();
24+
Object.assign(process.env, REQUIRED_ENV);
25+
delete process.env.AWS_ENDPOINT_URL;
26+
delete process.env.AWS_FORCE_PATH_STYLE;
27+
});
28+
29+
afterEach(() => {
30+
for (const key of Object.keys(REQUIRED_ENV)) {
31+
delete process.env[key];
32+
}
33+
delete process.env.AWS_ENDPOINT_URL;
34+
delete process.env.AWS_FORCE_PATH_STYLE;
35+
});
36+
37+
async function load() {
38+
const { S3Client: MockS3Client } = jest.requireMock('@aws-sdk/client-s3') as {
39+
S3Client: jest.MockedClass<typeof S3Client>;
40+
};
41+
const { initializeS3 } = await import('../s3');
42+
return { MockS3Client, initializeS3 };
43+
}
44+
45+
it('should initialize with region and credentials', async () => {
46+
const { MockS3Client, initializeS3 } = await load();
47+
initializeS3();
48+
expect(MockS3Client).toHaveBeenCalledWith(
49+
expect.objectContaining({
50+
region: 'us-east-1',
51+
credentials: { accessKeyId: 'test-key-id', secretAccessKey: 'test-secret' },
52+
}),
53+
);
54+
});
55+
56+
it('should include endpoint when AWS_ENDPOINT_URL is set', async () => {
57+
process.env.AWS_ENDPOINT_URL = 'https://fsn1.your-objectstorage.com';
58+
const { MockS3Client, initializeS3 } = await load();
59+
initializeS3();
60+
expect(MockS3Client).toHaveBeenCalledWith(
61+
expect.objectContaining({ endpoint: 'https://fsn1.your-objectstorage.com' }),
62+
);
63+
});
64+
65+
it('should not include endpoint when AWS_ENDPOINT_URL is not set', async () => {
66+
const { MockS3Client, initializeS3 } = await load();
67+
initializeS3();
68+
const config = MockS3Client.mock.calls[0][0] as Record<string, unknown>;
69+
expect(config).not.toHaveProperty('endpoint');
70+
});
71+
72+
it('should set forcePathStyle when AWS_FORCE_PATH_STYLE is true', async () => {
73+
process.env.AWS_FORCE_PATH_STYLE = 'true';
74+
const { MockS3Client, initializeS3 } = await load();
75+
initializeS3();
76+
expect(MockS3Client).toHaveBeenCalledWith(expect.objectContaining({ forcePathStyle: true }));
77+
});
78+
79+
it('should not set forcePathStyle when AWS_FORCE_PATH_STYLE is false', async () => {
80+
process.env.AWS_FORCE_PATH_STYLE = 'false';
81+
const { MockS3Client, initializeS3 } = await load();
82+
initializeS3();
83+
const config = MockS3Client.mock.calls[0][0] as Record<string, unknown>;
84+
expect(config).not.toHaveProperty('forcePathStyle');
85+
});
86+
87+
it('should not set forcePathStyle when AWS_FORCE_PATH_STYLE is not set', async () => {
88+
const { MockS3Client, initializeS3 } = await load();
89+
initializeS3();
90+
const config = MockS3Client.mock.calls[0][0] as Record<string, unknown>;
91+
expect(config).not.toHaveProperty('forcePathStyle');
92+
});
93+
94+
it('should return null and log error when AWS_REGION is not set', async () => {
95+
delete process.env.AWS_REGION;
96+
const { initializeS3 } = await load();
97+
const result = initializeS3();
98+
expect(result).toBeNull();
99+
expect(mockLogger.error).toHaveBeenCalledWith(
100+
'[initializeS3] AWS_REGION is not set. Cannot initialize S3.',
101+
);
102+
});
103+
104+
it('should return the same instance on subsequent calls', async () => {
105+
const { MockS3Client, initializeS3 } = await load();
106+
const first = initializeS3();
107+
const second = initializeS3();
108+
expect(first).toBe(second);
109+
expect(MockS3Client).toHaveBeenCalledTimes(1);
110+
});
111+
112+
it('should use default credentials chain when keys are not provided', async () => {
113+
delete process.env.AWS_ACCESS_KEY_ID;
114+
delete process.env.AWS_SECRET_ACCESS_KEY;
115+
const { MockS3Client, initializeS3 } = await load();
116+
initializeS3();
117+
const config = MockS3Client.mock.calls[0][0] as Record<string, unknown>;
118+
expect(config).not.toHaveProperty('credentials');
119+
expect(mockLogger.info).toHaveBeenCalledWith(
120+
'[initializeS3] S3 initialized using default credentials (IRSA).',
121+
);
122+
});
123+
});

‎packages/api/src/cdn/s3.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { S3Client } from '@aws-sdk/client-s3';
22
import { logger } from '@librechat/data-schemas';
3+
import { isEnabled } from '~/utils/common';
34

45
let s3: S3Client | null = null;
56

@@ -31,8 +32,8 @@ export const initializeS3 = (): S3Client | null => {
3132

3233
const config = {
3334
region,
34-
// Conditionally add the endpoint if it is provided
3535
...(endpoint ? { endpoint } : {}),
36+
...(isEnabled(process.env.AWS_FORCE_PATH_STYLE) ? { forcePathStyle: true } : {}),
3637
};
3738

3839
if (accessKeyId && secretAccessKey) {

0 commit comments

Comments
 (0)