Skip to content

Commit c583c61

Browse files
ClaudioESSilvapotatoqualitee
ClaudioESSilva
and
potatoqualitee
authored
S3 restores - Improve handling, docs and tests (#10137)
Co-authored-by: Chrissy LeMaire <potatoqualitee@users.noreply.github.com>
1 parent 6c70c78 commit c583c61

5 files changed

Lines changed: 265 additions & 13 deletions

File tree

‎.github/scripts/gh-s3actions.ps1‎

Lines changed: 219 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -300,13 +300,224 @@ Describe "S3 Backup Integration Tests" -Tag "IntegrationTests", "S3" {
300300
}
301301
}
302302

303+
Context "S3 directory enumeration limitations" {
304+
BeforeAll {
305+
# This context validates that SQL Server cannot enumerate S3 bucket contents using T-SQL
306+
# (xp_dirtree/sys.dm_os_enumerate_filesystem don't support S3 protocol)
307+
# Get-DbaBackupInformation should detect S3 URLs and handle them appropriately
308+
309+
$script:TestDbName5 = "dbatoolsci_s3enum"
310+
311+
# Create and backup a database to S3 for enumeration testing
312+
$null = New-DbaDatabase -SqlInstance localhost -SqlCredential $cred -Name $script:TestDbName5 -RecoveryModel Full
313+
314+
# Create multiple backup files in S3 to test enumeration behavior
315+
$script:S3EnumFolder = "enumtest"
316+
$splatBackup1 = @{
317+
SqlInstance = "localhost"
318+
SqlCredential = $cred
319+
Database = $script:TestDbName5
320+
StorageBaseUrl = "$($script:S3BaseUrl)/$($script:S3EnumFolder)"
321+
StorageCredential = $script:S3CredentialName
322+
FilePath = "$($script:TestDbName5)_full1.bak"
323+
Type = "Full"
324+
}
325+
$null = Backup-DbaDatabase @splatBackup1
326+
327+
$splatBackup2 = @{
328+
SqlInstance = "localhost"
329+
SqlCredential = $cred
330+
Database = $script:TestDbName5
331+
StorageBaseUrl = "$($script:S3BaseUrl)/$($script:S3EnumFolder)"
332+
StorageCredential = $script:S3CredentialName
333+
FilePath = "$($script:TestDbName5)_log1.trn"
334+
Type = "Log"
335+
}
336+
$null = Backup-DbaDatabase @splatBackup2
337+
}
338+
339+
AfterAll {
340+
$null = Remove-DbaDatabase -SqlInstance localhost -SqlCredential $cred -Database $script:TestDbName5 -Confirm:$false
341+
}
342+
343+
It "Should handle S3 folder paths when NoXpDirTree is not specified" {
344+
# Get-DbaBackupInformation internally uses xp_dirtree/sys.dm_os_enumerate_filesystem
345+
# When given an S3 folder path, it should detect S3 and skip enumeration
346+
$s3FolderPath = "$($script:S3BaseUrl)/$($script:S3EnumFolder)/"
347+
348+
$splatBackupInfo = @{
349+
SqlInstance = "localhost"
350+
SqlCredential = $cred
351+
Path = $s3FolderPath
352+
StorageCredential = $script:S3CredentialName
353+
WarningAction = "SilentlyContinue"
354+
}
355+
$result = Get-DbaBackupInformation @splatBackupInfo
356+
357+
# Should return empty because S3 enumeration is not supported via T-SQL
358+
$result | Should -BeNullOrEmpty
359+
}
360+
361+
It "Should write warning message when S3 folder enumeration is attempted" {
362+
$s3FolderPath = "$($script:S3BaseUrl)/$($script:S3EnumFolder)/"
363+
364+
# Capture warning messages
365+
$warningMessages = @()
366+
$splatBackupInfo = @{
367+
SqlInstance = "localhost"
368+
SqlCredential = $cred
369+
Path = $s3FolderPath
370+
StorageCredential = $script:S3CredentialName
371+
WarningVariable = "warningMessages"
372+
WarningAction = "SilentlyContinue"
373+
}
374+
$null = Get-DbaBackupInformation @splatBackupInfo
375+
376+
# Should have written a warning about S3 enumeration not being supported
377+
$warningMessages | Should -Not -BeNullOrEmpty
378+
$warningMessages -join " " | Should -BeLike "*S3 paths cannot be enumerated using T-SQL*"
379+
}
380+
381+
It "Should work with explicit S3 file paths (not folders)" {
382+
# While folder enumeration doesn't work, explicit file paths should work
383+
$s3FilePath = "$($script:S3BaseUrl)/$($script:S3EnumFolder)/$($script:TestDbName5)_full1.bak"
384+
385+
$splatBackupInfo = @{
386+
SqlInstance = "localhost"
387+
SqlCredential = $cred
388+
Path = $s3FilePath
389+
StorageCredential = $script:S3CredentialName
390+
}
391+
$result = Get-DbaBackupInformation @splatBackupInfo
392+
393+
# Should successfully read the specific file
394+
$result | Should -Not -BeNullOrEmpty
395+
$result.Database | Should -Be $script:TestDbName5
396+
$result.Type | Should -Be "Database"
397+
}
398+
399+
It "Should successfully enumerate local file system paths (contrast with S3)" {
400+
# Create a local backup to verify enumeration works for non-S3 paths
401+
# IMPORTANT: On Linux CI, SQL Server runs in Docker with an isolated filesystem.
402+
# We must use a path INSIDE the container that SQL Server can access.
403+
# Using SQL Server's default backup directory ensures accessibility.
404+
$server = Connect-DbaInstance -SqlInstance localhost -SqlCredential $cred
405+
$defaultPaths = Get-DbaDefaultPath -SqlInstance $server
406+
$localBackupPath = Join-Path -Path $defaultPaths.Backup -ChildPath "dbatools_s3test"
407+
408+
# Create the subdirectory using SQL Server's xp_create_subdir (runs inside the container)
409+
$server.Query("EXEC master.dbo.xp_create_subdir '$localBackupPath'")
410+
411+
# Verify the backup path is accessible via SQL Server
412+
Test-DbaPath -SqlInstance $server -Path $localBackupPath | Should -BeTrue
413+
414+
$localBackupFile = Join-Path -Path $localBackupPath -ChildPath "local_test.bak"
415+
$splatLocalBackup = @{
416+
SqlInstance = "localhost"
417+
SqlCredential = $cred
418+
Database = $script:TestDbName5
419+
FilePath = $localBackupFile
420+
Type = "Full"
421+
}
422+
$localBackupResult = Backup-DbaDatabase @splatLocalBackup
423+
$localBackupResult.BackupComplete | Should -BeTrue
424+
425+
# Verify the backup file exists after the backup
426+
Test-DbaPath -SqlInstance $server -Path $localBackupFile | Should -BeTrue
427+
428+
# Restore using folder path - this tests that local folder enumeration works
429+
# (in contrast to S3 where folder enumeration is not supported)
430+
# Drop the source database so we can restore with the original name
431+
$null = Remove-DbaDatabase -SqlInstance localhost -SqlCredential $cred -Database $script:TestDbName5 -Confirm:$false -ErrorAction SilentlyContinue
432+
433+
$splatRestore = @{
434+
SqlInstance = "localhost"
435+
SqlCredential = $cred
436+
Path = $localBackupPath
437+
}
438+
$result = Restore-DbaDatabase @splatRestore
439+
440+
# Should successfully restore - proving local folder enumeration works
441+
# Database name comes from the backup file itself
442+
$result | Should -Not -BeNullOrEmpty
443+
$result.RestoreComplete | Should -BeTrue
444+
$result.Database | Should -Be $script:TestDbName5
445+
446+
# Note: Backup files inside the container are cleaned up when container stops
447+
}
448+
449+
It "Should require PowerShell-based enumeration for S3 (validation test)" {
450+
# This test demonstrates the correct approach: using PowerShell to enumerate S3
451+
# Since we're using MinIO in tests, we need to use AWS PowerShell module
452+
# This is a validation that the workaround approach is correct
453+
454+
# Skip if AWS.Tools.S3 not available (CI environments may not have it)
455+
$hasAwsModule = $null -ne (Get-Module -ListAvailable -Name AWS.Tools.S3)
456+
if (-not $hasAwsModule) {
457+
Set-ItResult -Skipped -Because "AWS.Tools.S3 module not available"
458+
return
459+
}
460+
461+
# This demonstrates the recommended approach from the documentation
462+
# Users should use Get-S3Object to list files, then pass paths to Get-DbaBackupInformation/Restore-DbaDatabase
463+
Import-Module AWS.Tools.S3 -ErrorAction Stop
464+
465+
# For MinIO (S3-compatible), use EndpointUrl with ForcePathStyleAddressing
466+
# Note: $script:S3Endpoint is "minio:9000" which is a Docker network hostname
467+
# The AWS SDK runs on the host, so we need to use localhost:9000 instead
468+
$hostEndpoint = $script:S3Endpoint -replace "^minio:", "localhost:"
469+
$splatListObjects = @{
470+
BucketName = $script:S3Bucket
471+
Prefix = "$($script:S3EnumFolder)/"
472+
EndpointUrl = "https://$hostEndpoint"
473+
AccessKey = $script:S3AccessKey
474+
SecretKey = $script:S3SecretKey
475+
ForcePathStyleAddressing = $true
476+
}
477+
478+
# Try to enumerate S3 objects - may fail with SSL errors if certificate not trusted
479+
try {
480+
$s3Objects = Get-S3Object @splatListObjects
481+
} catch {
482+
if ($_.Exception.Message -like "*SSL*" -or $_.Exception.Message -like "*certificate*") {
483+
Set-ItResult -Skipped -Because "SSL certificate validation failed for self-signed MinIO certificate. In production, use a trusted certificate or configure certificate trust."
484+
return
485+
}
486+
throw
487+
}
488+
489+
# PowerShell CAN enumerate S3 - this is the correct approach
490+
$s3Objects | Should -Not -BeNullOrEmpty
491+
$s3Objects.Key | Should -Contain "$($script:S3EnumFolder)/$($script:TestDbName5)_full1.bak"
492+
$s3Objects.Key | Should -Contain "$($script:S3EnumFolder)/$($script:TestDbName5)_log1.trn"
493+
494+
# Now demonstrate using those paths with Get-DbaBackupInformation
495+
$backupPaths = $s3Objects | ForEach-Object {
496+
"$($script:S3BaseUrl)/$($_.Key)"
497+
}
498+
499+
$splatBackupInfo = @{
500+
SqlInstance = "localhost"
501+
SqlCredential = $cred
502+
Path = $backupPaths
503+
StorageCredential = $script:S3CredentialName
504+
}
505+
$backupInfo = Get-DbaBackupInformation @splatBackupInfo
506+
507+
# Should successfully read backup information for all files
508+
$backupInfo | Should -Not -BeNullOrEmpty
509+
$backupInfo.Count | Should -Be 2
510+
$backupInfo.Database | Should -Contain $script:TestDbName5
511+
}
512+
}
513+
303514
#Context "Test-DbaBackupInformation with S3" {
304515
# BeforeAll {
305516
# $script:TestDbName4 = "dbatoolsci_s3test"
306-
#
517+
#
307518
# # Create and backup test database
308519
# $null = New-DbaDatabase -SqlInstance localhost -SqlCredential $cred -Name $script:TestDbName4 -RecoveryModel Full
309-
#
520+
#
310521
# $script:S3TestBackupFile = "$($script:TestDbName4)_test.bak"
311522
# $splatBackup = @{
312523
# SqlInstance = "localhost"
@@ -319,30 +530,30 @@ Describe "S3 Backup Integration Tests" -Tag "IntegrationTests", "S3" {
319530
# }
320531
# $null = Backup-DbaDatabase @splatBackup
321532
# }
322-
#
533+
#
323534
# AfterAll {
324535
# $null = Remove-DbaDatabase -SqlInstance localhost -SqlCredential $cred -Database $script:TestDbName4 -Confirm:$false
325536
# }
326-
#
537+
#
327538
# It "Should validate S3 backup information" {
328539
# $s3Path = "$($script:S3BaseUrl)/$($script:S3TestBackupFile)"
329-
#
540+
#
330541
# $splatInfo = @{
331542
# SqlInstance = "localhost"
332543
# SqlCredential = $cred
333544
# Path = $s3Path
334545
# StorageCredential = $script:S3CredentialName
335546
# }
336547
# $backupInfo = Get-DbaBackupInformation @splatInfo
337-
#
548+
#
338549
# $splatTest = @{
339550
# BackupHistory = $backupInfo
340551
# SqlInstance = "localhost"
341552
# SqlCredential = $cred
342553
# VerifyOnly = $true
343554
# }
344555
# $result = Test-DbaBackupInformation @splatTest
345-
#
556+
#
346557
# $result | Should -Not -BeNullOrEmpty
347558
# # S3 URLs should pass validation - they are skipped for cloud paths
348559
# }
@@ -362,4 +573,4 @@ Describe "S3 Backup Integration Tests" -Tag "IntegrationTests", "S3" {
362573
$deletedCred | Should -BeNullOrEmpty
363574
}
364575
}
365-
}
576+
}

‎.github/workflows/integration-tests-s3.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,26 @@ jobs:
198198
throw "Failed to connect to SQL Server after $maxAttempts attempts"
199199
}
200200
201+
- name: Add MinIO certificate to system trust store
202+
shell: bash
203+
run: |
204+
# Add MinIO's self-signed certificate to the system CA trust store
205+
# This allows the AWS PowerShell SDK to connect to MinIO over HTTPS
206+
sudo cp $HOME/.minio/certs/public.crt /usr/local/share/ca-certificates/minio.crt
207+
sudo update-ca-certificates
208+
echo "MinIO certificate added to system trust store"
209+
210+
- name: Install AWS.Tools.S3 for S3 enumeration tests
211+
run: |
212+
Write-Host "Installing AWS.Tools.Installer..."
213+
Install-Module -Name AWS.Tools.Installer -Force -Scope CurrentUser
214+
215+
Write-Host "Installing AWS.Tools.S3 (includes AWS.Tools.Common dependency)..."
216+
Install-AWSToolsModule AWS.Tools.S3 -Force -Scope CurrentUser
217+
218+
Write-Host "Verifying installation..."
219+
Get-Module -ListAvailable -Name AWS.Tools.S3, AWS.Tools.Common | Select-Object Name, Version
220+
201221
- name: Run S3 backup tests
202222
env:
203223
S3_ENDPOINT: minio:9000

‎private/functions/Get-XpDirTreeRestoreFile.ps1‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,12 @@ function Get-XpDirTreeRestoreFile {
4343
# Determine the correct path separator based on whether this is a URL or file system path
4444
if ($Path -match '^https?://') {
4545
$pathSep = "/"
46+
} elseif ($Path -match '^s3://') {
47+
# S3 paths cannot be enumerated via T-SQL (xp_dirtree/dm_os_enumerate_filesystem don't support S3)
48+
# SQL Server 2022+ supports S3 for BACKUP/RESTORE but has no built-in function to list S3 objects
49+
# Return empty array - caller should handle S3 enumeration in PowerShell or use explicit file paths
50+
Write-Message -Level Warning -Message "S3 paths cannot be enumerated using T-SQL. Use explicit file paths or PowerShell-based enumeration for S3 storage."
51+
Stop-Function -Message "S3 path enumeration not supported. Path: $Path"
4652
} else {
4753
$pathSep = Get-DbaPathSep -Server $server
4854
}

‎public/Get-DbaBackupInformation.ps1‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -262,8 +262,9 @@ function Get-DbaBackupInformation {
262262
} else {
263263
$Files = @()
264264
$groupResults = @()
265+
265266
# Detect cloud storage URLs (Azure http:// or S3 s3://)
266-
if ($Path[0] -match 'http' -or $Path[0] -match 's3') { $NoXpDirTree = $true }
267+
if ($Path[0] -match '^https?://' -or $Path[0] -match '^s3://') { $NoXpDirTree = $true }
267268
if ($NoXpDirTree -ne $true) {
268269
foreach ($f in $path) {
269270
if ([System.IO.Path]::GetExtension($f).Length -gt 1) {

‎public/Restore-DbaDatabase.ps1‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,12 @@ function Restore-DbaDatabase {
2727
For MFA support, please use Connect-DbaInstance.
2828
2929
.PARAMETER Path
30-
Specifies the location of backup files to restore from, supporting local drives, UNC paths, or Azure blob storage URLs.
30+
Specifies the location of backup files to restore from, supporting local drives, UNC paths, Azure blob storage URLs, or S3 URLs.
3131
Use this when you need to restore from a specific backup location or when piping backup files from Get-ChildItem.
3232
Accepts multiple comma-separated paths for complex restore scenarios spanning multiple locations.
3333
34+
For S3 storage (SQL Server 2022+): S3 bucket contents cannot be enumerated using T-SQL. You must provide explicit file paths or use PowerShell's Get-S3Object cmdlet (from AWS.Tools.S3 module) to retrieve the list of backup files first, then pass them to this command. See examples for S3 usage patterns.
35+
3436
.PARAMETER DatabaseName
3537
Defines the target database name for the restored database when different from the original name.
3638
Use this when creating a copy of a production database for testing or when restoring to avoid name conflicts.
@@ -164,7 +166,7 @@ function Restore-DbaDatabase {
164166
Use this for log shipping secondary servers or when you need read-only access during restore operations.
165167
The directory must exist and be writable by the SQL Server service account for undo file creation.
166168
167-
.PARAMETER StorageCredential
169+
.PARAMETER StorageCredential
168170
Specifies the SQL Server credential name for authenticating to Azure blob storage or S3-compatible object storage during restore operations.
169171
Use this when restoring from Azure blob storage or S3 backups that require authentication.
170172
For Azure: The credential must contain valid Azure storage account keys or SAS tokens.
@@ -329,7 +331,7 @@ function Restore-DbaDatabase {
329331
c:\DataFiles and all the log files into c:\LogFiles
330332
331333
.EXAMPLE
332-
PS C:\> Restore-DbaDatabase -SqlInstance server1\instance1 -Path http://demo.blob.core.windows.net/backups/dbbackup.bak -StorageCredential MyAzureCredential
334+
PS C:\> Restore-DbaDatabase -SqlInstance server1\instance1 -Path http://demo.blob.core.windows.net/backups/dbbackup.bak -AzureCredential MyAzureCredential
333335
334336
Will restore the backup held at http://demo.blob.core.windows.net/backups/dbbackup.bak to server1\instance1. The connection to Azure will be made using the
335337
credential MyAzureCredential held on instance Server1\instance1
@@ -344,7 +346,7 @@ function Restore-DbaDatabase {
344346
345347
Will restore the backup from S3-compatible storage to sql2022. Requires SQL Server 2022 or higher. The credential must be configured with Identity = 'S3 Access Key' and Secret containing the access key and secret key.
346348
347-
.EXAMPLE
349+
.EXAMPLE
348350
PS C:\> $File = Get-ChildItem c:\backups, \\server1\backups
349351
PS C:\> $File | Restore-DbaDatabase -SqlInstance Server1\Instance -UseDestinationDefaultDirectories
350352
@@ -429,6 +431,18 @@ function Restore-DbaDatabase {
429431
Restores the backups from \\ServerName\ShareName\File as database, stops before the first 'OvernightStart' mark that occurs after '21:00 10/05/2020'.
430432
431433
Note that Date time needs to be specified in your local SQL Server culture
434+
435+
.EXAMPLE
436+
PS C:\> # Restore from S3 storage folder (SQL Server 2022+)
437+
PS C:\> # First, enumerate S3 bucket contents using AWS PowerShell module - You need to be authenticated!
438+
PS C:\> $s3Files = Get-S3Object -BucketName "mybucket" -KeyPrefix "backups/AdventureWorks/" -Region "us-west-2"
439+
PS C:\> $backupPaths = $s3Files | Where-Object { $_.Key -match '\.(bak|trn|dif)$' } | ForEach-Object { "s3://mybucket.s3.us-west-2.amazonaws.com/$($_.Key)" }
440+
PS C:\>
441+
PS C:\> # Then restore using the enumerated file paths
442+
PS C:\> Restore-DbaDatabase -SqlInstance sql2022 -Path $backupPaths -StorageCredential MyS3Credential
443+
444+
Demonstrates the recommended workflow for restoring from S3 storage. Since SQL Server cannot enumerate S3 bucket contents via T-SQL, you must first use PowerShell's Get-S3Object cmdlet to list backup files, then pass the full S3 URLs to Restore-DbaDatabase.
445+
The StorageCredential parameter should reference a SQL Server credential configured for S3 access.
432446
#>
433447
[CmdletBinding(SupportsShouldProcess, DefaultParameterSetName = "Restore")]
434448
[Diagnostics.CodeAnalysis.SuppressMessageAttribute("PSAvoidUsingPlainTextForPassword", "StorageCredential", Justification = "For Parameter StorageCredential")]

0 commit comments

Comments
 (0)