Skip to content

Commit 6da9604

Browse files
ClaudioESSilvaClaudioESSilva
authored
Support native backup/restore from/to s3 (#10126)
1 parent 6449a95 commit 6da9604

16 files changed

Lines changed: 873 additions & 137 deletions

‎.github/scripts/gh-s3actions.ps1‎

Lines changed: 365 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,365 @@
1+
Describe "S3 Backup Integration Tests" -Tag "IntegrationTests", "S3" {
2+
BeforeAll {
3+
$password = ConvertTo-SecureString "dbatools.IO" -AsPlainText -Force
4+
$cred = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList "sa", $password
5+
6+
$PSDefaultParameterValues["*:SqlInstance"] = "localhost"
7+
$PSDefaultParameterValues["*:SqlCredential"] = $cred
8+
$PSDefaultParameterValues["*:Confirm"] = $false
9+
$global:ProgressPreference = "SilentlyContinue"
10+
11+
# S3 configuration from environment variables
12+
$script:S3Endpoint = $env:S3_ENDPOINT
13+
$script:S3Bucket = $env:S3_BUCKET
14+
$script:S3AccessKey = $env:S3_ACCESS_KEY
15+
$script:S3SecretKey = $env:S3_SECRET_KEY
16+
17+
# S3 URL format for SQL Server: s3://endpoint/bucket/path
18+
# MinIO uses path-style URLs
19+
$script:S3BaseUrl = "s3://$($script:S3Endpoint)/$($script:S3Bucket)"
20+
21+
# Credential name for SQL Server
22+
$script:S3CredentialName = "S3BackupCredential"
23+
24+
# Load dbatools
25+
if (-not (Get-Module dbatools)) {
26+
Import-Module dbatools.library
27+
try {
28+
Import-Module dbatools -ErrorAction Stop
29+
} catch {
30+
Write-Warning "Importing dbatools from source"
31+
Import-Module ./dbatools.psd1 -Force
32+
}
33+
}
34+
}
35+
36+
Context "SQL Server 2022 S3 backup prerequisites" {
37+
It "Should be connected to SQL Server 2022 or later" {
38+
$server = Connect-DbaInstance -SqlInstance localhost -SqlCredential $cred
39+
$server.VersionMajor | Should -BeGreaterOrEqual 16
40+
Write-Host "Connected to SQL Server version: $($server.Version)"
41+
}
42+
43+
It "Should create an S3 credential on the SQL Server" {
44+
# S3 credential format: IDENTITY = 'S3 Access Key', SECRET = 'AccessKeyID:SecretKeyID'
45+
$secretString = "$($script:S3AccessKey):$($script:S3SecretKey)"
46+
$securePassword = ConvertTo-SecureString -String $secretString -AsPlainText -Force
47+
48+
$splatCredential = @{
49+
SqlInstance = "localhost"
50+
SqlCredential = $cred
51+
Name = $script:S3CredentialName
52+
Identity = "S3 Access Key"
53+
SecurePassword = $securePassword
54+
Force = $true
55+
}
56+
$newCred = New-DbaCredential @splatCredential
57+
58+
# Verify credential was created
59+
$newCred | Should -Not -BeNullOrEmpty
60+
$newCred.Name | Should -Be $script:S3CredentialName
61+
$newCred.Identity | Should -Be "S3 Access Key"
62+
}
63+
}
64+
65+
Context "Backup-DbaDatabase to S3" {
66+
BeforeAll {
67+
$script:TestDbName = "dbatoolsci_s3backup"
68+
69+
# Create test database
70+
$null = New-DbaDatabase -SqlInstance localhost -SqlCredential $cred -Name $script:TestDbName -RecoveryModel Full
71+
72+
# Insert some test data
73+
$server = Connect-DbaInstance -SqlInstance localhost -SqlCredential $cred
74+
$server.Query("CREATE TABLE TestTable (ID INT, Name VARCHAR(100))", $script:TestDbName)
75+
$server.Query("INSERT INTO TestTable VALUES (1, 'Test Row 1'), (2, 'Test Row 2')", $script:TestDbName)
76+
}
77+
78+
AfterAll {
79+
# Cleanup test database
80+
$null = Remove-DbaDatabase -SqlInstance localhost -SqlCredential $cred -Database $script:TestDbName -Confirm:$false
81+
}
82+
83+
It "Should backup database to S3 using StorageBaseUrl parameter" {
84+
$backupFile = "$($script:TestDbName)_full.bak"
85+
86+
$splatBackup = @{
87+
SqlInstance = "localhost"
88+
SqlCredential = $cred
89+
Database = $script:TestDbName
90+
StorageBaseUrl = $script:S3BaseUrl
91+
StorageCredential = $script:S3CredentialName
92+
FilePath = $backupFile
93+
Type = "Full"
94+
}
95+
$result = Backup-DbaDatabase @splatBackup
96+
97+
$result | Should -Not -BeNullOrEmpty
98+
$result.BackupComplete | Should -BeTrue
99+
$result.DeviceType | Should -Be "URL"
100+
$result.BackupPath | Should -BeLike "s3://*"
101+
}
102+
103+
It "Should backup database to S3 using AzureBaseUrl alias for backward compatibility" {
104+
$backupFile = "$($script:TestDbName)_full_alias.bak"
105+
106+
$splatBackup = @{
107+
SqlInstance = "localhost"
108+
SqlCredential = $cred
109+
Database = $script:TestDbName
110+
AzureBaseUrl = $script:S3BaseUrl
111+
AzureCredential = $script:S3CredentialName
112+
FilePath = $backupFile
113+
Type = "Full"
114+
}
115+
$result = Backup-DbaDatabase @splatBackup
116+
117+
$result | Should -Not -BeNullOrEmpty
118+
$result.BackupComplete | Should -BeTrue
119+
$result.DeviceType | Should -Be "URL"
120+
}
121+
122+
It "Should backup a transaction log to S3" {
123+
$backupFile = "$($script:TestDbName)_log.trn"
124+
125+
$splatBackup = @{
126+
SqlInstance = "localhost"
127+
SqlCredential = $cred
128+
Database = $script:TestDbName
129+
StorageBaseUrl = $script:S3BaseUrl
130+
StorageCredential = $script:S3CredentialName
131+
FilePath = $backupFile
132+
Type = "Log"
133+
}
134+
$result = Backup-DbaDatabase @splatBackup
135+
136+
$result | Should -Not -BeNullOrEmpty
137+
$result.BackupComplete | Should -BeTrue
138+
$result.Type | Should -Be "Log"
139+
}
140+
141+
It "Should backup a differential to S3" {
142+
$backupFile = "$($script:TestDbName)_diff.bak"
143+
144+
$splatBackup = @{
145+
SqlInstance = "localhost"
146+
SqlCredential = $cred
147+
Database = $script:TestDbName
148+
StorageBaseUrl = $script:S3BaseUrl
149+
StorageCredential = $script:S3CredentialName
150+
FilePath = $backupFile
151+
Type = "Differential"
152+
}
153+
$result = Backup-DbaDatabase @splatBackup
154+
155+
$result | Should -Not -BeNullOrEmpty
156+
$result.BackupComplete | Should -BeTrue
157+
$result.Type | Should -Be "Differential"
158+
}
159+
160+
It "Should backup with custom MaxTransferSize for S3" {
161+
$backupFile = "$($script:TestDbName)_maxtransfer.bak"
162+
163+
$splatBackup = @{
164+
SqlInstance = "localhost"
165+
SqlCredential = $cred
166+
Database = $script:TestDbName
167+
StorageBaseUrl = $script:S3BaseUrl
168+
StorageCredential = $script:S3CredentialName
169+
FilePath = $backupFile
170+
Type = "Full"
171+
MaxTransferSize = 10485760
172+
CompressBackup = $true
173+
}
174+
$result = Backup-DbaDatabase @splatBackup
175+
176+
$result | Should -Not -BeNullOrEmpty
177+
$result.BackupComplete | Should -BeTrue
178+
}
179+
}
180+
181+
Context "Get-DbaBackupInformation from S3" {
182+
BeforeAll {
183+
$script:TestDbName2 = "dbatoolsci_s3info"
184+
185+
# Create and backup test database
186+
$null = New-DbaDatabase -SqlInstance localhost -SqlCredential $cred -Name $script:TestDbName2 -RecoveryModel Full
187+
188+
$script:S3BackupFile = "$($script:TestDbName2)_info.bak"
189+
$splatBackup = @{
190+
SqlInstance = "localhost"
191+
SqlCredential = $cred
192+
Database = $script:TestDbName2
193+
StorageBaseUrl = $script:S3BaseUrl
194+
StorageCredential = $script:S3CredentialName
195+
FilePath = $script:S3BackupFile
196+
Type = "Full"
197+
}
198+
$null = Backup-DbaDatabase @splatBackup
199+
}
200+
201+
AfterAll {
202+
$null = Remove-DbaDatabase -SqlInstance localhost -SqlCredential $cred -Database $script:TestDbName2 -Confirm:$false
203+
}
204+
205+
It "Should read backup information from S3 URL" {
206+
$s3Path = "$($script:S3BaseUrl)/$($script:S3BackupFile)"
207+
208+
$splatInfo = @{
209+
SqlInstance = "localhost"
210+
SqlCredential = $cred
211+
Path = $s3Path
212+
StorageCredential = $script:S3CredentialName
213+
}
214+
$result = Get-DbaBackupInformation @splatInfo
215+
216+
$result | Should -Not -BeNullOrEmpty
217+
$result.Database | Should -Be $script:TestDbName2
218+
$result.Type | Should -Be "Database"
219+
}
220+
}
221+
222+
Context "Restore-DbaDatabase from S3" {
223+
BeforeAll {
224+
$script:TestDbName3 = "dbatoolsci_s3restore"
225+
$script:RestoreDbName = "dbatoolsci_s3restored"
226+
227+
# Create and backup test database
228+
$null = New-DbaDatabase -SqlInstance localhost -SqlCredential $cred -Name $script:TestDbName3 -RecoveryModel Full
229+
230+
# Add test data
231+
$server = Connect-DbaInstance -SqlInstance localhost -SqlCredential $cred
232+
$server.Query("CREATE TABLE RestoreTest (ID INT, Value VARCHAR(50))", $script:TestDbName3)
233+
$server.Query("INSERT INTO RestoreTest VALUES (100, 'S3 Restore Test')", $script:TestDbName3)
234+
235+
# Backup to S3
236+
$script:S3RestoreBackupFile = "$($script:TestDbName3)_restore.bak"
237+
$splatBackup = @{
238+
SqlInstance = "localhost"
239+
SqlCredential = $cred
240+
Database = $script:TestDbName3
241+
StorageBaseUrl = $script:S3BaseUrl
242+
StorageCredential = $script:S3CredentialName
243+
FilePath = $script:S3RestoreBackupFile
244+
Type = "Full"
245+
}
246+
$null = Backup-DbaDatabase @splatBackup
247+
}
248+
249+
AfterAll {
250+
$null = Remove-DbaDatabase -SqlInstance localhost -SqlCredential $cred -Database $script:TestDbName3 -Confirm:$false
251+
$null = Remove-DbaDatabase -SqlInstance localhost -SqlCredential $cred -Database $script:RestoreDbName -Confirm:$false
252+
}
253+
254+
It "Should restore database from S3 backup" {
255+
$s3Path = "$($script:S3BaseUrl)/$($script:S3RestoreBackupFile)"
256+
257+
$splatRestore = @{
258+
SqlInstance = "localhost"
259+
SqlCredential = $cred
260+
Path = $s3Path
261+
DatabaseName = $script:RestoreDbName
262+
StorageCredential = $script:S3CredentialName
263+
ReplaceDbNameInFile = $true
264+
}
265+
$result = Restore-DbaDatabase @splatRestore
266+
267+
$result | Should -Not -BeNullOrEmpty
268+
$result.RestoreComplete | Should -BeTrue
269+
$result.Database | Should -Be $script:RestoreDbName
270+
271+
# Verify data was restored
272+
$server = Connect-DbaInstance -SqlInstance localhost -SqlCredential $cred
273+
$data = $server.Query("SELECT * FROM RestoreTest", $script:RestoreDbName)
274+
$data.ID | Should -Be 100
275+
$data.Value | Should -Be "S3 Restore Test"
276+
}
277+
278+
It "Should restore using AzureCredential alias for backward compatibility" {
279+
$s3Path = "$($script:S3BaseUrl)/$($script:S3RestoreBackupFile)"
280+
$restoreDbAlias = "$($script:RestoreDbName)_alias"
281+
282+
# Remove if exists from previous test
283+
$null = Remove-DbaDatabase -SqlInstance localhost -SqlCredential $cred -Database $restoreDbAlias -Confirm:$false -ErrorAction SilentlyContinue
284+
285+
$splatRestore = @{
286+
SqlInstance = "localhost"
287+
SqlCredential = $cred
288+
Path = $s3Path
289+
DatabaseName = $restoreDbAlias
290+
AzureCredential = $script:S3CredentialName
291+
ReplaceDbNameInFile = $true
292+
}
293+
$result = Restore-DbaDatabase @splatRestore
294+
295+
$result | Should -Not -BeNullOrEmpty
296+
$result.RestoreComplete | Should -BeTrue
297+
298+
# Cleanup
299+
$null = Remove-DbaDatabase -SqlInstance localhost -SqlCredential $cred -Database $restoreDbAlias -Confirm:$false
300+
}
301+
}
302+
303+
#Context "Test-DbaBackupInformation with S3" {
304+
# BeforeAll {
305+
# $script:TestDbName4 = "dbatoolsci_s3test"
306+
#
307+
# # Create and backup test database
308+
# $null = New-DbaDatabase -SqlInstance localhost -SqlCredential $cred -Name $script:TestDbName4 -RecoveryModel Full
309+
#
310+
# $script:S3TestBackupFile = "$($script:TestDbName4)_test.bak"
311+
# $splatBackup = @{
312+
# SqlInstance = "localhost"
313+
# SqlCredential = $cred
314+
# Database = $script:TestDbName4
315+
# StorageBaseUrl = $script:S3BaseUrl
316+
# StorageCredential = $script:S3CredentialName
317+
# FilePath = $script:S3TestBackupFile
318+
# Type = "Full"
319+
# }
320+
# $null = Backup-DbaDatabase @splatBackup
321+
# }
322+
#
323+
# AfterAll {
324+
# $null = Remove-DbaDatabase -SqlInstance localhost -SqlCredential $cred -Database $script:TestDbName4 -Confirm:$false
325+
# }
326+
#
327+
# It "Should validate S3 backup information" {
328+
# $s3Path = "$($script:S3BaseUrl)/$($script:S3TestBackupFile)"
329+
#
330+
# $splatInfo = @{
331+
# SqlInstance = "localhost"
332+
# SqlCredential = $cred
333+
# Path = $s3Path
334+
# StorageCredential = $script:S3CredentialName
335+
# }
336+
# $backupInfo = Get-DbaBackupInformation @splatInfo
337+
#
338+
# $splatTest = @{
339+
# BackupHistory = $backupInfo
340+
# SqlInstance = "localhost"
341+
# SqlCredential = $cred
342+
# VerifyOnly = $true
343+
# }
344+
# $result = Test-DbaBackupInformation @splatTest
345+
#
346+
# $result | Should -Not -BeNullOrEmpty
347+
# # S3 URLs should pass validation - they are skipped for cloud paths
348+
# }
349+
#}
350+
351+
Context "Cleanup" {
352+
It "Should remove the S3 credential" {
353+
$splatRemoveCred = @{
354+
SqlInstance = "localhost"
355+
SqlCredential = $cred
356+
Credential = $script:S3CredentialName
357+
Confirm = $false
358+
}
359+
$null = Remove-DbaCredential @splatRemoveCred
360+
361+
$deletedCred = Get-DbaCredential -SqlInstance localhost -SqlCredential $cred -Name $script:S3CredentialName
362+
$deletedCred | Should -BeNullOrEmpty
363+
}
364+
}
365+
}

0 commit comments

Comments
 (0)