Skip to content

Commit 5b1781c

Browse files
author
Thomas Mahlberg
committed
Added hot cache changes for follower
1 parent 455541e commit 5b1781c

5 files changed

Lines changed: 227 additions & 2 deletions

File tree

‎KustoSchemaTools/Changes/DatabaseChanges.cs‎

Lines changed: 130 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1-
using KustoSchemaTools.Model;
1+
using Kusto.Language;
2+
using KustoSchemaTools.Model;
23
using Microsoft.Extensions.Logging;
34
using System.Data;
5+
using System.Text;
46

57
namespace KustoSchemaTools.Changes
68
{
@@ -175,6 +177,133 @@ private static List<IChange> GenerateScriptCompareChanges<T>(Database oldState,
175177

176178
return tmp;
177179
}
180+
181+
public static List<IChange> GenerateFollowerChanges(FollowerDatabase oldState, FollowerDatabase newState, ILogger log)
182+
{
183+
184+
185+
List<IChange> result =
186+
[
187+
.. GenerateFollowerCachingChanges(oldState, newState, db => db.Tables, "Table", "table"),
188+
.. GenerateFollowerCachingChanges(oldState, newState, db => db.MaterializedViews, "MV", "materialized-view"),
189+
190+
];
191+
if (oldState.Cache.DefaultHotCache != newState.Cache.DefaultHotCache)
192+
{
193+
if (newState.Cache.DefaultHotCache != null)
194+
{
195+
result.Add(new BasicChange("FollowerDatabase", "ChangeDefaultHotCache", $"From {oldState.Cache.DefaultHotCache} to {newState.Cache.DefaultHotCache}", new List<DatabaseScriptContainer>
196+
{
197+
new DatabaseScriptContainer(new DatabaseScript($".alter follower database {newState.DatabaseName} policy caching hot = {newState.Cache.DefaultHotCache}", 0), "FollowerChangeDefaultHotCache")
198+
}));
199+
}
200+
else
201+
{
202+
result.Add(new BasicChange("FollowerDatabase", "DeleteDefaultHotCache", $"Remove Default Hot Cache", new List<DatabaseScriptContainer>
203+
{
204+
new DatabaseScriptContainer(new DatabaseScript($".delete follower database {newState.DatabaseName} policy caching", 0), "FollowerDeleteDefaultHotCache")
205+
}));
206+
}
207+
}
208+
209+
return result;
210+
211+
212+
213+
/*
214+
* public class FollowerDatabase
215+
{
216+
public required string DatabaseName { get; set; }
217+
public FollowerCache Cache { get; set; } = new FollowerCache();
218+
public FollowerPermissions Permissions { get; set; } = new FollowerPermissions();
219+
}
220+
221+
public class FollowerPermissions
222+
{
223+
public FollowerModificationKind? ModificationKind { get; set; }
224+
public List<AADObject> Viewers { get; set; } = new List<AADObject>();
225+
public List<AADObject> Admins { get; set; } = new List<AADObject>();
226+
}
227+
228+
public class FollowerCache
229+
{
230+
public string? DefaultHotCache { get; set; }
231+
public FollowerModificationKind? ModificationKind { get; set; }
232+
public Dictionary<string, string> Tables { get; set; } = new Dictionary<string, string>();
233+
public Dictionary<string, string> MaterializedViews { get; set; } = new Dictionary<string, string>();
234+
}
235+
236+
public enum FollowerModificationKind
237+
{
238+
None,
239+
Union,
240+
Replace
241+
}
242+
*/
243+
244+
245+
return result;
246+
}
247+
248+
private static List<IChange> GenerateFollowerCachingChanges(FollowerDatabase oldState, FollowerDatabase newState, Func<FollowerCache, Dictionary<string,string>> selector, string type, string kustoType)
249+
{
250+
var result = new List<IChange>();
251+
var oldEntities = selector(oldState.Cache);
252+
var newEntities = selector(newState.Cache);
253+
254+
255+
var removedPolicyScripts = oldEntities.Keys.Except(newEntities.Keys)
256+
.Select(itm =>
257+
new
258+
{
259+
Name = itm,
260+
Script = new DatabaseScriptContainer(new DatabaseScript($".delete follower database {newState.DatabaseName} {kustoType} {itm} policy caching", 0), $"FollowerDelete{type}CachingPolicies")
261+
})
262+
.ToList();
263+
var changedPolicyScripts = newEntities
264+
.Where(itm => oldEntities.ContainsKey(itm.Key) == false
265+
|| oldEntities[itm.Key] != itm.Value)
266+
.Select(itm => new
267+
{
268+
Name = itm.Key,
269+
From = oldEntities.ContainsKey(itm.Key) ? oldEntities[itm.Key] : "default",
270+
To = itm.Value,
271+
Script = new DatabaseScriptContainer(new DatabaseScript($".alter follower database {newState.DatabaseName} {kustoType} {itm.Key} policy caching hot = {itm.Value}", 0), $"FollowerChange{type}CachingPolicies")
272+
})
273+
.ToList();
274+
275+
if (removedPolicyScripts.Any())
276+
{
277+
result.Add(new Heading($"Deleted {type} Caching Policies"));
278+
279+
var deletedPolicies = string.Join("\n", removedPolicyScripts.Select(itm => $"* {itm.Name}"));
280+
281+
result.Add(new BasicChange(
282+
"FollowerDatabase",
283+
$"Delete{type}CachingPolicy",
284+
deletedPolicies,
285+
removedPolicyScripts.Select(itm => itm.Script).ToList()));
286+
}
287+
if (changedPolicyScripts.Any())
288+
{
289+
result.Add(new Heading($"Changed {type} Caching Policies"));
290+
291+
var changePolicies = new StringBuilder();
292+
changePolicies.AppendLine($"{type} | From | To");
293+
foreach (var change in changedPolicyScripts)
294+
{
295+
changePolicies.AppendLine($"{change.Name} | {change.From} | {change.To}");
296+
}
297+
298+
result.Add(new BasicChange(
299+
"FollowerDatabase",
300+
$"Change{type}CachingPolicy",
301+
changePolicies.ToString(),
302+
changedPolicyScripts.Select(itm => itm.Script).ToList()));
303+
}
304+
305+
return result;
306+
}
178307
}
179308

180309
}

‎KustoSchemaTools/Changes/FollowerChange.cs‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Text;
5+
using System.Threading.Tasks;
6+
7+
namespace KustoSchemaTools.Changes
8+
{
9+
public class BasicChange : IChange
10+
{
11+
12+
public string EntityType { get; set; }
13+
14+
public BasicChange(string entityType, string entity, string markdown, List<DatabaseScriptContainer> scripts)
15+
{
16+
EntityType = entityType;
17+
Entity = entity;
18+
Markdown = markdown;
19+
Scripts = scripts;
20+
}
21+
22+
public string Entity { get; set; }
23+
24+
public string Markdown { get; set; }
25+
26+
public List<DatabaseScriptContainer> Scripts { get; set; }
27+
28+
}
29+
}

‎KustoSchemaTools/KustoSchemaHandler.cs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@
55
using Microsoft.Extensions.Logging;
66
using System.Collections.Concurrent;
77
using System.Data;
8+
using System.Diagnostics.Metrics;
89
using System.Text;
10+
using System.Threading.Channels;
911

1012
namespace KustoSchemaTools
1113
{
@@ -65,6 +67,26 @@ public KustoSchemaHandler(ILogger<KustoSchemaHandler<T>> schemaHandlerLogger, Ya
6567

6668
Log.LogInformation($"Following scripts will be applied:\n{scriptSb}");
6769
}
70+
71+
foreach(var follower in yamlDb.Followers)
72+
{
73+
74+
Log.LogInformation($"Generating diff markdown for {Path.Combine(path, databaseName)} => {follower.Cluster}/{databaseName}");
75+
76+
77+
78+
79+
80+
//var scriptSb = new StringBuilder();
81+
//foreach (var script in changes.SelectMany(itm => itm.Scripts).Where(itm => itm.IsValid == true).OrderBy(itm => itm.Order))
82+
//{
83+
// scriptSb.AppendLine(script.Text);
84+
//}
85+
86+
//Log.LogInformation($"Following scripts will be applied:\n{scriptSb}");
87+
88+
}
89+
6890
return (sb.ToString(), isValid);
6991
}
7092

‎KustoSchemaTools/Model/Database.cs‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
namespace KustoSchemaTools.Model
1+
using KustoSchemaTools.Changes;
2+
using System.Diagnostics.CodeAnalysis;
3+
4+
namespace KustoSchemaTools.Model
25
{
36
public class Database
47
{
@@ -31,6 +34,37 @@ public class Database
3134

3235
public Deletions Deletions { get; set; } = new Deletions();
3336

37+
public Dictionary<string, FollowerDatabase> Followers { get; set; } = new Dictionary<string, FollowerDatabase>();
38+
39+
}
40+
41+
public class FollowerDatabase
42+
{
43+
public required string DatabaseName { get; set; }
44+
public FollowerCache Cache { get; set; } = new FollowerCache();
45+
public FollowerPermissions Permissions { get; set; } = new FollowerPermissions();
46+
}
47+
48+
public class FollowerPermissions
49+
{
50+
public FollowerModificationKind? ModificationKind { get; set; }
51+
public List<AADObject> Viewers { get; set; } = new List<AADObject>();
52+
public List<AADObject> Admins { get; set; } = new List<AADObject>();
53+
}
54+
55+
public class FollowerCache
56+
{
57+
public string? DefaultHotCache { get; set; }
58+
public FollowerModificationKind? ModificationKind { get; set; }
59+
public Dictionary<string, string> Tables { get; set; } = new Dictionary<string, string>();
60+
public Dictionary<string, string> MaterializedViews { get; set; } = new Dictionary<string, string>();
61+
}
62+
63+
public enum FollowerModificationKind
64+
{
65+
None,
66+
Union,
67+
Replace
3468
}
3569

3670
}

‎KustoSchemaTools/Plugins/FollowerPlugin.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
using KustoSchemaTools.Model;
2+
3+
namespace KustoSchemaTools.Plugins
4+
{
5+
public class FollowerPlugin : EntityPlugin<FollowerDatabase>
6+
{
7+
public FollowerPlugin(string subFolder = "followers", int minRowLength = 5) : base(db => db.Followers, subFolder, minRowLength)
8+
{
9+
}
10+
}
11+
}

0 commit comments

Comments
 (0)