Skip to content

Commit 14448ec

Browse files
author
Thomas Mahlberg
committed
Add deletions
1 parent 41a5b88 commit 14448ec

5 files changed

Lines changed: 156 additions & 164 deletions

File tree

‎KustoSchemaTools/Changes/BaseChange.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
public abstract class BaseChange<T> : IChange
44
{
55

6-
protected BaseChange(string entityType, string entity, T? from, T to)
6+
protected BaseChange(string entityType, string entity, T from, T to)
77
{
88
EntityType = entityType;
99
Entity = entity;
@@ -14,7 +14,7 @@ protected BaseChange(string entityType, string entity, T? from, T to)
1414
public string EntityType { get; set; }
1515
public string Entity { get; set; }
1616

17-
public T? From { get; set; }
17+
public T From { get; set; }
1818
public T To { get; set; }
1919

2020
public List<DatabaseScriptContainer> Scripts { get; set; } = new List<DatabaseScriptContainer>();

‎KustoSchemaTools/Changes/DatabaseChanges.cs‎

Lines changed: 97 additions & 161 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
1-
using Kusto.Language;
2-
using KustoSchemaTools.Model;
1+
using KustoSchemaTools.Model;
32
using Microsoft.Extensions.Logging;
43

54
namespace KustoSchemaTools.Changes
65
{
76
public class DatabaseChanges
87
{
9-
public static List<IChange> GenerateChanges(Database oldState, Database newState, string name, Microsoft.Extensions.Logging.ILogger log)
8+
public static List<IChange> GenerateChanges(Database oldState, Database newState, string name, ILogger log)
109
{
1110

1211
var result = new List<IChange>();
@@ -35,7 +34,67 @@ public static List<IChange> GenerateChanges(Database oldState, Database newState
3534
result.Add(c);
3635
}
3736
}
37+
if (oldState == null)
38+
{
39+
oldState = new Database();
40+
}
41+
42+
result.AddRange(GeneratePermissionChanges(oldState, newState, name, log));
43+
44+
result.AddRange(GenerateDeletions(oldState, newState.Deletions, log));
45+
46+
result.AddRange(GenerateScriptCompareChanges(oldState, newState, db => db.Tables, nameof(newState.Tables), log));
47+
result.AddRange(GenerateScriptCompareChanges(oldState, newState, db => db.MaterializedViews, nameof(newState.MaterializedViews), log));
48+
result.AddRange(GenerateScriptCompareChanges(oldState, newState, db => db.ContinuousExports, nameof(newState.MaterializedViews), log));
49+
result.AddRange(GenerateScriptCompareChanges(oldState, newState, db => db.Functions, nameof(newState.MaterializedViews), log));
50+
result.AddRange(GenerateScriptCompareChanges(oldState, newState, db => db.ExternalTables, nameof(newState.ExternalTables), log));
51+
52+
if (newState.EntityGroups.Any())
53+
{
54+
List<IChange> changes = GenerateEntityGroupChanges(oldState, newState, name);
55+
if (changes.Any())
56+
{
57+
log.LogInformation($"Detected changes for Entity Groups: {changes.Count}");
58+
result.Add(new Heading("Entity Groups"));
59+
result.AddRange(changes);
60+
}
61+
}
62+
3863

64+
return result;
65+
}
66+
67+
private static IEnumerable<IChange> GenerateDeletions(Database oldState, Deletions deletions, ILogger log)
68+
{
69+
var scripts = new List<IChange>();
70+
scripts.AddRange(deletions.Tables.Where(oldState.Tables.ContainsKey).Select(itm => GenerateDeletionChange(itm, "table")));
71+
var colDel = deletions.Columns
72+
.Select(itm => itm.Split('.'))
73+
.Select(itm => new { Table = itm[0], Column = itm[1] })
74+
.Where(itm => oldState.Tables.ContainsKey(itm.Table) && oldState.Tables[itm.Table].Columns.ContainsKey(itm.Column))
75+
.Select(itm => GenerateDeletionChange($"{itm.Table}.{itm.Column}", "column"))
76+
.ToList();
77+
scripts.AddRange(colDel);
78+
scripts.AddRange(deletions.Functions.Where(oldState.Functions.ContainsKey).Select(itm => GenerateDeletionChange(itm, "function")));
79+
scripts.AddRange(deletions.ExternalTables.Where(oldState.ExternalTables.ContainsKey).Select(itm => GenerateDeletionChange(itm, "external table")));
80+
scripts.AddRange(deletions.MaterializedViews.Where(oldState.MaterializedViews.ContainsKey).Select(itm => GenerateDeletionChange(itm, "materialized-view")));
81+
scripts.AddRange(deletions.ContinuousExports.Where(oldState.ContinuousExports.ContainsKey).Select(itm => GenerateDeletionChange(itm, "continuous-export")));
82+
83+
if (scripts.Any())
84+
{
85+
scripts.Insert(0, new Heading("Deletions"));
86+
}
87+
return scripts;
88+
}
89+
90+
public static IChange GenerateDeletionChange(string entityName, string entityType)
91+
{
92+
return new DeletionChange(entityName, entityType);
93+
}
94+
95+
private static List<IChange> GeneratePermissionChanges(Database oldState, Database newState, string name, ILogger log)
96+
{
97+
var result = new List<IChange>();
3998
var permissionChanges = new List<IChange>
4099
{
41100
new PermissionChange(name, "Admins", oldState.Admins, newState.Admins),
@@ -46,189 +105,66 @@ public static List<IChange> GenerateChanges(Database oldState, Database newState
46105
new PermissionChange(name, "Ingestors", oldState.Ingestors, newState.Ingestors),
47106
}.Where(itm => itm.Scripts.Any()).ToList();
48107

49-
if (permissionChanges.Any() )
108+
if (permissionChanges.Any())
50109
{
51110
log.LogInformation($"Detected {permissionChanges.Count} permission changes");
52111
result.Add(new Heading("Permissions"));
53-
result.AddRange(permissionChanges);
112+
54113
}
55114

56-
if (newState.Tables.Any())
57-
{
58-
var tmp = new List<IChange>();
59-
var existingTables = oldState?.Tables ?? new Dictionary<string, Table>();
60-
log.LogInformation($"Existing tables: {string.Join(", ", existingTables.Keys)}");
61-
62-
foreach (var table in newState.Tables)
63-
{
64-
if (existingTables.ContainsKey(table.Key))
65-
{
66-
var change = new ScriptCompareChange(table.Key, existingTables[table.Key], table.Value);
67-
log.LogInformation($"Table {table.Key} exists, created {change.Scripts.Count} script to apply the diffs");
68-
tmp.Add(change);
69-
}
70-
else if (table.Value.Columns?.Count > 0)
71-
{
72-
var change = new ScriptCompareChange(table.Key, null, table.Value);
73-
log.LogInformation($"Table {table.Key} doesn't exist, created {change.Scripts.Count} scripts to create the table");
74-
tmp.Add(change);
75-
}
76-
}
77-
var changes = tmp.Where(itm => itm.Scripts.Any()).ToList();
78-
if (changes.Any())
79-
{
80-
log.LogInformation($"Detected changes for Tables: {changes.Count} changes with {changes.SelectMany(itm => itm.Scripts).Count()} scripts");
81-
result.Add(new Heading("Tables"));
82-
result.AddRange(changes);
83-
}
84-
}
115+
return result;
116+
}
85117

86-
if (newState.MaterializedViews.Any())
118+
private static List<IChange> GenerateEntityGroupChanges(Database oldState, Database newState, string name)
119+
{
120+
var changes = new List<IChange>();
121+
var existingEntityGroups = oldState?.EntityGroups ?? new Dictionary<string, List<Entity>>();
122+
foreach (var group in newState.EntityGroups)
87123
{
88-
var tmp = new List<IChange>();
89-
var existingMaterializedViews = oldState?.MaterializedViews ?? new Dictionary<string, MaterializedView>();
90-
log.LogInformation($"Existing materialized views: {string.Join(", ", existingMaterializedViews.Keys)}");
91-
92-
foreach (var view in newState.MaterializedViews)
93-
{
94-
if (existingMaterializedViews.ContainsKey(view.Key))
95-
{
96-
var change = new ScriptCompareChange(view.Key, existingMaterializedViews[view.Key], view.Value);
97-
log.LogInformation($"Materialized view {view.Key} exists, created {change.Scripts.Count} script to apply the diffs");
98-
tmp.Add(change);
99-
}
100-
else
101-
{
102-
var change = new ScriptCompareChange(view.Key, null, view.Value);
103-
log.LogInformation($"Materialized view {view.Key} doesn't exist, created {change.Scripts.Count} scripts to create the view");
104-
tmp.Add(change);
105-
}
106-
}
107-
var changes = tmp.Where(itm => itm.Scripts.Any()).ToList();
108-
if (changes.Any())
124+
var existing = existingEntityGroups.ContainsKey(group.Key) ? existingEntityGroups[group.Key] : null;
125+
var change = new EntityGroupChange(name, group.Key, existing, group.Value);
126+
if (change.Scripts.Any())
109127
{
110-
log.LogInformation($"Detected changes for Materialized Views: {changes.Count} changes with {changes.SelectMany(itm => itm.Scripts).Count()} scripts");
111-
result.Add(new Heading("Materialized Views"));
112-
result.AddRange(changes);
128+
changes.Add(change);
113129
}
114130
}
115131

116-
if (newState.ContinuousExports.Any())
117-
{
118-
var tmp = new List<IChange>();
119-
var existingContinuousExports = oldState?.ContinuousExports ?? new Dictionary<string, ContinuousExport>();
120-
log.LogInformation($"Existing materialized views: {string.Join(", ", existingContinuousExports.Keys)}");
132+
return changes;
133+
}
121134

122-
foreach (var view in newState.ContinuousExports)
123-
{
124-
if (existingContinuousExports.ContainsKey(view.Key))
125-
{
126-
var change = new ScriptCompareChange(view.Key, existingContinuousExports[view.Key], view.Value);
127-
log.LogInformation($"Continuous Exports {view.Key} exists, created {change.Scripts.Count} script to apply the diffs");
128-
tmp.Add(change);
129-
}
130-
else
131-
{
132-
var change = new ScriptCompareChange(view.Key, null, view.Value);
133-
log.LogInformation($"Continuous Exports {view.Key} doesn't exist, created {change.Scripts.Count} scripts to create the view");
134-
tmp.Add(change);
135-
}
136-
}
137-
var changes = tmp.Where(itm => itm.Scripts.Any()).ToList();
138-
if (changes.Any())
139-
{
140-
log.LogInformation($"Detected changes for Continuous Exports: {changes.Count} changes with {changes.SelectMany(itm => itm.Scripts).Count()} scripts");
141-
result.Add(new Heading("Continuous Exports "));
142-
result.AddRange(changes);
143-
}
144-
}
135+
private static List<IChange> GenerateScriptCompareChanges<T>(Database oldState, Database newState,Func<Database,Dictionary<string,T>> entitySelector,string entityName, ILogger log) where T: IKustoBaseEntity
136+
{
137+
var tmp = new List<IChange>();
138+
var existing = entitySelector(oldState) ?? new Dictionary<string, T>();
139+
var newItems = entitySelector(newState) ?? new Dictionary<string, T>();
145140

146-
if (newState.Functions.Any())
147-
{
148-
var tmp = new List<IChange>();
149-
var existingFunctions = oldState?.Functions ?? new Dictionary<string, Function>();
150-
log.LogInformation($"Existing functions: {string.Join(", ", existingFunctions.Keys)}");
151141

152-
foreach (var function in newState.Functions)
153-
{
154-
if (existingFunctions.ContainsKey(function.Key))
155-
{
156-
var existingFunction = existingFunctions[function.Key];
157-
var change = new ScriptCompareChange(function.Key, existingFunction, function.Value);
158-
log.LogInformation($"Function {function.Key} exists, created {change.Scripts.Count} script to apply the diffs");
159-
tmp.Add(change);
160-
}
161-
else
162-
{
163-
var change = new ScriptCompareChange(function.Key, null, function.Value);
164-
log.LogInformation($"Function {function.Key} doesn't exist, created {change.Scripts.Count} scripts to create the function");
165-
tmp.Add(change);
166-
}
167-
}
168-
var changes = tmp.Where(itm => itm.Scripts.Any()).ToList();
169-
if (changes.Any())
170-
{
171-
log.LogInformation($"Detected changes for Functions: {changes.Count} changes with {changes.SelectMany(itm => itm.Scripts).Count()} scripts");
172-
result.Add(new Heading("Functions"));
173-
result.AddRange(changes);
174-
}
175-
}
142+
log.LogInformation($"Existing {entityName}: {string.Join(", ", existing.Keys)}");
176143

177-
if(newState.EntityGroups.Any())
144+
foreach (var item in newItems)
178145
{
179-
var changes = new List<IChange>();
180-
var existingEntityGroups = oldState?.EntityGroups ?? new Dictionary<string, List<Entity>>();
181-
foreach (var group in newState.EntityGroups)
146+
if (existing.ContainsKey(item.Key))
182147
{
183-
var existing = existingEntityGroups.ContainsKey(group.Key) ? existingEntityGroups[group.Key] : null;
184-
var change = new EntityGroupChange(name, group.Key, existing, group.Value);
185-
if(change.Scripts.Any())
186-
{
187-
changes.Add(change);
188-
}
148+
var change = new ScriptCompareChange(item.Key, existing[item.Key], item.Value);
149+
log.LogInformation($"{item.Key} already exists, created {change.Scripts.Count} script to apply the diffs");
150+
tmp.Add(change);
189151
}
190-
if (changes.Any())
152+
else
191153
{
192-
log.LogInformation($"Detected changes for Entity Groups: {changes.Count}");
193-
result.Add(new Heading("Entity Groups"));
194-
result.AddRange(changes);
154+
var change = new ScriptCompareChange(item.Key, null, item.Value);
155+
log.LogInformation($"{item.Key} doesn't exist, created {change.Scripts.Count} scripts to create it.");
156+
tmp.Add(change);
195157
}
196158
}
197159

198-
if (newState.ExternalTables.Any())
199-
{
200-
var tmp = new List<IChange>();
201-
var existingExternalTable = oldState?.ExternalTables ?? new Dictionary<string, ExternalTable>();
202-
log.LogInformation($"Existing functions: {string.Join(", ", existingExternalTable.Keys)}");
203-
204-
foreach (var extTable in newState.ExternalTables)
205-
{
206-
if (existingExternalTable.ContainsKey(extTable.Key))
207-
{
208-
var existingFunction = existingExternalTable[extTable.Key];
209-
var change = new ScriptCompareChange(extTable.Key, existingFunction, extTable.Value);
210-
log.LogInformation($"Function {extTable.Key} exists, created {change.Scripts.Count} script to apply the diffs");
211-
tmp.Add(change);
212-
}
213-
else
214-
{
215-
var change = new ScriptCompareChange(extTable.Key, null, extTable.Value);
216-
log.LogInformation($"Function {extTable.Key} doesn't exist, created {change.Scripts.Count} scripts to create the function");
217-
tmp.Add(change);
218-
}
219-
}
220-
var changes = tmp.Where(itm => itm.Scripts.Any()).ToList();
221-
if (changes.Any())
222-
{
223-
log.LogInformation($"Detected changes for Functions: {changes.Count} changes with {changes.SelectMany(itm => itm.Scripts).Count()} scripts");
224-
result.Add(new Heading("Functions"));
225-
result.AddRange(changes);
226-
}
160+
tmp = tmp.Where(itm => itm.Scripts?.Any() == true).ToList();
227161

162+
if(tmp.Count > 0)
163+
{
164+
tmp.Insert(0, new Heading(entityName));
228165
}
229166

230-
231-
return result;
167+
return tmp;
232168
}
233169
}
234170

‎KustoSchemaTools/Changes/PermissionChange.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ namespace KustoSchemaTools.Changes
99

1010
public class PermissionChange : BaseChange<List<AADObject>>
1111
{
12-
public PermissionChange(string db, string entity, List<AADObject>? from, List<AADObject> to) : base("Permissions", entity, from ?? new List<AADObject>(), to)
12+
public PermissionChange(string db, string entity, List<AADObject> from, List<AADObject> to) : base("Permissions", entity, from ?? new List<AADObject>(), to)
1313
{
1414
Db = db;
1515
Init();

‎KustoSchemaTools/DeletionChange.cs‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
using KustoSchemaTools.Changes;
2+
using KustoSchemaTools.Parser;
3+
using System.Text;
4+
5+
namespace KustoSchemaTools
6+
{
7+
public class DeletionChange : IChange
8+
{
9+
public DeletionChange(string entity, string entityType)
10+
{
11+
EntityType = entityType;
12+
Entity = entity;
13+
}
14+
15+
public string EntityType { get; set; }
16+
17+
public string Entity { get; set; }
18+
19+
public List<DatabaseScriptContainer> Scripts => new List<DatabaseScriptContainer> { new DatabaseScriptContainer("Deletion",0,$".drop {EntityType} {Entity}") };
20+
21+
public string Markdown
22+
{
23+
get
24+
{
25+
var sb = new StringBuilder();
26+
sb.AppendLine($"## {Entity}");
27+
sb.AppendLine();
28+
sb.AppendLine("<table>");
29+
sb.AppendLine($"<tr></tr>");
30+
31+
sb.AppendLine("<tr>");
32+
sb.AppendLine($"<td colspan=\"2\">:recycle:</td>");
33+
sb.AppendLine($"<td colspan=\"10\"><pre lang=\"kql\">{Scripts[0].Script.Text.PrettifyKql()}</pre></td>");
34+
sb.AppendLine("</tr>");
35+
sb.AppendLine("</table>");
36+
37+
return sb.ToString();
38+
39+
}
40+
}
41+
42+
43+
}
44+
}

‎KustoSchemaTools/Model/Database.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,18 @@ public class Database
2929

3030
public List<Metadata> Metadata { get; set; } = new List<Metadata> { };
3131

32+
public Deletions Deletions { get; set; } = new Deletions();
33+
34+
}
35+
36+
public class Deletions
37+
{
38+
public List<string> Tables { get; set; } = new List<string>();
39+
public List<string> Columns { get; set; } = new List<string>();
40+
public List<string> MaterializedViews { get; set; } = new List<string>();
41+
public List<string> Functions { get; set; } = new List<string>();
42+
public List<string> ContinuousExports { get; set; } = new List<string>();
43+
public List<string> ExternalTables { get; set; } = new List<string>();
3244
}
3345

3446
}

0 commit comments

Comments
 (0)