Skip to content

Commit 157e99d

Browse files
author
Thomas Mahlberg
committed
First implementation of async commands
1 parent 36b8834 commit 157e99d

13 files changed

Lines changed: 228 additions & 18 deletions

‎KustoSchemaTools/Changes/BaseChange.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ protected BaseChange(string entityType, string entity, T from, T to)
2222

2323
public string Markdown { get; protected set; }
2424

25+
public bool IsAsync { get; set; }
26+
2527
}
2628

2729
}

‎KustoSchemaTools/Changes/DatabaseScriptContainer.cs‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,24 +9,25 @@ public DatabaseScriptContainer()
99

1010
}
1111

12-
public DatabaseScriptContainer(DatabaseScript script, string kind, bool? isValid = null)
12+
public DatabaseScriptContainer(DatabaseScript script, string kind, bool isAsync = false)
1313
{
1414
Script = script;
1515
Kind = kind;
16-
IsValid = isValid;
16+
IsAsync = isAsync;
1717
}
1818

19-
public DatabaseScriptContainer(string kind, int order, string script, bool? isValid = null)
19+
public DatabaseScriptContainer(string kind, int order, string script, bool isAsync = false)
2020
{
2121
Script = new DatabaseScript(script, order);
2222
Kind = kind;
23-
IsValid = isValid;
23+
IsAsync = isAsync;
2424
}
2525

2626
public DatabaseScript Script { get; set; }
2727
public string Kind{ get; set; }
2828
public bool? IsValid { get; set; }
2929
public string Text => Script.Text;
3030
public int Order => Script.Order;
31+
public bool IsAsync { get;set; }
3132
}
3233
}

‎KustoSchemaTools/Changes/GenericBaseEntity.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ public GenericBaseEntity(List<DatabaseScriptContainer> scripts)
1111

1212
public List<DatabaseScriptContainer> Scripts { get; }
1313

14-
public List<DatabaseScriptContainer> CreateScripts(string name)
14+
public List<DatabaseScriptContainer> CreateScripts(string name, bool isNew)
1515
{
1616
return Scripts;
1717
}

‎KustoSchemaTools/Changes/MaterializedViewChange.cs‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
using Kusto.Language;
2+
using KustoSchemaTools.Model;
3+
using KustoSchemaTools.Parser;
4+
using System.Text;
5+
6+
namespace KustoSchemaTools.Changes
7+
{
8+
9+
10+
public class MaterializedViewChange : BaseChange<MaterializedView>
11+
{
12+
public MaterializedViewChange(string db, string entity, MaterializedView from, MaterializedView to) : base("MaterializedView", entity, from, to)
13+
{
14+
Db = db;
15+
Init();
16+
}
17+
18+
public string Db { get; }
19+
20+
private void Init()
21+
{
22+
if(From == null && To.EffectiveDateTime != null)
23+
{
24+
IsAsync = true;
25+
26+
}
27+
else
28+
{
29+
30+
31+
}
32+
33+
34+
35+
36+
/*
37+
38+
39+
DatabaseScript toScript;
40+
DatabaseScript fromScript;
41+
if (To.Count > 0)
42+
{
43+
var x = string.Join(",", To.Select(a => "\"" + a.Id + "\""));
44+
toScript = new DatabaseScript { Text = $".set database {Db} {Entity.ToLower()} ({x})", Order = 0 };
45+
}
46+
else
47+
{
48+
toScript = new DatabaseScript { Text = $".set database {Db} {Entity.ToLower()} none", Order = 0 };
49+
}
50+
if (To.Count > 0)
51+
{
52+
var x = string.Join(",", From.Select(a => "\"" + a.Id + "\""));
53+
fromScript = new DatabaseScript { Text = $".set database {Db} {Entity.ToLower()} ({x})", Order = 0 };
54+
}
55+
else
56+
{
57+
fromScript = new DatabaseScript { Text = $".set database {Db} {Entity.ToLower()} none", Order = 0 };
58+
}
59+
60+
61+
var script = "No database changes";
62+
63+
var scriptContainer = new DatabaseScriptContainer(toScript, "");
64+
65+
if (fromScript.Text != toScript.Text)
66+
{
67+
Scripts.Add(scriptContainer);
68+
script = toScript.Text;
69+
}
70+
71+
var code = KustoCode.Parse(toScript.Text);
72+
var diagnostics = code.GetDiagnostics();
73+
scriptContainer.IsValid = diagnostics.Any() == false;
74+
var logo = scriptContainer.IsValid.Value ? ":green_circle:" : ":red_circle:";
75+
76+
var changed = From
77+
.Join(To, itm => itm.Id, itm => itm.Id, (from, to) => new { from, to })
78+
.Where(itm => itm.from.Name != itm.to.Name)
79+
.ToList();
80+
var added = To.Where(itm => From.All(t => t.Id != itm.Id)).ToList();
81+
var removed = From.Where(itm => To.All(t => t.Id != itm.Id)).ToList();
82+
83+
var sb = new StringBuilder();
84+
sb.AppendLine($"## {Entity}");
85+
sb.AppendLine();
86+
sb.AppendLine("<table>");
87+
sb.AppendLine($"<tr></tr>");
88+
89+
90+
91+
if (added.Any())
92+
{
93+
sb.AppendLine("<tr>");
94+
sb.AppendLine("<td colspan=\"2\">Added:</td>");
95+
var addedIds = string.Join("<br>", added.Select(t => $"{t.Name} ({t.Id})"));
96+
sb.AppendLine($"<td colspan=\"10\">{addedIds}:</td>");
97+
sb.AppendLine("</tr>");
98+
}
99+
if (removed.Any())
100+
{
101+
sb.AppendLine("<tr>");
102+
sb.AppendLine("<td colspan=\"2\">Removed:</td>");
103+
var removedIds = string.Join("<br>", removed.Select(t => $"{t.Name} ({t.Id})"));
104+
sb.AppendLine($"<td colspan=\"10\">{removedIds}:</td>");
105+
sb.AppendLine("</tr>");
106+
}
107+
if (changed.Any())
108+
{
109+
Scripts.Add(new DatabaseScriptContainer("PermissionRenamed", -1,"// No Database Change"));
110+
sb.AppendLine("<tr>");
111+
sb.AppendLine("<td colspan=\"2\">Changed:</td>");
112+
var changedIds = string.Join("<br>", changed.Select(t => $"{t.from.Name} => {t.to.Name} ({t.to.Id})"));
113+
sb.AppendLine($"<td colspan=\"10\">{changedIds}:</td>");
114+
sb.AppendLine("</tr>");
115+
}
116+
sb.AppendLine("<tr>");
117+
sb.AppendLine($"<td colspan=\"2\">{logo}</td>");
118+
sb.AppendLine($"<td colspan=\"10\"><pre lang=\"kql\">{script.PrettifyKql()}</pre></td>");
119+
sb.AppendLine("</tr>");
120+
sb.AppendLine("</table>");
121+
122+
123+
Markdown = sb.ToString();*/
124+
}
125+
}
126+
127+
}

‎KustoSchemaTools/Changes/ScriptCompareChange.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ public ScriptCompareChange(string entity, IKustoBaseEntity? from, IKustoBaseEnti
1919

2020
private void Init()
2121
{
22-
var from = From?.CreateScripts(Entity).ToDictionary(itm => itm.Kind) ?? new Dictionary<string, DatabaseScriptContainer>();
23-
var to = To.CreateScripts(Entity);
22+
var from = From?.CreateScripts(Entity, false).ToDictionary(itm => itm.Kind) ?? new Dictionary<string, DatabaseScriptContainer>();
23+
var to = To.CreateScripts(Entity, from != null);
2424
Markdown = string.Empty;
2525

2626
if (to.Any() == false) return;

‎KustoSchemaTools/Model/ContinuousExport.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ public class ContinuousExport : IKustoBaseEntity
1414
[YamlMember(ScalarStyle = YamlDotNet.Core.ScalarStyle.Literal)]
1515
public string Query { get; set; }
1616

17-
public List<DatabaseScriptContainer> CreateScripts(string name)
17+
public List<DatabaseScriptContainer> CreateScripts(string name, bool isNew)
1818
{
1919
return new List<DatabaseScriptContainer>
2020
{

‎KustoSchemaTools/Model/ExternalTable.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ public class ExternalTable : IKustoBaseEntity
4040

4141
#endregion
4242

43-
public List<DatabaseScriptContainer> CreateScripts(string name)
43+
public List<DatabaseScriptContainer> CreateScripts(string name, bool isNew)
4444
{
4545
var container = new DatabaseScriptContainer
4646
{

‎KustoSchemaTools/Model/Function.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ public class Function : IKustoBaseEntity
1818

1919
public string Body { get; set; }
2020

21-
public List<DatabaseScriptContainer> CreateScripts(string name)
21+
public List<DatabaseScriptContainer> CreateScripts(string name, bool isNew)
2222
{
2323
var properties = GetType().GetProperties()
2424
.Where(p => p.GetValue(this) != null && p.Name != "Body" && p.Name != "Parameters")

‎KustoSchemaTools/Model/IKustoBaseEntity.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ namespace KustoSchemaTools.Model
44
{
55
public interface IKustoBaseEntity
66
{
7-
List<DatabaseScriptContainer> CreateScripts(string name);
7+
List<DatabaseScriptContainer> CreateScripts(string name, bool isNew);
88
}
99

1010
}

‎KustoSchemaTools/Model/MaterializedView.cs‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,27 +14,46 @@ public class MaterializedView : IKustoBaseEntity
1414
public string? EffectiveDateTime { get; set; }
1515
public string Lookback { get; set; }
1616
public bool? UpdateExtentsCreationTime { get; set; }
17+
public bool? Backfill { get; set; }
1718
public bool AutoUpdateSchema { get; set; } = false;
1819
public List<string> DimensionTables { get; set; }
1920
public RetentionAndCachePolicy RetentionAndCachePolicy { get; set; } = new RetentionAndCachePolicy();
2021
[YamlMember(ScalarStyle = ScalarStyle.Literal)]
2122
public string Query { get; set; }
2223
public string? RowLevelSecurity { get; set; }
2324

24-
public List<DatabaseScriptContainer> CreateScripts(string name)
25+
public List<DatabaseScriptContainer> CreateScripts(string name, bool isNew)
2526
{
26-
var excludedProperies = new HashSet<string>( new[] { "Query", "Source", "Kind", "RetentionAndCachePolicy", "RowLevelSecurity" });
27+
var asyncSetup = isNew && Backfill == true && !string.IsNullOrWhiteSpace(EffectiveDateTime);
28+
29+
30+
var excludedProperies = new HashSet<string>(["Query", "Source", "Kind", "RetentionAndCachePolicy", "RowLevelSecurity"]);
31+
if (!asyncSetup)
32+
{
33+
excludedProperies.Add("EffectiveDateTime");
34+
excludedProperies.Add("Backfill");
35+
}
2736

2837
var scripts = new List<DatabaseScriptContainer>();
2938
var properties = string.Join(", ", GetType().GetProperties()
3039
.Where(p => p.GetValue(this) != null && excludedProperies.Contains(p.Name) == false)
3140
.Select(p => new {Name = p.Name, Value = p.GetValue(this) })
3241
.Where(p => !string.IsNullOrWhiteSpace(p.Value?.ToString()))
3342
.Select(p => $"{p.Name}=\"{p.Value}\""));
34-
scripts.Add(new DatabaseScriptContainer("CreateOrAlterMaterializedView", 40, $".create-or-alter materialized-view with ({properties}) {name} on {Kind} {Source} {{ {Query} }}"));
43+
44+
if (asyncSetup)
45+
{
46+
scripts.Add(new DatabaseScriptContainer("CreateMaterializedView", Kind == "table" ? 40 : 41, $".create async ifnotexists materialized-view with ({properties}) {name} on {Kind} {Source} {{ {Query} }}", true));
47+
}
48+
else
49+
{
50+
scripts.Add(new DatabaseScriptContainer("CreateMaterializedView", Kind == "table" ? 40 : 41, $".create-or-alter materialized-view with ({properties}) {name} on {Kind} {Source} {{ {Query} }}"));
51+
}
3552

3653
if (RetentionAndCachePolicy != null)
54+
{
3755
scripts.AddRange(RetentionAndCachePolicy.CreateScripts(name, "materialized-view"));
56+
}
3857

3958

4059
if (!string.IsNullOrEmpty(RowLevelSecurity))

0 commit comments

Comments
 (0)