Unpack sequences (#1153)

* Add nullable annotations

* Unpack sequences

Author: slozier

Src/IronPython/Compiler/Ast/AstMethods.cs

@@ -81,6 +81,11 @@ internal static class AstMethods {
 
         // methods matching Python opcodes
         public static readonly MethodInfo DictUpdate = GetMethod((Action<CodeContext, PythonDictionary, object>)PythonOps.DictUpdate);
+        public static readonly MethodInfo ListAppend = GetMethod((Action<PythonList, object>)PythonOps.ListAppend);
+        public static readonly MethodInfo ListExtend = GetMethod((Action<PythonList, object>)PythonOps.ListExtend);
+        public static readonly MethodInfo ListToTuple = GetMethod((Func<PythonList, PythonTuple>)PythonOps.ListToTuple);
+        public static readonly MethodInfo SetAdd = GetMethod((Action<SetCollection, object>)PythonOps.SetAdd);
+        public static readonly MethodInfo SetUpdate = GetMethod((Action<SetCollection, object>)PythonOps.SetUpdate);
 
         private static MethodInfo GetMethod(Delegate x) {
             return x.Method;

Src/IronPython/Compiler/Ast/Expression.cs

@@ -2,18 +2,39 @@
 // The .NET Foundation licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information.
 
-using MSAst = System.Linq.Expressions;
+#nullable enable
 
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
+using System.Reflection;
+using System.Runtime.CompilerServices;
+
+using IronPython.Runtime.Binding;
 
 using Microsoft.Scripting;
 
-using IronPython.Runtime.Binding;
+using AstUtils = Microsoft.Scripting.Ast.Utils;
+using MSAst = System.Linq.Expressions;
 
 namespace IronPython.Compiler.Ast {
     public abstract class Expression : Node {
-        internal static Expression[] EmptyArray = new Expression[0];
+        internal static readonly Expression[] EmptyArray = Array.Empty<Expression>();
+
+        protected internal static MSAst.BlockExpression UnpackSequenceHelper<T>(IList<Expression> items, MethodInfo makeEmpty, MethodInfo append, MethodInfo extend) {
+            var expressions = new ReadOnlyCollectionBuilder<MSAst.Expression>(items.Count + 2);
+            var varExpr = Expression.Variable(typeof(T), "$coll");
+            expressions.Add(Expression.Assign(varExpr, Expression.Call(makeEmpty)));
+            foreach (var item in items) {
+                if (item is StarredExpression starredExpression) {
+                    expressions.Add(Expression.Call(extend, varExpr, AstUtils.Convert(starredExpression.Value, typeof(object))));
+                } else {
+                    expressions.Add(Expression.Call(append, varExpr, AstUtils.Convert(item, typeof(object))));
+                }
+            }
+            expressions.Add(varExpr);
+            return Expression.Block(typeof(T), new MSAst.ParameterExpression[] { varExpr }, expressions);
+        }
 
         internal virtual MSAst.Expression TransformSet(SourceSpan span, MSAst.Expression right, PythonOperationKind op) {
             // unreachable, CheckAssign prevents us from calling this at parse time.
@@ -26,23 +47,15 @@ internal virtual MSAst.Expression TransformDelete() {
             throw new InvalidOperationException();
         }
 
-        internal virtual ConstantExpression ConstantFold() => null;
+        internal virtual ConstantExpression? ConstantFold() => null;
 
-        internal virtual string CheckAssign() => "can't assign to " + NodeName;
+        internal virtual string? CheckAssign() => "can't assign to " + NodeName;
 
-        internal virtual string CheckAugmentedAssign() => CheckAssign();
+        internal virtual string? CheckAugmentedAssign() => CheckAssign();
 
-        internal virtual string CheckDelete() => "can't delete " + NodeName;
+        internal virtual string? CheckDelete() => "can't delete " + NodeName;
 
-        internal virtual bool IsConstant {
-            get {
-                var folded = ConstantFold();
-                if (folded != null) {
-                    return folded.IsConstant;
-                }
-                return false;
-            }
-        }
+        internal virtual bool IsConstant => ConstantFold()?.IsConstant ?? false;
 
         internal virtual object GetConstantValue() {            
             var folded = ConstantFold();

Src/IronPython/Compiler/Ast/ListExpression.cs

@@ -2,21 +2,25 @@
 // The .NET Foundation licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information.
 
+#nullable enable
+
+using IronPython.Runtime;
+
 using MSAst = System.Linq.Expressions;
 
 namespace IronPython.Compiler.Ast {
-    using Ast = MSAst.Expression;
-
     public class ListExpression : SequenceExpression {
         public ListExpression(params Expression[] items)
             : base(items) {
         }
 
         public override MSAst.Expression Reduce() {
             if (Items.Count == 0) {
-                return Ast.Call(
-                    AstMethods.MakeEmptyList
-                );
+                return Expression.Call(AstMethods.MakeEmptyList);
+            }
+
+            if (HasStarredExpression) {
+                return UnpackSequenceHelper<PythonList>(Items, AstMethods.MakeEmptyList, AstMethods.ListAppend, AstMethods.ListExtend);
             }
 
             return Call(

Src/IronPython/Compiler/Ast/SequenceExpression.cs

@@ -2,23 +2,21 @@
 // The .NET Foundation licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information.
 
+#nullable enable
+
 using System.Collections.Generic;
-using System.Collections.ObjectModel;
+using System.Linq;
 using System.Runtime.CompilerServices;
 
-using Microsoft.Scripting;
-using Microsoft.Scripting.Runtime;
-
 using IronPython.Runtime.Binding;
-using IronPython.Runtime.Operations;
 
-using MSAst = System.Linq.Expressions;
+using Microsoft.Scripting;
+using Microsoft.Scripting.Runtime;
 
 using AstUtils = Microsoft.Scripting.Ast.Utils;
+using MSAst = System.Linq.Expressions;
 
 namespace IronPython.Compiler.Ast {
-    using Ast = MSAst.Expression;
-
     public abstract class SequenceExpression : Expression {
         private readonly Expression[] _items;
 
@@ -28,6 +26,8 @@ protected SequenceExpression(Expression[] items) {
 
         public IList<Expression> Items => _items;
 
+        protected bool HasStarredExpression => Items.OfType<StarredExpression>().Any();
+
         internal override MSAst.Expression TransformSet(SourceSpan span, MSAst.Expression right, PythonOperationKind op) {
             // if we just have a simple named multi-assignment  (e.g. a, b = 1,2)
             // then go ahead and step over the entire statement at once.  If we have a 
@@ -58,7 +58,7 @@ internal override MSAst.Expression TransformSet(SourceSpan span, MSAst.Expressio
             }
 
             // 1. Evaluate the expression and assign the value to the temp.
-            MSAst.ParameterExpression right_temp = Ast.Variable(typeof(object), "unpacking");
+            MSAst.ParameterExpression right_temp = Expression.Variable(typeof(object), "unpacking");
 
             // 2. Add the assignment "right_temp = right" into the suite/block
             MSAst.Expression assignStmt1 = MakeAssignment(right_temp, right);
@@ -92,7 +92,7 @@ internal override MSAst.Expression TransformSet(SourceSpan span, MSAst.Expressio
             ), typeof(object[]));
 
             // 4. Create temporary variable for the array
-            MSAst.ParameterExpression array_temp = Ast.Variable(typeof(object[]), "array");
+            MSAst.ParameterExpression array_temp = Expression.Variable(typeof(object[]), "array");
 
             // 5. Assign the value of the method call (mce) into the array temp
             // And add the assignment "array_temp = Ops.GetEnumeratorValues(...)" into the block
@@ -112,7 +112,7 @@ internal override MSAst.Expression TransformSet(SourceSpan span, MSAst.Expressio
                 }
 
                 // 6. array_temp[i]
-                MSAst.Expression element = Ast.ArrayAccess(
+                MSAst.Expression element = Expression.ArrayAccess(
                     array_temp,                             // array expression
                     AstUtils.Constant(i)                         // index
                 );
@@ -129,13 +129,13 @@ internal override MSAst.Expression TransformSet(SourceSpan span, MSAst.Expressio
             }
             // 9. add the sets as their own block so they can be marked as a single span, if necessary.
             sets.Add(AstUtils.Empty());
-            MSAst.Expression itemSet = GlobalParent.AddDebugInfo(Ast.Block(sets.ToReadOnlyCollection()), leftSpan);
+            MSAst.Expression itemSet = GlobalParent.AddDebugInfo(Expression.Block(sets.ToReadOnlyCollection()), leftSpan);
 
             // 10. Return the suite statement (block)
-            return GlobalParent.AddDebugInfo(Ast.Block(new[] { array_temp, right_temp }, assignStmt1, assignStmt2, itemSet, AstUtils.Empty()), totalSpan);
+            return GlobalParent.AddDebugInfo(Expression.Block(new[] { array_temp, right_temp }, assignStmt1, assignStmt2, itemSet, AstUtils.Empty()), totalSpan);
         }
 
-        internal override string CheckAssign() {
+        internal override string? CheckAssign() {
             var starCount = 0;
             foreach (var item in Items) {
                 if (item.CheckAssign() is { } checkAssign) {
@@ -157,7 +157,7 @@ internal override string CheckAssign() {
             return null;
         }
 
-        internal override string CheckDelete() => null;
+        internal override string? CheckDelete() => null;
 
         internal override string CheckAugmentedAssign()
             => CheckAssign() ?? "illegal expression for augmented assignment";
@@ -170,7 +170,7 @@ internal override MSAst.Expression TransformDelete() {
                 statements[i] = _items[i].TransformDelete();
             }
             statements[_items.Length] = AstUtils.Empty();
-            return GlobalParent.AddDebugInfo(Ast.Block(statements), Span);
+            return GlobalParent.AddDebugInfo(Expression.Block(statements), Span);
         }
 
         internal override bool CanThrow {

Src/IronPython/Compiler/Ast/SetExpression.cs

@@ -2,17 +2,19 @@
 // The .NET Foundation licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information.
 
-using MSAst = System.Linq.Expressions;
+#nullable enable
 
 using System.Collections.Generic;
+using System.Linq;
+
+using IronPython.Runtime;
 
 using Microsoft.Scripting.Utils;
 
 using AstUtils = Microsoft.Scripting.Ast.Utils;
+using MSAst = System.Linq.Expressions;
 
 namespace IronPython.Compiler.Ast {
-    using Ast = MSAst.Expression;
-
     public class SetExpression : Expression {
         private readonly Expression[] _items;
 
@@ -24,7 +26,17 @@ public SetExpression(params Expression[] items) {
 
         public IList<Expression> Items => _items;
 
+        protected bool HasStarredExpression => Items.OfType<StarredExpression>().Any();
+
         public override MSAst.Expression Reduce() {
+            if (Items.Count == 0) {
+                return Expression.Call(AstMethods.MakeEmptySet);
+            }
+
+            if (HasStarredExpression) {
+                return UnpackSequenceHelper<SetCollection>(Items, AstMethods.MakeEmptySet, AstMethods.SetAdd, AstMethods.SetUpdate);
+            }
+
             return Expression.Call(
                 AstMethods.MakeSet,
                 NewArrayInit(

Src/IronPython/Compiler/Ast/StarredExpression.cs

@@ -29,9 +29,9 @@ public StarredExpression(Expression value) {
         internal override MSAst.Expression TransformSet(SourceSpan span, MSAst.Expression right, PythonOperationKind op)
             => Value.TransformSet(span, right, op);
 
-        internal override string CheckAssign() => Value.CheckAssign();
+        internal override string? CheckAssign() => Value.CheckAssign();
 
-        internal override string CheckDelete() => "can use starred expression only as assignment target";
+        internal override string CheckDelete() => "can't use starred expression here"; // TODO: change error message in 3.9
 
         internal override MSAst.Expression TransformDelete() => Value.TransformDelete();
 
@@ -76,10 +76,16 @@ public override bool Walk(ForStatement node) {
         }
 
         public override bool Walk(StarredExpression node) {
-            ReportSyntaxError("can use starred expression only as assignment target", node);
+            ReportSyntaxError("can't use starred expression here", node);
             return base.Walk(node);
         }
 
+        public override bool Walk(ListExpression node) => WalkItems(node.Items);
+
+        public override bool Walk(SetExpression node) => WalkItems(node.Items);
+
+        public override bool Walk(TupleExpression node) => WalkItems(node.Items);
+
         private void ReportSyntaxError(string message, Node node) {
             context.Errors.Add(context.SourceUnit, message, node.Span, ErrorCodes.SyntaxError, Severity.FatalError);
         }

Src/IronPython/Compiler/Ast/TupleExpression.cs

@@ -2,28 +2,21 @@
 // The .NET Foundation licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information.
 
-using System;
-
-using Microsoft.Scripting;
-using Microsoft.Scripting.Runtime;
+#nullable enable
 
 using IronPython.Runtime;
-using IronPython.Runtime.Binding;
 using IronPython.Runtime.Operations;
 
 using MSAst = System.Linq.Expressions;
 
-
 namespace IronPython.Compiler.Ast {
-    using Ast = MSAst.Expression;
-
     public class TupleExpression : SequenceExpression {
         public TupleExpression(bool expandable, params Expression[] items)
             : base(items) {
             IsExpandable = expandable;
         }
 
-        internal override string CheckAssign() {
+        internal override string? CheckAssign() {
             if (Items.Count == 0) {
                 //  TODO: remove this when we get to 3.6
                 return "can't assign to ()";
@@ -32,30 +25,34 @@ internal override string CheckAssign() {
             return base.CheckAssign();
         }
 
-        internal override string CheckDelete() {
+        internal override string? CheckDelete() {
             if (Items.Count == 0)
                 return "can't delete ()"; //  TODO: remove this when we get to 3.6
             return base.CheckDelete();
         }
 
         public override MSAst.Expression Reduce() {
             if (IsExpandable) {
-                return Ast.NewArrayInit(
+                return Expression.NewArrayInit(
                     typeof(object),
                     ToObjectArray(Items)
                 );
             }
 
             if (Items.Count == 0) {
-                return Ast.Field(
-                    null,
-                    typeof(PythonOps).GetField(nameof(PythonOps.EmptyTuple))
+                return Expression.Field(
+                    null!,
+                    typeof(PythonOps).GetField(nameof(PythonOps.EmptyTuple))!
                 );
             }
 
-            return Ast.Call(
+            if (HasStarredExpression) {
+                return Expression.Call(AstMethods.ListToTuple, UnpackSequenceHelper<PythonList>(Items, AstMethods.MakeEmptyList, AstMethods.ListAppend, AstMethods.ListExtend));
+            }
+
+            return Expression.Call(
                 AstMethods.MakeTuple,
-                Ast.NewArrayInit(
+                Expression.NewArrayInit(
                     typeof(object),
                     ToObjectArray(Items)
                 )