Merge branch 'master' into 3.6

slozier · slozier · commit f67e9978f46d · 2022-07-30T16:30:30.000-04:00
diff --git a/Build.proj b/Build.proj
@@ -23,6 +23,7 @@
     <PackageVersion Condition="'$(ReleaseLevel)' != 'final' or '$(ReleaseSerial)' != '0'">$(MajorVersion).$(MinorVersion).$(MicroVersion)-$(ReleaseLevel)$(ReleaseSerial)</PackageVersion>
     <StageDir>$(RootDir)Package\$(Configuration)\Stage\IronPython-$(PackageVersion)</StageDir>
     <PackageDir>$(RootDir)Package\$(Configuration)\Packages\IronPython-$(PackageVersion)</PackageDir>
+    <StdLibDirName>lib</StdLibDirName>
   </PropertyGroup>
 
   <PropertyGroup>
@@ -105,7 +106,7 @@
 
   <Target Name="_CopyStdLib" DependsOnTargets="_MakeStageDir">
     <MSBuild Projects="$(RootDir)\Src\StdLib\StdLib.pyproj"
-             Properties="OutputPath=$(StageDir)"
+             Properties="OutputPath=$(StageDir);StdLibDirName=$(StdLibDirName)"
              Targets="Stage" />
   </Target>
 
diff --git a/Package/choco/IronPython.nuspec b/Package/choco/IronPython.nuspec
@@ -21,7 +21,7 @@
   <files>
     <file src="$STAGEDIR$\net462\*.dll;$STAGEDIR$\net462\*.exe" exclude="**\rowantest*.dll;**\IronPythonTest.dll" />
     <file src="$STAGEDIR$\net462\DLLs\**" target="DLLs" exclude="**\*.xml" />
-    <file src="$STAGEDIR$\Lib\**" target="Lib" />
+    <file src="$STAGEDIR$\lib\**" target="lib" />
     <file src="$STAGEDIR$\LICENSE;$STAGEDIR$\README.md" />
     <file src="tools\*" target="tools" />
   </files>
diff --git a/Package/deb/Deb.Packaging.targets b/Package/deb/Deb.Packaging.targets
@@ -8,7 +8,7 @@
     </PropertyGroup>
 
     <ItemGroup>
-      <StdLibFiles Include="$(StageDir)/Lib/**/*.*" />
+      <StdLibFiles Include="$(StageDir)/$(StdLibDirName)/**/*.*" />
     </ItemGroup>
 
     <ItemGroup>
@@ -31,7 +31,7 @@
           Replacement="$(PackageVersion)" />
 
     <ItemGroup>
-      <StdLibOutputFiles Include="$(StageDir)/Lib/**/*.py" />
+      <StdLibOutputFiles Include="$(StageDir)/$(StdLibDirName)/**/*.py" />
     </ItemGroup>
     <Exec Command="dos2unix %(StdLibOutputFiles.Identity)" />
     <Exec Command="dos2unix $(PackageTempFolder)/DEBIAN/control"/>
diff --git a/Package/msi/IronPython.Installer.wixproj b/Package/msi/IronPython.Installer.wixproj
@@ -35,7 +35,7 @@
       <Value>$(BuildDir)</Value>
     </WixConstant>
     <WixConstant Include="StdLibDir">
-      <Value>$(BindInputPaths)\Lib</Value>
+      <Value>$(BindInputPaths)\lib</Value>
     </WixConstant>
     <WixConstant Include="PlatformDir">
       <Value>$(BindInputPaths)\net462</Value>
diff --git a/Package/nuget/IronPython.StdLib.nuspec b/Package/nuget/IronPython.StdLib.nuspec
@@ -22,7 +22,7 @@
   </metadata>
   <files>
     <file src="StdLib.License.txt" />
-    <file src="Lib\**" target="contentFiles\any\any\Lib" />
-    <file src="Lib\**" target="content\Lib" />
+    <file src="lib\**" target="contentFiles\any\any\lib" />
+    <file src="lib\**" target="content\lib" />
   </files>
 </package>
diff --git a/Package/pkg/Pkg.Packaging.targets b/Package/pkg/Pkg.Packaging.targets
@@ -11,7 +11,7 @@
     </PropertyGroup>
 
     <ItemGroup>
-      <StdLibFiles Include="$(StageDir)/Lib/**/*.*" />
+      <StdLibFiles Include="$(StageDir)/$(StdLibDirName)/**/*.*" />
     </ItemGroup>
 
     <ItemGroup>
@@ -28,7 +28,7 @@
     <Exec Command="mkdir -p $(PackageTempFolder)" />
 
     <ItemGroup>
-      <StdLibOutputFiles Include="$(StageDir)/Lib/**/*.py" />
+      <StdLibOutputFiles Include="$(StageDir)/$(StdLibDirName)/**/*.py" />
     </ItemGroup>
 
     <!-- Change line endings of the .py files -->
diff --git a/Package/zip/Zip.Packaging.targets b/Package/zip/Zip.Packaging.targets
@@ -9,8 +9,8 @@
     <Zip Files="@(ZipFiles)" ZipFileName="$(PackageDir)\IronPython.$(PackageVersion).zip" WorkingDirectory="$(StageDir)" />
 
     <ItemGroup>
-      <StdLibFiles Include="$(StageDir)\Lib\**\*.*" />
+      <StdLibFiles Include="$(StageDir)\$(StdLibDirName)\**\*.*" />
     </ItemGroup>
-    <Zip Files="@(StdLibFiles)" ZipFileName="$(PackageDir)\IronPython.StdLib.$(PackageVersion).zip" WorkingDirectory="$(StageDir)\Lib"/>
+    <Zip Files="@(StdLibFiles)" ZipFileName="$(PackageDir)\IronPython.StdLib.$(PackageVersion).zip" WorkingDirectory="$(StageDir)\$(StdLibDirName)"/>
   </Target>
 </Project>
diff --git a/Src/DLR b/Src/DLR
@@ -1 +1 @@
-Subproject commit 5ddc2453417ccae8a1ac3bd0e286a7cfb0e864df
+Subproject commit 0b16ce011d40f7f9bfb66f97d23e6089b9f9b2b8
diff --git a/Src/IronPython.Modules/ResourceMetaPathImporter.cs b/Src/IronPython.Modules/ResourceMetaPathImporter.cs
@@ -88,8 +88,8 @@ fully qualified (dotted) module name. It returns the imported
             )]
         public object load_module(CodeContext /*!*/ context, string fullname) {
             var modules = context.LanguageContext.SystemStateModules;
-            if (modules.ContainsKey(fullname))
-                return modules[fullname];
+            if (modules.TryGetValue(fullname, out var module))
+                return module;
 
             bool ispackage;
             string modpath;
diff --git a/Src/IronPython.Modules/_csv.cs b/Src/IronPython.Modules/_csv.cs
@@ -286,10 +286,9 @@ public static Dialect Create(CodeContext/*!*/ context,
                 kwArgs.TryGetValue("strict", out strict);
 
                 if (dialect != null) {
-                    if (dialect is string) {
-                        string dialectName = (string)dialect;
-                        if (dialects.ContainsKey(dialectName))
-                            dialect = dialects[dialectName];
+                    if (dialect is string dialectName) {
+                        if (dialects.TryGetValue(dialectName, out var value))
+                            dialect = value;
                         else
                             throw MakeError("unknown dialect");
                     }
diff --git a/Src/IronPython.Modules/re.cs b/Src/IronPython.Modules/re.cs
@@ -1061,8 +1061,8 @@ static string ApplyVerbose(string pattern) {
                                                     if (tmpIndex == -1) throw PythonExceptions.CreateThrowable(error(context), "unexpected end of regex");
 
                                                     var namedGroup = pattern.Substring(nameIndex + 2, tmpIndex - (nameIndex + 2));
-                                                    if (namedGroups.ContainsKey(namedGroup)) {
-                                                        throw PythonExceptions.CreateThrowable(error(context), $"redefinition of group name '{namedGroup}' as group {groupCount}; was group {namedGroups[namedGroup]}");
+                                                    if (namedGroups.TryGetValue(namedGroup, out var value)) {
+                                                        throw PythonExceptions.CreateThrowable(error(context), $"redefinition of group name '{namedGroup}' as group {groupCount}; was group {value}");
                                                     }
 
                                                     namedGroups[namedGroup] = groupCount;
diff --git a/Src/IronPython/Hosting/PythonCommandLine.cs b/Src/IronPython/Hosting/PythonCommandLine.cs
@@ -296,8 +296,8 @@ private void InitializeModules() {
                 }
 
                 prefix = pyvenv_prefix ?? prefix;
-                // Make sure there an IronPython Lib directory, and if not keep looking up
-                while (prefix != null && !File.Exists(Path.Combine(prefix, "Lib/os.py"))) {
+                // Make sure there an IronPython lib directory, and if not keep looking up
+                while (prefix != null && !File.Exists(Path.Combine(prefix, "lib", "os.py"))) {
                     prefix = Path.GetDirectoryName(prefix);
                 }
             }
diff --git a/Src/IronPython/Hosting/PythonOptionsParser.cs b/Src/IronPython/Hosting/PythonOptionsParser.cs
@@ -205,23 +205,24 @@ protected override void ParseArgument(string/*!*/ arg) {
         }
 
         protected override void HandleImplementationSpecificOption(string arg, string val) {
+            object frames;
             switch (arg) {
                 case "NoFrames":
-                    if (LanguageSetup.Options.ContainsKey("Frames") && LanguageSetup.Options["Frames"] != ScriptingRuntimeHelpers.False) {
+                    if (LanguageSetup.Options.TryGetValue("Frames", out frames) && frames != ScriptingRuntimeHelpers.False) {
                         throw new InvalidOptionException("Only one of -X [Full]Frames/NoFrames may be specified");
                     }
                     LanguageSetup.Options["Frames"] = ScriptingRuntimeHelpers.False;
                     break;
 
                 case "Frames":
-                    if (LanguageSetup.Options.ContainsKey("Frames") && LanguageSetup.Options["Frames"] != ScriptingRuntimeHelpers.True) {
+                    if (LanguageSetup.Options.TryGetValue("Frames", out frames) && frames != ScriptingRuntimeHelpers.True) {
                         throw new InvalidOptionException("Only one of -X [Full]Frames/NoFrames may be specified");
                     }
                     LanguageSetup.Options["Frames"] = ScriptingRuntimeHelpers.True;
                     break;
 
                 case "FullFrames":
-                    if (LanguageSetup.Options.ContainsKey("Frames") && LanguageSetup.Options["Frames"] != ScriptingRuntimeHelpers.True) {
+                    if (LanguageSetup.Options.TryGetValue("Frames", out frames) && frames != ScriptingRuntimeHelpers.True) {
                         throw new InvalidOptionException("Only one of -X [Full]Frames/NoFrames may be specified");
                     }
                     LanguageSetup.Options["Frames"] = LanguageSetup.Options["FullFrames"] = ScriptingRuntimeHelpers.True;
diff --git a/Src/IronPython/IronPython.csproj b/Src/IronPython/IronPython.csproj
@@ -35,10 +35,10 @@
   </ItemGroup>
 
   <ItemGroup>
-    <None Update="Lib\**\*.py">
+    <None Include="Lib\**\*.py" Link="lib\%(RecursiveDir)%(Filename)%(Extension)">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
-    <None Include="..\StdLib\Lib\os.py" Link="Lib\os.py">
+    <None Include="..\StdLib\Lib\os.py" Link="lib\os.py">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
   </ItemGroup>
diff --git a/Src/IronPython/Runtime/Binding/MetaPythonFunction.cs b/Src/IronPython/Runtime/Binding/MetaPythonFunction.cs
@@ -365,11 +365,21 @@ private CallSignature Signature {
             /// Makes the test for our rule.
             /// </summary>
             private BindingRestrictions/*!*/ GetRestrictions() {
-                if (!Signature.HasKeywordArgument()) {
-                    return GetSimpleRestriction();
+                if (_extractedDefault) {
+                    return BindingRestrictions.GetInstanceRestriction(_func.Expression, _func.Value);
+                } else if (_needCodeTest) {
+                    return GetSimpleRestriction().Merge(
+                        BindingRestrictions.GetInstanceRestriction(
+                            Expression.Property(
+                                GetFunctionParam(),
+                                "__code__"
+                            ),
+                            _func.Value.__code__
+                        )
+                    );
                 }
 
-                return GetComplexRestriction();
+                return GetSimpleRestriction();
             }
 
             /// <summary>
@@ -389,28 +399,6 @@ private CallSignature Signature {
                 );
             }
 
-            /// <summary>
-            /// Makes the test when we have a keyword argument call or splatting.
-            /// </summary>
-            /// <returns></returns>
-            private BindingRestrictions/*!*/ GetComplexRestriction() {
-                if (_extractedDefault) {
-                    return BindingRestrictions.GetInstanceRestriction(_func.Expression, _func.Value);
-                } else if (_needCodeTest) {
-                    return GetSimpleRestriction().Merge(
-                        BindingRestrictions.GetInstanceRestriction(
-                            Expression.Property(
-                                GetFunctionParam(),
-                                "__code__"
-                            ),
-                            _func.Value.__code__
-                        )
-                    );
-                }
-
-                return GetSimpleRestriction();
-            }
-
             /// <summary>
             /// Gets the array of expressions which correspond to each argument for the function.  These
             /// correspond with the function as it's defined in Python and must be transformed for our
@@ -524,6 +512,8 @@ private bool FinishArguments(Expression[] exprArgs, List<Expression> paramsArgs,
 
                 var keywordOnlyArgumentCount = _func.Value.KeywordOnlyArgumentCount;
 
+                if (_func.Value.NeedsCodeTest) _needCodeTest = true;
+
                 for (int i = normalArgumentCount; i < normalArgumentCount + keywordOnlyArgumentCount; i++) {
                     var argName = _func.Value.ArgNames[i];
 
diff --git a/Src/IronPython/Runtime/Operations/PythonOps.cs b/Src/IronPython/Runtime/Operations/PythonOps.cs
@@ -1492,9 +1492,13 @@ public static object MakeClass(FunctionCode funcCode, Func<CodeContext, CodeCont
                 return PythonType.__new__(parentContext, TypeCache.PythonType, name, bases, vars, selfNames);
             }
 
+            CodeContext classContext = func(parentContext);
+            // If __classcell__ is defined, verify later that it makes all the way to type.__new__
+            var classCell = (ClosureCell?)classContext.Dict.get("__classcell__");
+
             // Prepare classdict
             // TODO: prepared classdict should be used by `func` (PEP 3115)
-            object? classdict = CallPrepare(parentContext, metaclass, name, bases, keywords, func(parentContext).Dict);
+            object? classdict = CallPrepare(parentContext, metaclass, name, bases, keywords, classContext.Dict);
 
             // Dispatch to the metaclass to do class creation and initialization
             // metaclass could be simply a callable, eg:
@@ -1513,6 +1517,13 @@ public static object MakeClass(FunctionCode funcCode, Func<CodeContext, CodeCont
                 keywords ?? MakeEmptyDict()
             );
 
+            if (classCell is not null && classCell.Value == Uninitialized.Instance) {
+                // Python 3.8: RuntimeError
+                Warn(parentContext, PythonExceptions.DeprecationWarning,
+                    "__class__ not set defining '{0}' as {1}. Was __classcell__ propagated to type.__new__?", name, Repr(parentContext, obj));
+                classCell.Value = obj;
+            }
+
             // Ensure the class derives from `object`
             if (obj is PythonType newType && newType.BaseTypes.Count == 0) {
                 newType.BaseTypes.Add(TypeCache.Object);
diff --git a/Src/IronPython/Runtime/PythonContext.cs b/Src/IronPython/Runtime/PythonContext.cs
@@ -220,7 +220,7 @@ public PythonContext(ScriptDomainManager/*!*/ manager, IDictionary<string, objec
                 // Can be null if called from unmanaged code (VS integration scenario)
                 // or in self-contained single file scenarios
                 if (entry != null) {
-                    string lib = Path.Combine(entry, "Lib");
+                    string lib = Path.Combine(entry, "lib");
                     path.append(lib);
 
                     // add DLLs directory for user-defined extention modules
@@ -1768,7 +1768,7 @@ internal void SetHostVariables(string prefix, string executable, string versionS
             _initialExecutable = executable ?? "";
             InitialPrefix = prefix;
 
-            AddToPath(Path.Combine(prefix, "Lib"), 0);
+            AddToPath(Path.Combine(prefix, "lib"), 0);
 
             SetHostVariables(SystemState.__dict__);
         }
diff --git a/Src/IronPython/Runtime/PythonFunction.cs b/Src/IronPython/Runtime/PythonFunction.cs
@@ -33,6 +33,8 @@ public sealed partial class PythonFunction : PythonTypeSlot, IWeakReferenceable,
         public readonly MutableTuple Closure;
 
         private object[]/*!*/ _defaults;                // the default parameters of the method
+        private PythonDictionary _kwdefaults;           // the keyword-only arguments defaults for the method
+
         internal PythonDictionary _dict;                // a dictionary to story arbitrary members on the function object
         private object _module;                         // the module name
 
@@ -89,7 +91,7 @@ internal PythonFunction(CodeContext/*!*/ context, FunctionCode funcInfo, object
 
             _context = context;
             _defaults = defaults ?? ArrayUtils.EmptyObjects;
-            __kwdefaults__ = kwdefaults;
+            _kwdefaults = kwdefaults;
             _code = funcInfo;
             _doc = funcInfo._initialDoc;
             _name = funcInfo.co_name;
@@ -145,8 +147,13 @@ public PythonTuple __defaults__ {
             }
         }
 
-        // TODO: update CalculatedCachedCompat?
-        public PythonDictionary __kwdefaults__ { get; set; }
+        public PythonDictionary __kwdefaults__ {
+            get => _kwdefaults;
+            set {
+                _kwdefaults = value;
+                FunctionCompatibility = CalculatedCachedCompat();
+            }
+        }
 
         [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly")]
         public object __closure__ {
@@ -277,6 +284,14 @@ internal string GetSignatureString() {
         /// </summary>
         internal int FunctionCompatibility { get; private set; }
 
+        internal bool NeedsCodeTest {
+            get {
+                return NormalArgumentCount > 0x3ff
+                    || Defaults.Length > 0x3ff
+                    || KeywordOnlyArgumentCount > 0x1ff;
+            }
+        }
+
         /// <summary>
         /// Calculates the _compat value which is used for call-compatibility checks
         /// for simple calls.  Whenver any of the dependent values are updated this
@@ -289,20 +304,20 @@ internal string GetSignatureString() {
         ///     expand dict/list - based on nparams and flags, both read-only
         ///     
         /// Bits are allocated as:
-        ///     000000ff - Normal argument count
-        ///     0000ff00 - Default count
-        ///     007f0000 - Keyword-only argument count
-        ///     3f800000 - Keyword-only defaults count
+        ///     000003ff - Normal argument count
+        ///     000ffc00 - Default count
+        ///     1ff00000 - Keyword-only argument count
+        ///     20000000 - has keyword-only defaults
         ///     40000000 - expand list
         ///     80000000 - expand dict
         ///     
         /// Enforce recursion is added at runtime.
         /// </summary>
         private int CalculatedCachedCompat() {
-            return (NormalArgumentCount) |
-                (Defaults.Length << 8) |
-                (KeywordOnlyArgumentCount << 16) |
-                ((__kwdefaults__?.Count ?? 0) << 23) |
+            return NormalArgumentCount |
+                (Defaults.Length << 10) |
+                (KeywordOnlyArgumentCount << 20) |
+                (__kwdefaults__ is not null ? 0x20000000 : 0) |
                 ((ExpandListPosition != -1) ? 0x40000000 : 0) |
                 ((ExpandDictPosition != -1) ? unchecked((int)0x80000000) : 0);
         }
diff --git a/Src/IronPythonTest/Cases/CaseExecuter.cs b/Src/IronPythonTest/Cases/CaseExecuter.cs
@@ -148,15 +148,15 @@ private static string GetIronPythonPath() {
             return string.Empty;
         }
 
-        private string ReplaceVariables(string input, IDictionary<string, string> replacements) {
+        private string ReplaceVariables(string input, Dictionary<string, string> replacements) {
             Regex variableRegex = new Regex(@"\$\(([^}]+)\)", RegexOptions.Compiled);
 
             var result = input;
             var match = variableRegex.Match(input);
             while (match.Success) {
                 var variable = match.Groups[1].Value;
-                if (replacements.ContainsKey(variable)) {
-                    result = result.Replace(match.Groups[0].Value, replacements[variable]);
+                if (replacements.TryGetValue(variable, out var replacement)) {
+                    result = result.Replace(match.Groups[0].Value, replacement);
                 }
                 match = match.NextMatch();
             }
diff --git a/Src/Scripts/Install-IronPython.ps1 b/Src/Scripts/Install-IronPython.ps1
diff --git a/Src/StdLib/Lib/test/test_functools.py b/Src/StdLib/Lib/test/test_functools.py
diff --git a/Src/StdLib/StdLib.pyproj b/Src/StdLib/StdLib.pyproj
diff --git a/Tests/modules/network_related/test__ssl.py b/Tests/modules/network_related/test__ssl.py
diff --git a/Tests/test_class.py b/Tests/test_class.py
diff --git a/Tests/test_regressions.py b/Tests/test_regressions.py
