Merge branch 'master' into 3.6

Author: slozier

Src/IronPython/Lib/iptest/__init__.py

@@ -2,7 +2,7 @@
 # The .NET Foundation licenses this file to you under the Apache 2.0 License.
 # See the LICENSE file in the project root for more information.
 
-from .ipunittest import IronPythonTestCase, stdout_trapper, stderr_trapper, path_modifier, retryOnFailure, run_test, skipUnlessIronPython, source_root
+from .ipunittest import IronPythonTestCase, stdout_trapper, stderr_trapper, path_modifier, retryOnFailure, run_test, skipUnlessIronPython, source_root, expectedFailureIf
 from .test_env import *
 from .type_util import *
 from .misc_util import ip_supported_encodings

Src/IronPython/Lib/iptest/ipunittest.py

@@ -53,6 +53,15 @@ def skipUnlessIronPython():
     """Skips the test unless currently running on IronPython"""
     return unittest.skipUnless(is_cli, 'IronPython specific test')
 
+def expectedFailureIf(condition):
+    """The test is marked as an expectedFailure if the condition is satisfied."""
+    def wrapper(func):
+        if condition:
+            return unittest.expectedFailure(func)
+        else:
+            return func
+    return wrapper
+
 MAX_FAILURE_RETRY = 3
 def retryOnFailure(f, times=MAX_FAILURE_RETRY, *args, **kwargs):
     '''

Src/IronPython/Runtime/Exceptions/PythonExceptions.cs

@@ -129,7 +129,7 @@ public partial class _OSError {
                             errno = WinErrorToErrno(winerror);
                         }
                     }
-                    cls = ErrnoToPythonType((Error)errno);
+                    cls = ErrnoToPythonType(ErrnoToErrorEnum(errno));
                 }
                 return Activator.CreateInstance(cls.UnderlyingSystemType, cls);
             }
@@ -183,25 +183,26 @@ public override void __init__(params object[] args) {
             }
 
             private enum Error {
+                UNSPECIFIED = -1,
                 EPERM = 1,
                 ENOENT = 2,
                 ESRCH = 3,
                 EINTR = 4,
                 ECHILD = 10,
-                EAGAIN = 11,
+                EAGAIN = 11, // 35 on OSX
                 EACCES = 13,
                 EEXIST = 17,
                 ENOTDIR = 20,
                 EISDIR = 21,
                 EPIPE = 32,
                 // Linux
-                ECONNABORTED = 103,
-                ECONNRESET = 104,
-                ESHUTDOWN = 108,
-                ETIMEDOUT = 110,
-                ECONNREFUSED = 111,
-                EALREADY = 114,
-                EINPROGRESS = 115,
+                ECONNABORTED = 103, // 53 on OSX
+                ECONNRESET = 104, // 54 on OSX
+                ESHUTDOWN = 108, // 58 on OSX
+                ETIMEDOUT = 110, // 60 on OSX
+                ECONNREFUSED = 111, // 61 on OSX
+                EALREADY = 114, // 37 on OSX
+                EINPROGRESS = 115, // 36 on OSX
                 // Windows
                 WSAEWOULDBLOCK = 10035,
                 WSAEINPROGRESS = 10036,
@@ -213,6 +214,14 @@ private enum Error {
                 WSAECONNREFUSED = 10061,
             }
 
+            private static Error ErrnoToErrorEnum(int errno) {
+                if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) {
+                    if (errno == 11) return Error.UNSPECIFIED; // EAGAIN on Linux/Windows but EDEADLK on OSX, which is not being remapped
+                    if (errno >= 35) errno += 10000; // add WSABASEERR to map to Windows error range
+                }
+                return (Error)errno;
+            }
+
             private static PythonType ErrnoToPythonType(Error errno) {
                 var res = errno switch
                 {
@@ -229,10 +238,10 @@ private static PythonType ErrnoToPythonType(Error errno) {
                     Error.EPIPE => BrokenPipeError,
                     _ => null
                 };
-                if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) {
+                if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) {
                     res ??= errno switch
                     {
-                        // Windows
+                        // Windows or remapped OSX
                         Error.WSAEWOULDBLOCK => BlockingIOError,
                         Error.WSAEINPROGRESS => BlockingIOError,
                         Error.WSAEALREADY => BlockingIOError,
@@ -243,8 +252,7 @@ private static PythonType ErrnoToPythonType(Error errno) {
                         Error.WSAECONNREFUSED => ConnectionRefusedError,
                         _ => null
                     };
-                } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) {
-                    // TODO: verify that these are the same on OSX
+                } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) {
                     res ??= errno switch
                     {
                         // Linux

Src/IronPython/Runtime/FormattingHelper.cs

@@ -1,62 +1,46 @@
-/* ****************************************************************************
- *
- * Copyright (c) Microsoft Corporation.
- *
- * This source code is subject to terms and conditions of the Apache License, Version 2.0. A
- * copy of the license can be found in the License.html file at the root of this distribution. If
- * you cannot locate the  Apache License, Version 2.0, please send an email to
- * dlr@microsoft.com. By using this source code in any fashion, you are agreeing to be bound
- * by the terms of the Apache License, Version 2.0.
- *
- * You must not remove this notice, or any other, from this software.
- *
- *
- * ***************************************************************************/
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information.
 
 using System;
-using System.Collections.Generic;
 using System.Globalization;
-using System.Linq;
 using System.Text;
 using System.Threading;
 
-namespace IronPython.Runtime
-{
-    internal static class FormattingHelper
-    {
-        private static NumberFormatInfo _invariantCommaSeperatorInfo;
+namespace IronPython.Runtime {
+    internal static class FormattingHelper {
+        private static NumberFormatInfo _invariantUnderscoreSeperatorInfo;
 
         /// <summary>
         /// Helper NumberFormatInfo for use by int/BigInteger __format__ routines
-        /// for width specified leading zero support that contains ','s every 3 digits.
+        /// for width specified leading zero support that contains '_'s every 3 digits.
         /// i.e. For use by d/g/G format specifiers. NOT for use by n format specifiers.
         /// </summary>
-        public static NumberFormatInfo InvariantCommaNumberInfo {
+        public static NumberFormatInfo InvariantUnderscoreNumberInfo {
             get {
-                if (_invariantCommaSeperatorInfo == null) {
+                if (_invariantUnderscoreSeperatorInfo == null) {
                     Interlocked.CompareExchange(
-                        ref _invariantCommaSeperatorInfo,
-                        new NumberFormatInfo()
-                        {
-                            NumberGroupSeparator = ",",
+                        ref _invariantUnderscoreSeperatorInfo,
+                        new NumberFormatInfo() {
+                            NumberGroupSeparator = "_",
                             NumberDecimalSeparator = ".",
-                            NumberGroupSizes = new int[] {3}
+                            NumberGroupSizes = new int[] { 3 }
                         },
                         null
                     );
                 }
-                return _invariantCommaSeperatorInfo;
+                return _invariantUnderscoreSeperatorInfo;
             }
         }
 
-        public static string/*!*/ ToCultureString<T>(T/*!*/ val, NumberFormatInfo/*!*/ nfi, StringFormatSpec spec) {
+        public static string/*!*/ ToCultureString<T>(T/*!*/ val, NumberFormatInfo/*!*/ nfi, StringFormatSpec spec, int? overrideWidth = null) {
             string separator = nfi.NumberGroupSeparator;
             int[] separatorLocations = nfi.NumberGroupSizes;
             string digits = val.ToString();
 
             // If we're adding leading zeros, we need to know how
             // many we need.
-            int width = spec.Width ?? 0;
+            int width = overrideWidth ?? spec.Width ?? 0;
             int fillerLength = Math.Max(width - digits.Length, 0);
             bool addLeadingZeros = (spec.Fill ?? '\0') == '0' && width > digits.Length;
             int beginningOfDigits = fillerLength;
@@ -127,12 +111,10 @@ public static NumberFormatInfo InvariantCommaNumberInfo {
                                     break;
                                 }
                             }
-                        }
-                        else {
+                        } else {
                             res.Remove(0, beginningOfMaximumWidth);
                         }
-                    }
-                    else {
+                    } else {
                         // If we ran out of remainingWidth just formatting
                         // the actual digits, then remove any extra leading zeros
                         // we added.

Src/IronPython/Runtime/NewStringFormatter.cs

@@ -1,24 +1,11 @@
-/* ****************************************************************************
- *
- * Copyright (c) Microsoft Corporation.
- *
- * This source code is subject to terms and conditions of the Apache License, Version 2.0. A
- * copy of the license can be found in the License.html file at the root of this distribution. If
- * you cannot locate the  Apache License, Version 2.0, please send an email to
- * dlr@microsoft.com. By using this source code in any fashion, you are agreeing to be bound
- * by the terms of the Apache License, Version 2.0.
- *
- * You must not remove this notice, or any other, from this software.
- *
- *
- * ***************************************************************************/
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information.
 
 using System;
 using System.Collections.Generic;
 using System.Text;
-using System.Diagnostics;
 
-using Microsoft.Scripting;
 using Microsoft.Scripting.Runtime;
 using Microsoft.Scripting.Utils;
 

Src/IronPython/Runtime/Operations/BigIntegerOps.cs

@@ -742,10 +742,8 @@ public static BigInteger ToBigInteger(BigInteger self) {
                 throw PythonOps.ValueError("Precision not allowed in integer format specifier");
             }
 
-            BigInteger val = self;
-            if (self < 0) {
-                val = -self;
-            }
+            BigInteger val = self.Abs();
+
             string digits;
 
             switch (spec.Type) {
@@ -758,84 +756,50 @@ public static BigInteger ToBigInteger(BigInteger self) {
                         goto case 'd';
                     }
 
-                    digits = FormattingHelper.ToCultureString(val, context.LanguageContext.NumericCulture.NumberFormat, spec);
+                    if (spec.Fill == '0' && spec.Width > 1) {
+                        digits = FormattingHelper.ToCultureString(val, culture.NumberFormat, spec, (spec.Sign != null && spec.Sign != '-' || self < 0) ? spec.Width - 1 : null);
+                    }
+                    else {
+                        digits = FormattingHelper.ToCultureString(val, culture.NumberFormat, spec);
+                    }
                     break;
                 case null:
                 case 'd':
-                    if (spec.ThousandsComma) {
-                        var width = spec.Width ?? 0;
+                    if (spec.ThousandsComma || spec.ThousandsUnderscore) {
+                        var numberFormat = spec.ThousandsUnderscore ? FormattingHelper.InvariantUnderscoreNumberInfo : CultureInfo.InvariantCulture.NumberFormat;
+
                         // If we're inserting commas, and we're padding with leading zeros.
                         // AlignNumericText won't know where to place the commas,
-                        // so force .Net to help us out here.
-                        if (spec.Fill.HasValue && spec.Fill.Value == '0' && width > 1) {
-                            digits = val.ToString(FormattingHelper.ToCultureString(self, FormattingHelper.InvariantCommaNumberInfo, spec));
+                        // so use FormattingHelper.ToCultureString for that support.
+                        if (spec.Fill == '0' && spec.Width > 1) {
+                            digits = FormattingHelper.ToCultureString(val, numberFormat, spec, (spec.Sign != null && spec.Sign != '-' || self < 0) ? spec.Width - 1 : null);
+                        } else {
+                            digits = val.ToString("#,0", numberFormat);
                         }
-                        else {
-                        digits = val.ToString("#,0", CultureInfo.InvariantCulture);
-                        }
-                    }
-                    else {
+                    } else {
                         digits = val.ToString("D", CultureInfo.InvariantCulture);
                     }
                     break;
                 case '%':
-                    if (spec.ThousandsComma) {
-                        digits = val.ToString("#,0.000000%", CultureInfo.InvariantCulture);
-                    } else {
-                        digits = val.ToString("0.000000%", CultureInfo.InvariantCulture);
-                    }
-                    break;
                 case 'e':
-                    if (spec.ThousandsComma) {
-                        digits = val.ToString("#,0.000000e+00", CultureInfo.InvariantCulture);
-                    } else {
-                        digits = val.ToString("0.000000e+00", CultureInfo.InvariantCulture);
-                    }
-                    break;
                 case 'E':
-                    if (spec.ThousandsComma) {
-                        digits = val.ToString("#,0.000000E+00", CultureInfo.InvariantCulture);
-                    } else {
-                        digits = val.ToString("0.000000E+00", CultureInfo.InvariantCulture);
-                    }
-                    break;
                 case 'f':
                 case 'F':
-                    if (spec.ThousandsComma) {
-                        digits = val.ToString("#,########0.000000", CultureInfo.InvariantCulture);
-                    } else {
-                        digits = val.ToString("#########0.000000", CultureInfo.InvariantCulture);
-                    }
-                    break;
                 case 'g':
-                    if (val >= 1000000) {
-                        digits = val.ToString("0.#####e+00", CultureInfo.InvariantCulture);
-                    } else if (spec.ThousandsComma) {
-                        goto case 'd';
-                    } else {
-                        digits = val.ToString(CultureInfo.InvariantCulture);
-                    }
-                    break;
                 case 'G':
-                    if (val >= 1000000) {
-                        digits = val.ToString("0.#####E+00", CultureInfo.InvariantCulture);
-                    } else if (spec.ThousandsComma) {
-                        goto case 'd';
-                    } else {
-                        digits = val.ToString(CultureInfo.InvariantCulture);
-                    }
+                    digits = DoubleOps.DoubleToFormatString(context, ToDouble(val), spec);
                     break;
                 case 'X':
-                    digits = AbsToHex(val, false);
+                    digits = AbsToHex(val, lowercase: false);
                     break;
                 case 'x':
-                    digits = AbsToHex(val, true);
+                    digits = AbsToHex(val, lowercase: true);
                     break;
                 case 'o': // octal
-                    digits = ToOctal(val, true);
+                    digits = ToOctal(val, lowercase: true);
                     break;
                 case 'b': // binary
-                    digits = ToBinary(val, false, true);
+                    digits = ToBinary(val, includeType: false, lowercase: true);
                     break;
                 case 'c': // single char
                     int iVal;

Src/IronPython/Runtime/Operations/FloatOps.cs

@@ -813,7 +813,7 @@ public static string __format__(CodeContext/*!*/ context, double self, [NotNone]
         /// <summary>
         /// Returns the digits for the format spec, no sign is included.
         /// </summary>
-        private static string DoubleToFormatString(CodeContext/*!*/ context, double self, StringFormatSpec/*!*/ spec) {
+        internal static string DoubleToFormatString(CodeContext/*!*/ context, double self, StringFormatSpec/*!*/ spec) {
             self = Math.Abs(self);
             const int DefaultPrecision = 6;
             int precision = spec.Precision ?? DefaultPrecision;