Re-enable some tests (#1457)

* Re-enable test_itertools_stdlib

* Re-enable test_sys_stdlib

* Re-enable test_codecs_stdlib

* Re-enable test_dict_stdlib

* Re-enable test_httplib_stdlib

* Re-enable test_float_stdlib

* Re-enable test_struct_stdlib

* Re-enable test_datetime_stdlib

* Re-enable test_strptime_stdlib

* Re-enable test_functools_stdlib

* Fix failing tests

* Disable failing tests

Author: slozier

Src/IronPython.Modules/IterTools.cs

@@ -168,9 +168,23 @@ public PythonTuple __reduce__() {
             }
 
             public void __setstate__(PythonTuple state) {
-                // TODO: error handling?
-                ie = state[0] as IEnumerator;
-                inner = (state.Count > 1) ? state[1] as IEnumerator : null;
+                IEnumerator iter;
+                IEnumerator innerIter;
+                switch (state.Count) {
+                    case 0: throw PythonOps.TypeError("function takes at least 1 argument (0 given)");
+                    case 1:
+                        iter = state[0] as IEnumerator ?? throw PythonOps.TypeError("Arguments must be iterators.");
+                        innerIter = null;
+                        break;
+                    case 2:
+                        iter = state[0] as IEnumerator ?? throw PythonOps.TypeError("Arguments must be iterators.");
+                        innerIter = state[1] as IEnumerator ?? throw PythonOps.TypeError("Arguments must be iterators.");
+                        break;
+                    default:
+                        throw PythonOps.TypeError("function takes at most 2 argument ({0} given)", state.Count);
+                }
+                ie = iter;
+                inner = innerIter;
                 InnerEnumerator = LazyYielder();
             }
 
@@ -515,11 +529,14 @@ public void __setstate__(object state) {
             private IEnumerator<object> Yielder(IEnumerator iter) {
                 object curKey = _starterKey;
                 if (MoveNextHelper(iter)) {
+                    curKey = GetKey(iter.Current);
+                    yield return PythonTuple.MakeTuple(curKey, Grouper(iter, curKey));
+
                     while (!_fFinished) {
                         while (PythonContext.Equal(GetKey(iter.Current), curKey)) {
-                            if (!MoveNextHelper(iter)) { 
-                                _fFinished = true; 
-                                yield break; 
+                            if (!MoveNextHelper(iter)) {
+                                _fFinished = true;
+                                yield break;
                             }
                         }
                         curKey = GetKey(iter.Current);

Src/IronPython.Modules/_datetime.cs

@@ -1301,15 +1301,6 @@ public PythonTuple __reduce__() {
                 );
             }
 
-            // TODO: get rid of __bool__ in 3.5
-            public bool __bool__() {
-                return this.UtcTime.TimeSpan.Ticks != 0 || this.UtcTime.LostMicroseconds != 0;
-            }
-
-            public static explicit operator bool(time time) {
-                return time.__bool__();
-            }
-
             // instance methods
             public object replace() {
                 return this;

Src/IronPython.Modules/_socket.cs

@@ -135,14 +135,19 @@ public void __init__(CodeContext/*!*/ context, int family = DefaultAddressFamily
                     throw MakeException(context, new SocketException((int)SocketError.ProtocolNotSupported));
                 }
 
-                Socket? socket;
+                Socket? socket = null;
                 if (fileno is socket sock) {
                     socket = sock._socket;
                     _hostName = sock._hostName;
                     // we now own the lifetime of the socket
                     GC.SuppressFinalize(sock);
-                } else if (fileno != null && (socket = HandleToSocket((long)fileno)) != null) {
-                    // nothing to do here
+                } else if (fileno != null) {
+                    if (Converter.TryConvertToInt64(fileno, out long l)) {
+                        socket = HandleToSocket(l);
+                    }
+                    if (socket is null) {
+                        throw PythonOps.OSError("Bad file descriptor");
+                    }
                 } else {
                     try {
                         socket = new Socket(addressFamily, socketType, protocolType);

Src/IronPython.Modules/_struct.cs

@@ -60,7 +60,7 @@ public Struct(params object[] args) {
             }
 
             [Documentation("creates a new uninitialized struct object - all arguments are ignored")]
-            public Struct([ParamDictionary]IDictionary<object, object> kwArgs, params object[] args) {
+            public Struct([ParamDictionary] IDictionary<object, object> kwArgs, params object[] args) {
             }
 
             [Documentation("initializes or re-initializes the compiled struct object with a new format")]
@@ -191,9 +191,22 @@ public void __init__(CodeContext/*!*/ context, object fmt) {
                             break;
                         case FormatType.Float:
                             for (int j = 0; j < curFormat.Count; j++) {
-                                WriteFloat(res, _isLittleEndian, (float)GetDoubleValue(context, curObj++, values));
+                                var d = GetDoubleValue(context, curObj++, values);
+                                var val = (float)d;
+                                if (float.IsInfinity(val) && !double.IsInfinity(d)) throw PythonOps.OverflowError("float too large to pack with f format");
+                                WriteFloat(res, _isLittleEndian, val);
                             }
                             break;
+#if NET6_0_OR_GREATER
+                        case FormatType.Half:
+                            for (int j = 0; j < curFormat.Count; j++) {
+                                var d = GetDoubleValue(context, curObj++, values);
+                                var val = (Half)d;
+                                if (Half.IsInfinity(val) && !double.IsInfinity(d)) throw PythonOps.OverflowError("float too large to pack with e format");
+                                WriteHalf(res, _isLittleEndian, val);
+                            }
+                            break;
+#endif
                         case FormatType.CString:
                             WriteString(res, curFormat.Count, GetStringValue(context, curObj++, values));
                             break;
@@ -274,7 +287,7 @@ public void pack_into(CodeContext/*!*/ context, [NotNone] ByteArray/*!*/ buffer,
                             break;
                         case FormatType.SignedChar:
                             for (int j = 0; j < curFormat.Count; j++) {
-                                res[res_idx++] = (int)(sbyte)CreateCharValue(context, ref curIndex, data);
+                                res[res_idx++] = (int)unchecked((sbyte)CreateCharValue(context, ref curIndex, data));
                             }
                             break;
                         case FormatType.UnsignedChar:
@@ -337,7 +350,7 @@ public void pack_into(CodeContext/*!*/ context, [NotNone] ByteArray/*!*/ buffer,
                             break;
                         case FormatType.SizeT:
                             for (int j = 0; j < curFormat.Count; j++) {
-                                res[res_idx++] = CreateUIntValue(context, ref curIndex, _isLittleEndian, data);
+                                res[res_idx++] = BigIntegerOps.__int__(CreateUIntValue(context, ref curIndex, _isLittleEndian, data));
                             }
                             break;
                         case FormatType.LongLong:
@@ -350,9 +363,16 @@ public void pack_into(CodeContext/*!*/ context, [NotNone] ByteArray/*!*/ buffer,
                                 res[res_idx++] = BigIntegerOps.__int__(CreateULongValue(context, ref curIndex, _isLittleEndian, data));
                             }
                             break;
+#if NET6_0_OR_GREATER
+                        case FormatType.Half:
+                            for (int j = 0; j < curFormat.Count; j++) {
+                                res[res_idx++] = (double)CreateHalfValue(context, ref curIndex, _isLittleEndian, data);
+                            }
+                            break;
+#endif
                         case FormatType.Float:
                             for (int j = 0; j < curFormat.Count; j++) {
-                                res[res_idx++] = CreateFloatValue(context, ref curIndex, _isLittleEndian, data);
+                                res[res_idx++] = (double)CreateFloatValue(context, ref curIndex, _isLittleEndian, data);
                             }
                             break;
                         case FormatType.Double:
@@ -487,6 +507,12 @@ private static Struct CompileAndCache(CodeContext/*!*/ context, string/*!*/ fmt)
                             res.Add(new Format(FormatType.UnsignedLongLong, count));
                             count = 1;
                             break;
+#if NET6_0_OR_GREATER
+                        case 'e': // half
+                            res.Add(new Format(FormatType.Half, count));
+                            count = 1;
+                            break;
+#endif
                         case 'f': // float
                             res.Add(new Format(FormatType.Float, count));
                             count = 1;
@@ -725,6 +751,9 @@ private enum FormatType {
 
             Short,
             UnsignedShort,
+#if NET6_0_OR_GREATER
+            Half,
+#endif
 
             Int,
             UnsignedInt,
@@ -758,6 +787,9 @@ private static int GetNativeSize(FormatType c) {
                     return 1;
                 case FormatType.Short:
                 case FormatType.UnsignedShort:
+#if NET6_0_OR_GREATER
+                case FormatType.Half:
+#endif
                     return 2;
                 case FormatType.Int:
                 case FormatType.UnsignedInt:
@@ -953,6 +985,18 @@ private static void WriteSignedNetPointer(this MemoryStream res, bool fLittleEnd
             res.WritePointer(fLittleEndian, unchecked((ulong)val.ToInt64()));
         }
 
+#if NET6_0_OR_GREATER
+        private static void WriteHalf(this MemoryStream res, bool fLittleEndian, Half val) {
+            byte[] bytes = BitConverter.GetBytes(val);
+            if (BitConverter.IsLittleEndian == fLittleEndian) {
+                res.Write(bytes, 0, bytes.Length);
+            } else {
+                res.WriteByte(bytes[1]);
+                res.WriteByte(bytes[0]);
+            }
+        }
+#endif
+
         private static void WriteFloat(this MemoryStream res, bool fLittleEndian, float val) {
             byte[] bytes = BitConverter.GetBytes(val);
             if (BitConverter.IsLittleEndian == fLittleEndian) {
@@ -1176,7 +1220,7 @@ internal static ulong GetULongLongValue(CodeContext/*!*/ context, int index, obj
         internal static double GetDoubleValue(CodeContext/*!*/ context, int index, object[] args) {
             object val = GetValue(context, index, args);
             if (Converter.TryConvertToDouble(val, out double res)) return res;
-            throw Error(context, "expected double value");
+            throw Error(context, "required argument is not a float");
         }
 
         internal static IList<byte> GetStringValue(CodeContext/*!*/ context, int index, object[] args) {
@@ -1236,6 +1280,28 @@ internal static ushort CreateUShortValue(CodeContext/*!*/ context, ref int index
             }
         }
 
+#if NET6_0_OR_GREATER
+        internal static Half CreateHalfValue(CodeContext/*!*/ context, ref int index, bool fLittleEndian, IList<byte> data) {
+            byte[] bytes = new byte[2];
+            if (fLittleEndian) {
+                bytes[0] = (byte)ReadData(context, ref index, data);
+                bytes[1] = (byte)ReadData(context, ref index, data);
+            } else {
+                bytes[1] = (byte)ReadData(context, ref index, data);
+                bytes[0] = (byte)ReadData(context, ref index, data);
+            }
+            Half res = BitConverter.ToHalf(bytes, 0);
+
+            if (context.LanguageContext.FloatFormat == FloatFormat.Unknown) {
+                if (Half.IsNaN(res) || Half.IsInfinity(res)) {
+                    throw PythonOps.ValueError("can't unpack IEEE 754 special value on non-IEEE platform");
+                }
+            }
+
+            return res;
+        }
+#endif
+
         internal static float CreateFloatValue(CodeContext/*!*/ context, ref int index, bool fLittleEndian, IList<byte> data) {
             byte[] bytes = new byte[4];
             if (fLittleEndian) {

Src/IronPython.Modules/bz2/dotnetzip/BZip2/BZip2InputStream.cs

@@ -1105,7 +1105,7 @@ private void SetupBlock()
             /* Check: unzftab entries in range. */
             for (i = 0; i <= 255; i++)
             {
-                if (s.unzftab[i] < 0 || s.unzftab[i] > this.last)
+                if (s.unzftab[i] < 0 || s.unzftab[i] > this.last+1)
                     throw new Exception("BZ_DATA_ERROR");
             }
 

Src/IronPython/Runtime/DictionaryOps.cs

@@ -51,7 +51,12 @@ public static class DictionaryOps {
                         buf.Append(PythonOps.Repr(context, kv.Key));
                     buf.Append(": ");
 
-                    buf.Append(PythonOps.Repr(context, kv.Value));
+                    try {
+                        PythonOps.FunctionPushFrame(context.LanguageContext);
+                        buf.Append(PythonOps.Repr(context, kv.Value));
+                    } finally {
+                        PythonOps.FunctionPopFrame();
+                    }
                 }
                 buf.Append("}");
                 return buf.ToString();

Src/IronPython/Runtime/Operations/StringOps.cs

@@ -2367,7 +2367,14 @@ private class BackslashReplaceFallbackBuffer : EncoderFallbackBuffer {
                 private int _index;
 
                 public override bool Fallback(char charUnknownHigh, char charUnknownLow, int index) {
-                    return false;
+                    var val = char.ConvertToUtf32(charUnknownHigh, charUnknownLow);
+                    _buffer.Add('\\');
+                    _buffer.Add('U');
+                    AddCharacter(val >> 24);
+                    AddCharacter((val >> 16) & 0xFF);
+                    AddCharacter((val >> 8) & 0xFF);
+                    AddCharacter(val & 0xFF);
+                    return true;
                 }
 
                 public override bool Fallback(char charUnknown, int index) {

Src/IronPythonTest/Cases/IronPythonCasesManifest.ini

@@ -17,13 +17,17 @@ IsolationLevel=PROCESS # requires System.Drawing assembly to be unloaded
 [IronPython.test_compiler]
 IsolationLevel=PROCESS # TODO: figure out - breaks the sys module in other tests
 
+[IronPython.test_delegate]
+RetryCount=2
+
+[IronPython.test_dict_stdlib]
+IsolationLevel=ENGINE
+MaxRecursion=100
+
 [IronPython.test_dllsite]
 Ignore=true
 Reason=disabled in IronLanguages/main, needs lots of work
 
-[IronPython.test_delegate]
-RetryCount=2
-
 [IronPython.test_doc]
 RunCondition=NOT $(IS_OSX) # TODO: figure out
 IsolationLevel=PROCESS # required to have quit/exit
@@ -179,33 +183,12 @@ Ignore=true # test_doc
 [IronPython.test_builtin_stdlib]
 Ignore=true # multiple failures
 
-[IronPython.test_codecs_stdlib]
-Ignore=true # multiple failures
-
 [IronPython.test_codeccallbacks_stdlib]
 Ignore=true # multiple failures
 
-[IronPython.test_datetime_stdlib]
-Ignore=true # multiple failures
-
-[IronPython.test_dict_stdlib]
-Ignore=true # blocking
-
-[IronPython.test_float_stdlib]
-Ignore=true # 2 failures https://github.com/IronLanguages/ironpython3/issues/105
-
-[IronPython.test_functools_stdlib]
-Ignore=true # StackOverflowException
-
-[IronPython.test_httplib_stdlib]
-Ignore=true # TypeError: Unable to cast object of type 'System.Int32' to type 'System.Int64'.
-
 [IronPython.test_imp]
 Ignore=true # 1 failure
 
-[IronPython.test_itertools_stdlib]
-Ignore=true # 4 failures
-
 [IronPython.test_re_stdlib]
 Ignore=true # AssertionError: SRE module mismatch
 
@@ -215,17 +198,8 @@ Ignore=true # multiple failures
 [IronPython.test_socket_stdlib]
 Ignore=true # blocking
 
-[IronPython.test_strptime_stdlib]
-Ignore=true # multiple failures
-
-[IronPython.test_struct_stdlib]
-Ignore=true # struct.error: bad char in struct format
-
 [IronPython.test_ssl_stdlib]
 Ignore=true # unittest.case.SkipTest: Cannot import name SSLSession
 
-[IronPython.test_sys_stdlib]
-Ignore=true # 3 failures
-
 [IronPython.test_types_stdlib]
 Ignore=true # blocked by https://github.com/IronLanguages/ironpython3/issues/98

Src/StdLib/Lib/test/test_float.py

@@ -620,6 +620,7 @@ def test_float_specials_do_unpack(self):
             struct.unpack(fmt, data)
 
     @support.requires_IEEE_754
+    @support.cpython_only
     def test_serialized_float_rounding(self):
         from _testcapi import FLT_MAX
         self.assertEqual(struct.pack("<f", 3.40282356e38), struct.pack("<f", FLT_MAX))

Tests/modules/misc/test_datetime.py

@@ -985,7 +985,7 @@ def test_time(self):
         #bool
         self.assertTrue(datetime.time(0, 0, 0, 1))
         #CodePlex Work Item 5139
-        self.assertTrue(not (datetime.time(0, 0, 0, 0)))
+        self.assertFalse(not (datetime.time(0, 0, 0, 0))) # __bool__ removed in 3.5
 
         #replace
         x = datetime.time(1, 2, 33, 444)