Enable test_file (#1039)

* Enable test_file

* Fix test on posix

* More cleanup

* Fix test_fd

* Set windows eol

* IOException to OSError

* Don't map EndOfStreamException to IOError

* Fix test failures

* Fix test

* Cleanup

* fix typo

Author: slozier

.gitattributes

@@ -13,6 +13,10 @@
 *.html text
 *.ps1 text
 
+# Explicitly declare files that should not have line endings modified, ever
+Tests/file_with_BOM.txt -text
+Tests/file_without_BOM.txt -text
+
 # Declare files that will always have CRLF line endings on checkout.
 *.bat text eol=crlf
 *.sln text eol=crlf
@@ -23,7 +27,6 @@
 *.vcxproj text eol=crlf
 *.vdproj text eol=crlf
 
-
 # Declare files that will always have LF line endings on checkout
 .gitmodules text eol=lf
 *.sh text eol=lf

Src/IronPython.Modules/nt.cs

@@ -1571,9 +1571,9 @@ public static int WSTOPSIG(int status) {
 
         [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Interoperability", "CA1404:CallGetLastErrorImmediatelyAfterPInvoke")]
         private static Exception ToPythonException(Exception e, string filename = null) {
-            if (e is IPythonAwareException) {
+            // already a Python Exception
+            if (e.GetPythonException() is not null)
                 return e;
-            }
 
             if (e is ArgumentException || e is ArgumentNullException || e is ArgumentTypeException) {
                 // rethrow reasonable exceptions

Src/IronPython/Modules/_fileio.cs

@@ -71,25 +71,10 @@ public FileIO(CodeContext/*!*/ context, int fd, string mode = "r", bool closefd
                     throw PythonOps.ValueError("fd must be >= 0");
                 }
 
-                PythonContext pc = context.LanguageContext;
-
-                _context = pc;
-                switch (StandardizeMode(mode)) {
-                    case "r": this.mode = "rb"; break;
-                    case "w": this.mode = "wb"; break;
-                    case "a": this.mode = "w"; break;
-                    case "r+":
-                    case "+r": this.mode = "rb+"; break;
-                    case "w+":
-                    case "+w": this.mode = "rb+"; break;
-                    case "a+":
-                    case "+a": this.mode = "r+"; break;
-                    default:
-                        BadMode(mode);
-                        break;
-                }
+                _context = context.LanguageContext;
+                this.mode = NormalizeMode(mode, out _);
 
-                object fileObject = pc.FileManager.GetObjectFromId(fd); // OSError here if no such fd
+                object fileObject = _context.FileManager.GetObjectFromId(fd); // OSError here if no such fd
 
                 // FileManager interface guarantees fileObject is
                 // one of these two types
@@ -117,55 +102,38 @@ public FileIO(CodeContext/*!*/ context, string name, string mode = "r", bool clo
                 this.name = name;
                 PlatformAdaptationLayer pal = context.LanguageContext.DomainManager.Platform;
 
-                int flags = 0;
-                switch (StandardizeMode(mode)) {
-                    case "r":
-                        _readStream = _writeStream = OpenFile(context, pal, name, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
-                        this.mode = "rb";
-                        flags |= O_RDONLY;
-                        break;
-                    case "w":
-                        _readStream = _writeStream = OpenFile(context, pal, name, FileMode.Create, FileAccess.Write, FileShare.ReadWrite);
-                        this.mode = "wb";
-                        flags |= O_CREAT | O_TRUNC | O_WRONLY;
-                        break;
-                    case "a":
-                        _readStream = _writeStream = OpenFile(context, pal, name, FileMode.Append, FileAccess.Write, FileShare.ReadWrite);
-                        _readStream.Seek(0L, SeekOrigin.End);
-                        this.mode = "ab";
-                        flags |= O_APPEND | O_CREAT;
-                        break;
-                    case "r+":
-                    case "+r":
-                        _readStream = _writeStream = OpenFile(context, pal, name, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite);
-                        this.mode = "rb+";
-                        flags |= O_RDWR;
-                        break;
-                    case "w+":
-                    case "+w":
-                        _readStream = _writeStream = OpenFile(context, pal, name, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite);
-                        this.mode = "rb+";
-                        flags |= O_CREAT | O_TRUNC | O_RDWR;
-                        break;
-                    case "a+":
-                    case "+a":
-                        _readStream = OpenFile(context, pal, name, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
-                        _writeStream = OpenFile(context, pal, name, FileMode.Append, FileAccess.Write, FileShare.ReadWrite);
-                        _readStream.Seek(0L, SeekOrigin.End);
-                        _writeStream.Seek(0L, SeekOrigin.End);
-                        this.mode = "ab+";
-                        flags |= O_APPEND | O_CREAT | O_RDWR;
-                        break;
-                    default:
-                        BadMode(mode);
-                        break;
-                }
-
-                _closefd = true;
-
                 _context = context.LanguageContext;
-
-                if (opener != null) {
+                this.mode = NormalizeMode(mode, out int flags);
+
+                if (opener is null) {
+                    switch (this.mode) {
+                        case "rb":
+                            _readStream = _writeStream = OpenFile(context, pal, name, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+                            break;
+                        case "wb":
+                            _readStream = _writeStream = OpenFile(context, pal, name, FileMode.Create, FileAccess.Write, FileShare.ReadWrite);
+                            break;
+                        case "ab":
+                            _readStream = _writeStream = OpenFile(context, pal, name, FileMode.Append, FileAccess.Write, FileShare.ReadWrite);
+                            _readStream.Seek(0L, SeekOrigin.End);
+                            break;
+                        case "rb+":
+                            _readStream = _writeStream = OpenFile(context, pal, name, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite);
+                            break;
+                        case "wb+":
+                            _readStream = _writeStream = OpenFile(context, pal, name, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite);
+                            break;
+                        case "ab+":
+                            _readStream = OpenFile(context, pal, name, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+                            _writeStream = OpenFile(context, pal, name, FileMode.Append, FileAccess.Write, FileShare.ReadWrite);
+                            _readStream.Seek(0L, SeekOrigin.End);
+                            _writeStream.Seek(0L, SeekOrigin.End);
+                            break;
+                        default:
+                            throw new InvalidOperationException();
+                    }
+                }
+                else {
                     object fdobj = PythonOps.CallWithContext(context, opener, name, flags);
                     if (fdobj is int fd) {
                         if (fd < 0) {
@@ -183,58 +151,86 @@ public FileIO(CodeContext/*!*/ context, string name, string mode = "r", bool clo
                         throw PythonOps.TypeError("expected integer from opener");
                     }
                 }
+
+                _closefd = true;
             }
 
-            /// <summary>
-            /// Remove all 'b's from mode string to simplify parsing
-            /// </summary>
-            private static string StandardizeMode(string mode) {
-                int index = mode.IndexOf('b');
+            private static string NormalizeMode(string mode, out int flags) {
+                switch (StandardizeMode(mode)) {
+                    case "r":
+                        flags = O_RDONLY;
+                        return "rb";
+                    case "w":
+                        flags = O_CREAT | O_TRUNC | O_WRONLY;
+                        return "wb";
+                    case "a":
+                        flags = O_APPEND | O_CREAT;
+                        return "ab";
+                    case "r+":
+                    case "+r":
+                        flags = O_RDWR;
+                        return "rb+";
+                    case "w+":
+                    case "+w":
+                        flags = O_CREAT | O_TRUNC | O_RDWR;
+                        return "wb+";
+                    case "a+":
+                    case "+a":
+                        flags = O_APPEND | O_CREAT | O_RDWR;
+                        return "ab+";
+                    default:
+                        throw BadMode(mode);
+                }
 
-                if (index == mode.Length - 1) {
-                    mode = mode.Substring(0, index);
-                } else if (index >= 0) {
-                    StringBuilder sb = new StringBuilder(mode.Substring(0, index), mode.Length - 1);
-                    for (int pos = index + 1; pos < mode.Length; pos++) {
-                        if (mode[pos] != 'b') {
-                            sb.Append(mode[pos]);
+                // remove all 'b's from mode string to simplify parsing
+                static string StandardizeMode(string mode) {
+                    int index = mode.IndexOf('b');
+
+                    if (index == mode.Length - 1) {
+                        mode = mode.Substring(0, index);
+                    } else if (index >= 0) {
+                        StringBuilder sb = new StringBuilder(mode.Substring(0, index), mode.Length - 1);
+                        for (int pos = index + 1; pos < mode.Length; pos++) {
+                            if (mode[pos] != 'b') {
+                                sb.Append(mode[pos]);
+                            }
                         }
+                        mode = sb.ToString();
                     }
-                    mode = sb.ToString();
-                }
 
-                return mode;
-            }
+                    return mode;
+                }
 
-            private static void BadMode(string mode) {
-                bool foundMode = false, foundPlus = false;
-                foreach (char c in mode) {
-                    switch (c) {
-                        case 'r':
-                        case 'w':
-                        case 'a':
-                            if (foundMode) {
-                                throw PythonOps.ValueError("Must have exactly one of read/write/append mode");
-                            } else {
-                                foundMode = true;
+                static Exception BadMode(string mode) {
+                    bool foundMode = false, foundPlus = false;
+                    foreach (char c in mode) {
+                        switch (c) {
+                            case 'r':
+                            case 'w':
+                            case 'a':
+                                if (foundMode) {
+                                    return PythonOps.ValueError("Must have exactly one of read/write/append mode");
+                                } else {
+                                    foundMode = true;
+                                    continue;
+                                }
+                            case '+':
+                                if (foundPlus) {
+                                    return PythonOps.ValueError("Must have exactly one of read/write/append mode");
+                                } else {
+                                    foundPlus = true;
+                                    continue;
+                                }
+                            case 'b':
+                                // any number of 'b's is acceptable
                                 continue;
-                            }
-                        case '+':
-                            if (foundPlus) {
-                                throw PythonOps.ValueError("Must have exactly one of read/write/append mode");
-                            } else {
-                                foundPlus = true;
-                                continue;
-                            }
-                        case 'b':
-                            // any number of 'b's is acceptable
-                            continue;
-                        default:
-                            throw PythonOps.ValueError("invalid mode: {0}", mode);
+                            default:
+                                return PythonOps.ValueError("invalid mode: {0}", mode);
+                        }
                     }
-                }
 
-                throw PythonOps.ValueError("Must have exactly one of read/write/append mode");
+                    return PythonOps.ValueError("Must have exactly one of read/write/append mode");
+                }
             }
 
             #endregion

Src/IronPython/Modules/_io.cs

@@ -92,7 +92,7 @@ public void _checkReadable() {
 
             public void _checkReadable(string msg) {
                 if (!readable(context)) {
-                    throw PythonOps.ValueError(msg ?? "File or stream is not readable.");
+                    throw UnsupportedOperationWithMessage(context, msg ?? "File or stream is not readable.");
                 }
             }
 
@@ -112,7 +112,7 @@ public void _checkWritable() {
 
             public void _checkWritable(string msg) {
                 if (!writable(context)) {
-                    throw PythonOps.ValueError(msg ?? "File or stream is not writable.");
+                    throw UnsupportedOperationWithMessage(context, msg ?? "File or stream is not writable.");
                 }
             }
 
@@ -397,6 +397,9 @@ internal Exception UnsupportedOperation(CodeContext/*!*/ context, string attr) {
                 );
             }
 
+            internal Exception UnsupportedOperationWithMessage(CodeContext/*!*/ context, string msg)
+                => PythonExceptions.CreateThrowable((PythonType)context.LanguageContext.GetModuleState(_unsupportedOperationKey), msg);
+
             internal Exception AttributeError(string attrName) {
                 throw PythonOps.AttributeError("'{0}' object has no attribute '{1}'", PythonTypeOps.GetName(this), attrName);
             }
@@ -2452,7 +2455,7 @@ public override BigInteger seek(CodeContext/*!*/ context, BigInteger cookie, [Op
             public override object read(CodeContext/*!*/ context, object length=null) {
                 _checkClosed();
                 if (!readable(context)) {
-                    throw PythonOps.IOError("not readable");
+                    throw UnsupportedOperationWithMessage(context, "not readable");
                 }
 
                 int size = GetInt(length, -1);
@@ -2497,6 +2500,9 @@ public override object read(CodeContext/*!*/ context, object length=null) {
 
             public override object readline(CodeContext/*!*/ context, int limit=-1) {
                 _checkClosed("read from closed file");
+                if (!readable(context)) {
+                    throw UnsupportedOperationWithMessage(context, "not readable");
+                }
 
                 string line = GetDecodedChars();
 
@@ -2635,13 +2641,19 @@ IEnumerator IEnumerable.GetEnumerator() {
 
             #region ICodeFormattable Members
 
+#nullable enable
             public string __repr__(CodeContext/*!*/ context) {
-                if (PythonOps.TryGetBoundAttr(buffer, "name", out object nameObj)) {
-                    return $"<_io.TextIOWrapper name={PythonOps.Repr(context, nameObj)} encoding='{_encoding}'>";
-                }
+                string name = string.Empty;
+                if (PythonOps.TryGetBoundAttr(buffer, "name", out var nameObj))
+                    name = $" name={PythonOps.Repr(context, nameObj)}";
+
+                string mode = string.Empty;
+                if (PythonOps.TryGetBoundAttr(this, "mode", out var modeObj))
+                    mode = $" mode={PythonOps.Repr(context, modeObj)}";
 
-                return $"<_io.TextIOWrapper encoding='{_encoding}'>";
+                return $"<_io.TextIOWrapper{name}{mode} encoding='{_encoding}'>";
             }
+#nullable restore
 
             #endregion
 
@@ -2806,7 +2818,7 @@ public static _IOBase open(
 
             HashSet<char> modes = MakeSet(mode);
             if (modes.Count < mode.Length || !_validModes.IsSupersetOf(modes)) {
-                throw PythonOps.ValueError("invalid mode: {0}", mode);
+                throw PythonOps.ValueError("invalid mode: '{0}'", mode);
             }
 
             bool reading = modes.Contains('r');

Src/IronPython/Runtime/Exceptions/PythonExceptions.cs

@@ -781,22 +781,22 @@ internal static object ToPython(System.Exception/*!*/ clrException) {
         /// Creates a new style Python exception from the .NET exception
         /// </summary>
         private static BaseException/*!*/ ToPythonNewStyle(System.Exception/*!*/ clrException) {
+            // EndOfStreamException is not part of the OSError hierarchy
+            if (clrException is Win32Exception || clrException is IOException && clrException is not EndOfStreamException) {
+                int errorCode = clrException.HResult;
+
+                if ((errorCode & ~0xfff) == unchecked((int)0x80070000)) {
+                    errorCode &= 0xfff;
+                }
+
+                // TODO: can we get filename and such?
+                return CreatePythonThrowable(OSError, errorCode, clrException.Message, null, errorCode);
+            }
+
             BaseException pyExcep;
             if (clrException is InvalidCastException || clrException is ArgumentNullException) {
                 // explicit extra conversions outside the generated hierarchy
                 pyExcep = new BaseException(TypeError);
-            } else if (clrException is Win32Exception) {
-                Win32Exception win32 = (Win32Exception)clrException;
-                int errorCode = win32.ErrorCode;
-
-                pyExcep = new _OSError();
-                if ((errorCode & 0x80070000) == 0x80070000) {
-                    errorCode &= 0xffff;
-                }
-                pyExcep.__init__(errorCode, win32.Message, null, errorCode);
-                return pyExcep;
-            } else if (clrException is DirectoryNotFoundException) {
-                pyExcep = new _OSError(FileNotFoundError);
             } else {
                 // conversions from generated code (in the generated hierarchy)...
                 pyExcep = ToPythonHelper(clrException);