Misc changes (#1384)

* Update mappingproxy

* Improve perf on BufferedReader.readline

* _datetime changes

* Fix CVE-2022-0391

Author: slozier

Src/IronPython.Modules/_datetime.cs

@@ -670,14 +670,21 @@ public object __ne__(object other) {
                 return string.Format("datetime.date({0}, {1}, {2})", _dateTime.Year, _dateTime.Month, _dateTime.Day);
             }
 
-            public virtual string __format__(CodeContext/*!*/ context, string dateFormat){
+            public virtual string __format__(CodeContext/*!*/ context, [NotNull] string dateFormat){
                 if (string.IsNullOrEmpty(dateFormat)) {
                     return PythonOps.ToString(context, this);
                 } else {
                     return strftime(context, dateFormat);
                 }
             }
 
+            // overload to make test_datetime happy
+            public string __format__(CodeContext/*!*/ context, object spec) {
+                if (spec is string s) return __format__(context, s);
+                if (spec is Extensible<string> es) return __format__(context, es.Value);
+                throw PythonOps.TypeError("__format__() argument 1 must be str, not {0}", PythonOps.GetPythonTypeName(spec));
+            }
+
             #endregion
         }
 
@@ -1294,6 +1301,7 @@ public PythonTuple __reduce__() {
                 );
             }
 
+            // TODO: get rid of __bool__ in 3.5
             public bool __bool__() {
                 return this.UtcTime.TimeSpan.Ticks != 0 || this.UtcTime.LostMicroseconds != 0;
             }
@@ -1474,7 +1482,7 @@ private int CompareTo(object other) {
 
             #endregion
 
-            public object __format__(CodeContext/*!*/ context, string dateFormat) {
+            public object __format__(CodeContext/*!*/ context, [NotNull] string dateFormat) {
                 if (string.IsNullOrEmpty(dateFormat)) {
                     return PythonOps.ToString(context, this);
                 }
@@ -1490,6 +1498,13 @@ public object __format__(CodeContext/*!*/ context, string dateFormat) {
                 }
             }
 
+            // overload to make test_datetime happy
+            public object __format__(CodeContext/*!*/ context, object spec) {
+                if (spec is string s) return __format__(context, s);
+                if (spec is Extensible<string> es) return __format__(context, es.Value);
+                throw PythonOps.TypeError("__format__() argument 1 must be str, not {0}", PythonOps.GetPythonTypeName(spec));
+            }
+
             private class UnifiedTime {
                 public TimeSpan TimeSpan;
                 public int LostMicroseconds;

Src/IronPython/Modules/_io.cs

@@ -886,6 +886,79 @@ public override Bytes read1(CodeContext/*!*/ context, int length=0) {
                 }
             }
 
+            public override object readline(CodeContext context, int limit) {
+                _checkClosed();
+
+                if (limit == 0) {
+                    return Bytes.Empty;
+                }
+
+                lock (this) {
+                    bool limited = limit > 0;
+
+                    List<Bytes> chunks = null;
+                    int cnt = 0;
+                    while (true) {
+                        var buf = _readBuf.AsSpan().Slice(_readBufPos);
+                        if (buf.Length > 0) {
+                            // we hit the limit so we're done
+                            bool done = false;
+                            if (limited && buf.Length > limit - cnt) {
+                                buf = buf.Slice(0, limit - cnt);
+                                done = true;
+                            }
+
+                            // we found the eol so we're done
+                            var idx = buf.IndexOf((byte)'\n');
+                            if (idx != -1) {
+                                buf = buf.Slice(0, idx + 1);
+                                done = true;
+                            }
+
+                            if (done) {
+                                _readBufPos += buf.Length;
+                                if (_readBufPos == _readBuf.Count) {
+                                    ResetReadBuf();
+                                }
+                                var bytes = Bytes.Make(buf.ToArray());
+                                if (chunks is null) {
+                                    return bytes;
+                                }
+                                chunks.Add(bytes);
+                                cnt += buf.Length;
+                                return Bytes.Concat(chunks, cnt);
+                            }
+
+                            (chunks ??= new List<Bytes>()).Add(ResetReadBuf());
+                            cnt += buf.Length;
+                        }
+
+                        // end of file
+                        if (!TryReadNextChunk(context)) {
+                            if (chunks is null) {
+                                return Bytes.Empty;
+                            }
+                            Debug.Assert(cnt > 0);
+                            return Bytes.Concat(chunks, cnt);
+                        }
+                    }
+                }
+
+                bool TryReadNextChunk(CodeContext context) {
+                    object chunkObj;
+                    if (_rawIO != null) {
+                        chunkObj = _rawIO.read(context, _bufSize);
+                    } else {
+                        chunkObj = PythonOps.Invoke(context, _raw, "read", _bufSize);
+                    }
+
+                    Bytes chunk = chunkObj != null ? GetBytes(chunkObj, "read()") : Bytes.Empty;
+
+                    _readBuf = chunk;
+                    return chunk.Count != 0;
+                }
+            }
+
             public override BigInteger tell(CodeContext/*!*/ context) {
                 BigInteger res = _rawIO != null ?
                     _rawIO.tell(context) :

Src/IronPython/Runtime/Types/MappingProxy.cs

@@ -2,6 +2,12 @@
 // The .NET Foundation licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information.
 
+#nullable enable
+
+#if NETCOREAPP3_1 // IDictionary<object?, object?> is incorrectly annotated with TKey : notnull
+#pragma warning disable CS8714 // The type cannot be used as type parameter in the generic type or method. Nullability of type argument doesn't match 'notnull' constraint.
+#endif
+
 using System;
 using System.Collections;
 using System.Collections.Generic;
@@ -13,36 +19,52 @@
 
 namespace IronPython.Runtime.Types {
     [PythonType("mappingproxy")]
-    public sealed class MappingProxy : IDictionary<object, object>, IDictionary {
-        internal PythonDictionary GetDictionary(CodeContext context) => dictionary ?? type.GetMemberDictionary(context, false);
+    public sealed class MappingProxy : IDictionary<object?, object?>, IDictionary {
+        internal PythonDictionary GetDictionary(CodeContext context) => dictionary ?? type!.GetMemberDictionary(context, false);
 
-        private readonly PythonDictionary dictionary;
-        private readonly PythonType type;
+        private readonly PythonDictionary? dictionary;
+        private readonly PythonType? type;
 
         internal MappingProxy(CodeContext context, PythonType/*!*/ dt) {
             Debug.Assert(dt != null);
             type = dt;
         }
 
-        public MappingProxy([NotNull]PythonDictionary dict) {
+        public MappingProxy([NotNull] PythonDictionary dict) {
             dictionary = dict;
         }
 
         #region Python Public API Surface
 
         public int __len__(CodeContext context) => GetDictionary(context).Count;
 
-        public bool __contains__(CodeContext/*!*/ context, object value) => GetDictionary(context).TryGetValue(value, out _);
+        public bool __contains__(CodeContext/*!*/ context, object? value) => GetDictionary(context).TryGetValue(value, out _);
 
         public string/*!*/ __str__(CodeContext/*!*/ context) => DictionaryOps.__repr__(context, this);
 
-        public object get(CodeContext/*!*/ context, [NotNull]object k, object d=null) {
-            object res;
-            if (!GetDictionary(context).TryGetValue(k, out res)) {
-                res = d;
+        public string __repr__(CodeContext/*!*/ context) {
+            var dict = GetDictionary(context);
+            List<object>? infinite = PythonOps.GetAndCheckInfinite(this);
+            if (infinite == null) {
+                return "mappingproxy({...})";
             }
 
-            return res;
+            int infiniteIndex = infinite.Count;
+            infinite.Add(this);
+            try {
+                return $"mappingproxy({PythonOps.Repr(context, dict)})";
+            } finally {
+                System.Diagnostics.Debug.Assert(infiniteIndex == infinite.Count - 1);
+                infinite.RemoveAt(infiniteIndex);
+            }
+        }
+
+        public object? get(CodeContext/*!*/ context, object? k, object? d = null) {
+            if (GetDictionary(context).TryGetValue(k, out object? res)) {
+                return res;
+            }
+
+            return d;
         }
 
         public object keys(CodeContext context) {
@@ -66,11 +88,11 @@ public object items(CodeContext context) {
             return items;
         }
 
-        public PythonDictionary copy(CodeContext/*!*/ context) => new PythonDictionary(context, this);
+        public PythonDictionary copy(CodeContext/*!*/ context) => GetDictionary(context).copy(context);
 
         public const object __hash__ = null;
 
-        public object __eq__(CodeContext/*!*/ context, object other) {
+        public object __eq__(CodeContext/*!*/ context, object? other) {
             if (other is MappingProxy proxy) {
                 if (type == null) {
                     return __eq__(context, proxy.GetDictionary(context));
@@ -90,7 +112,7 @@ public object __eq__(CodeContext/*!*/ context, object other) {
 
         #region IDictionary Members
 
-        public object this[object key] {
+        public object? this[object? key] {
             get => GetDictionary(DefaultContext.Default)[key];
             [PythonHidden]
             set => throw PythonOps.TypeError("'mappingproxy' object does not support item assignment");
@@ -109,7 +131,7 @@ public object this[object key] {
         #region IDictionary Members
 
         [PythonHidden]
-        public void Add(object key, object value) {
+        public void Add(object? key, object? value) {
             this[key] = value;
         }
 
@@ -149,11 +171,7 @@ ICollection IDictionary.Values {
 
         #region ICollection Members
 
-        void ICollection.CopyTo(Array array, int index) {
-            foreach (DictionaryEntry de in (IDictionary)this) {
-                array.SetValue(de, index++);
-            }
-        }
+        void ICollection.CopyTo(Array array, int index) => throw new NotImplementedException("The method or operation is not implemented.");
 
         int ICollection.Count => __len__(DefaultContext.Default);
 
@@ -165,43 +183,43 @@ void ICollection.CopyTo(Array array, int index) {
 
         #region IDictionary<object,object> Members
 
-        bool IDictionary<object, object>.ContainsKey(object key) => __contains__(DefaultContext.Default, key);
+        bool IDictionary<object?, object?>.ContainsKey(object? key) => __contains__(DefaultContext.Default, key);
 
-        ICollection<object> IDictionary<object, object>.Keys => GetDictionary(DefaultContext.Default).Keys;
+        ICollection<object?> IDictionary<object?, object?>.Keys => GetDictionary(DefaultContext.Default).Keys;
 
-        bool IDictionary<object, object>.Remove(object key) => throw new InvalidOperationException("mappingproxy is read-only");
+        bool IDictionary<object?, object?>.Remove(object? key) => throw new InvalidOperationException("mappingproxy is read-only");
 
-        bool IDictionary<object, object>.TryGetValue(object key, out object value) => GetDictionary(DefaultContext.Default).TryGetValue(key, out value);
+        bool IDictionary<object?, object?>.TryGetValue(object? key, out object? value) => GetDictionary(DefaultContext.Default).TryGetValue(key, out value);
 
-        ICollection<object> IDictionary<object, object>.Values => GetDictionary(DefaultContext.Default).Values;
+        ICollection<object?> IDictionary<object?, object?>.Values => GetDictionary(DefaultContext.Default).Values;
 
         #endregion
 
         #region ICollection<KeyValuePair<object,object>> Members
 
-        void ICollection<KeyValuePair<object, object>>.Add(KeyValuePair<object, object> item) {
+        void ICollection<KeyValuePair<object?, object?>>.Add(KeyValuePair<object?, object?> item) {
             this[item.Key] = item.Value;
         }
 
-        bool ICollection<KeyValuePair<object, object>>.Contains(KeyValuePair<object, object> item) => __contains__(DefaultContext.Default, item.Key);
+        bool ICollection<KeyValuePair<object?, object?>>.Contains(KeyValuePair<object?, object?> item) => __contains__(DefaultContext.Default, item.Key);
 
-        void ICollection<KeyValuePair<object, object>>.CopyTo(KeyValuePair<object, object>[] array, int arrayIndex) {
-            foreach (KeyValuePair<object, object> de in (IEnumerable<KeyValuePair<object, object>>)this) {
+        void ICollection<KeyValuePair<object?, object?>>.CopyTo(KeyValuePair<object?, object?>[] array, int arrayIndex) {
+            foreach (KeyValuePair<object?, object?> de in (IEnumerable<KeyValuePair<object?, object?>>)this) {
                 array.SetValue(de, arrayIndex++);
             }
         }
 
-        int ICollection<KeyValuePair<object, object>>.Count => __len__(DefaultContext.Default);
+        int ICollection<KeyValuePair<object?, object?>>.Count => __len__(DefaultContext.Default);
 
-        bool ICollection<KeyValuePair<object, object>>.IsReadOnly => true;
+        bool ICollection<KeyValuePair<object?, object?>>.IsReadOnly => true;
 
-        bool ICollection<KeyValuePair<object, object>>.Remove(KeyValuePair<object, object> item) => ((IDictionary<object, object>)this).Remove(item.Key);
+        bool ICollection<KeyValuePair<object?, object?>>.Remove(KeyValuePair<object?, object?> item) => ((IDictionary<object?, object?>)this).Remove(item.Key);
 
         #endregion
 
         #region IEnumerable<KeyValuePair<object,object>> Members
 
-        IEnumerator<KeyValuePair<object, object>> IEnumerable<KeyValuePair<object, object>>.GetEnumerator() => GetDictionary(DefaultContext.Default).GetEnumerator();
+        IEnumerator<KeyValuePair<object?, object?>> IEnumerable<KeyValuePair<object?, object?>>.GetEnumerator() => GetDictionary(DefaultContext.Default).GetEnumerator();
 
         #endregion
     }

Src/StdLib/Lib/test/test_urlparse.py

@@ -538,6 +538,54 @@ def test_urlsplit_attributes(self):
         p = urllib.parse.urlsplit(url)
         self.assertEqual(p.port, None)
 
+    def test_urlsplit_remove_unsafe_bytes(self):
+        # Remove ASCII tabs and newlines from input, for http common case scenario.
+        url = "h\nttp://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
+        p = urllib.parse.urlsplit(url)
+        self.assertEqual(p.scheme, "http")
+        self.assertEqual(p.netloc, "www.python.org")
+        self.assertEqual(p.path, "/javascript:alert('msg')/")
+        self.assertEqual(p.query, "query=something")
+        self.assertEqual(p.fragment, "fragment")
+        self.assertEqual(p.username, None)
+        self.assertEqual(p.password, None)
+        self.assertEqual(p.hostname, "www.python.org")
+        self.assertEqual(p.port, None)
+        self.assertEqual(p.geturl(), "http://www.python.org/javascript:alert('msg')/?query=something#fragment")
+
+        # Remove ASCII tabs and newlines from input as bytes, for http common case scenario.
+        url = b"h\nttp://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
+        p = urllib.parse.urlsplit(url)
+        self.assertEqual(p.scheme, b"http")
+        self.assertEqual(p.netloc, b"www.python.org")
+        self.assertEqual(p.path, b"/javascript:alert('msg')/")
+        self.assertEqual(p.query, b"query=something")
+        self.assertEqual(p.fragment, b"fragment")
+        self.assertEqual(p.username, None)
+        self.assertEqual(p.password, None)
+        self.assertEqual(p.hostname, b"www.python.org")
+        self.assertEqual(p.port, None)
+        self.assertEqual(p.geturl(), b"http://www.python.org/javascript:alert('msg')/?query=something#fragment")
+
+        # any scheme
+        url = "x-new-scheme\t://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
+        p = urllib.parse.urlsplit(url)
+        self.assertEqual(p.geturl(), "x-new-scheme://www.python.org/javascript:alert('msg')/?query=something#fragment")
+
+        # Remove ASCII tabs and newlines from input as bytes, any scheme.
+        url = b"x-new-scheme\t://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
+        p = urllib.parse.urlsplit(url)
+        self.assertEqual(p.geturl(), b"x-new-scheme://www.python.org/javascript:alert('msg')/?query=something#fragment")
+
+        # Unsafe bytes is not returned from urlparse cache.
+        # scheme is stored after parsing, sending an scheme with unsafe bytes *will not* return an unsafe scheme
+        url = "https://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
+        scheme = "htt\nps"
+        for _ in range(2):
+            p = urllib.parse.urlsplit(url, scheme=scheme)
+            self.assertEqual(p.scheme, "https")
+            self.assertEqual(p.geturl(), "https://www.python.org/javascript:alert('msg')/?query=something#fragment")
+
     def test_attributes_bad_port(self):
         """Check handling of non-integer ports."""
         p = urllib.parse.urlsplit("http://www.example.net:foo")

Src/StdLib/Lib/urllib/parse.py

@@ -65,6 +65,9 @@
                 '0123456789'
                 '+-.')
 
+# Unsafe bytes to be removed per WHATWG spec
+_UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n']
+
 # XXX: Consider replacing with functools.lru_cache
 MAX_CACHE_SIZE = 20
 _parse_cache = {}
@@ -331,13 +334,20 @@ def _checknetloc(netloc):
             raise ValueError("netloc '" + netloc2 + "' contains invalid " +
                              "characters under NFKC normalization")
 
+def _remove_unsafe_bytes_from_url(url):
+    for b in _UNSAFE_URL_BYTES_TO_REMOVE:
+        url = url.replace(b, "")
+    return url
+
 def urlsplit(url, scheme='', allow_fragments=True):
     """Parse a URL into 5 components:
     <scheme>://<netloc>/<path>?<query>#<fragment>
     Return a 5-tuple: (scheme, netloc, path, query, fragment).
     Note that we don't break the components up in smaller bits
     (e.g. netloc is a single string) and we don't expand % escapes."""
     url, scheme, _coerce_result = _coerce_args(url, scheme)
+    url = _remove_unsafe_bytes_from_url(url)
+    scheme = _remove_unsafe_bytes_from_url(scheme)
     allow_fragments = bool(allow_fragments)
     key = url, scheme, allow_fragments, type(url), type(scheme)
     cached = _parse_cache.get(key, None)