Scale max(skipna=True) (#541)

Create numpy_nanmax()
Move nanmax to sdc/functions/numpy_like.py

Add nan_min_max_factory()

Use numpy_like.nanmin()

Min tests

Correct implementation. Eliminates race condition.

Add perf test for nanmin/max

Add tests for nanmin/max

Author: PokhodenkoSA

sdc/datatypes/hpat_pandas_series_functions.py

@@ -61,6 +61,7 @@
                              create_str_arr_from_list, str_arr_set_na_by_mask)
 from sdc.utilities.utils import to_array, sdc_overload, sdc_overload_method, sdc_overload_attribute
 from sdc import sdc_autogenerated
+from sdc.functions import numpy_like
 
 from .pandas_series_functions import apply
 from .pandas_series_functions import map as _map
@@ -3634,7 +3635,7 @@ def hpat_pandas_series_min_impl(self, axis=None, skipna=None, level=None, numeri
             _skipna = skipna
 
         if _skipna:
-            return numpy.nanmin(self._data)
+            return numpy_like.nanmin(self._data)
 
         return self._data.min()
 
@@ -3715,7 +3716,7 @@ def hpat_pandas_series_max_impl(self, axis=None, skipna=None, level=None, numeri
             _skipna = skipna
 
         if _skipna:
-            return numpy.nanmax(self._data)
+            return numpy_like.nanmax(self._data)
 
         return self._data.max()
 

sdc/functions/numpy_like.py

@@ -257,3 +257,54 @@ def sdc_nansum_number_impl(self):
             return result
 
         return sdc_nansum_number_impl
+
+
+def nanmin(a):
+    pass
+
+
+def nanmax(a):
+    pass
+
+
+def nan_min_max_overload_factory(reduce_op):
+    def ov_impl(a):
+        if not isinstance(a, types.Array):
+            return
+
+        if isinstance(a.dtype, (types.Float, types.Complex)):
+            isnan = get_isnan(a.dtype)
+            initial_result = {
+                min: numpy.inf,
+                max: -numpy.inf,
+            }[reduce_op]
+
+            def impl(a):
+                result = initial_result
+                nan_count = 0
+                length = len(a)
+                for i in prange(length):
+                    v = a[i]
+                    if not isnan(v):
+                        result = reduce_op(result, v)
+                    else:
+                        nan_count += 1
+
+                if nan_count == length:
+                    return numpy.nan
+
+                return result
+            return impl
+        else:
+            def impl(a):
+                result = a[0]
+                for i in prange(len(a) - 1):
+                    result = reduce_op(result, a[i + 1])
+                return result
+            return impl
+
+    return ov_impl
+
+
+sdc_overload(nanmin)(nan_min_max_overload_factory(min))
+sdc_overload(nanmax)(nan_min_max_overload_factory(max))

sdc/tests/test_sdc_numpy.py

@@ -188,20 +188,47 @@ def sdc_impl(a):
             with self.subTest(data=case):
                 np.testing.assert_array_equal(sdc_func(a), ref_impl(a))
 
-    def test_sum(self):
+
+class TestArrayReductions(TestCase):
+
+    def check_reduction_basic(self, pyfunc, alt_pyfunc, all_nans=True):
+        alt_cfunc = self.jit(alt_pyfunc)
+
+        def cases():
+            yield np.array([5, 2, 0, 333, -4])
+            yield np.array([3.3, 5.4, np.nan, 7.9, np.nan])
+            yield np.float64([1.0, 2.0, 0.0, -0.0, 1.0, -1.5])
+            yield np.float64([-0.0, -1.5])
+            yield np.float64([-1.5, 2.5, 'inf'])
+            yield np.float64([-1.5, 2.5, '-inf'])
+            yield np.float64([-1.5, 2.5, 'inf', '-inf'])
+            yield np.float64(['nan', -1.5, 2.5, 'nan', 3.0])
+            yield np.float64(['nan', -1.5, 2.5, 'nan', 'inf', '-inf', 3.0])
+            if all_nans:
+                # Only NaNs
+                yield np.float64(['nan', 'nan'])
+
+        for case in cases():
+            with self.subTest(data=case):
+                np.testing.assert_array_equal(alt_cfunc(case), pyfunc(case))
+
+    def test_nanmin(self):
         def ref_impl(a):
-            return np.sum(a)
+            return np.nanmin(a)
 
         def sdc_impl(a):
-            return numpy_like.sum(a)
+            return numpy_like.nanmin(a)
 
-        sdc_func = self.jit(sdc_impl)
+        self.check_reduction_basic(ref_impl, sdc_impl)
 
-        cases = [[5, 2, 0, 333, -4], [3.3, 5.4, np.nan, 7.9, np.nan]]
-        for case in cases:
-            a = np.array(case)
-            with self.subTest(data=case):
-                np.testing.assert_array_equal(sdc_func(a), ref_impl(a))
+    def test_nanmax(self):
+        def ref_impl(a):
+            return np.nanmax(a)
+
+        def sdc_impl(a):
+            return numpy_like.nanmax(a)
+
+        self.check_reduction_basic(ref_impl, sdc_impl)
 
     def test_nansum(self):
         def ref_impl(a):
@@ -210,13 +237,13 @@ def ref_impl(a):
         def sdc_impl(a):
             return numpy_like.nansum(a)
 
-        sdc_func = self.jit(sdc_impl)
+        self.check_reduction_basic(ref_impl, sdc_impl)
 
-        cases = [[5, 2, 0, 333, -4], [3.3, 5.4, np.nan, 7.9, np.nan]]
-        for case in cases:
-            a = np.array(case)
-            with self.subTest(data=case):
-                np.testing.assert_array_equal(sdc_func(a), ref_impl(a))
+    def test_sum(self):
+        def ref_impl(a):
+            return np.sum(a)
+
+        def sdc_impl(a):
+            return numpy_like.sum(a)
 
-if __name__ == "__main__":
-    unittest.main()
+        self.check_reduction_basic(ref_impl, sdc_impl)

sdc/tests/test_series.py

@@ -2470,9 +2470,12 @@ def test_impl(S):
         hpat_func = self.jit(test_impl)
 
         # TODO type_min/type_max
-        for input_data in [[np.nan, 2., np.nan, 3., np.inf, 1, -1000],
-                           [8, 31, 1123, -1024],
-                           [2., 3., 1, -1000, np.inf]]:
+        for input_data in [
+                [np.nan, 2., np.nan, 3., np.inf, 1, -1000],
+                [8, 31, 1123, -1024],
+                [2., 3., 1, -1000, np.inf],
+                [np.nan, np.nan, np.inf, np.nan],
+            ]:
             S = pd.Series(input_data)
 
             result_ref = test_impl(S)
@@ -2500,14 +2503,32 @@ def test_impl(S):
         hpat_func = self.jit(test_impl)
 
         # TODO type_min/type_max
-        for input_data in [[np.nan, 2., np.nan, 3., np.inf, 1, -1000],
-                           [8, 31, 1123, -1024],
-                           [2., 3., 1, -1000, np.inf]]:
-            S = pd.Series(input_data)
+        for input_data in [
+                [np.nan, 2., np.nan, 3., np.inf, 1, -1000],
+                [8, 31, 1123, -1024],
+                [2., 3., 1, -1000, np.inf],
+                [np.inf, np.inf, np.inf, np.inf],
+                [np.inf, np.nan, np.nan, np.nan],
 
-            result_ref = test_impl(S)
-            result = hpat_func(S)
-            self.assertEqual(result, result_ref)
+                [np.nan, np.nan, np.nan, np.nan],
+                [np.nan, 1.0, np.nan, np.nan],
+                [np.nan, 1.0, 1.0, np.nan],
+
+                [np.nan, np.nan, 1.0, np.nan],
+                [np.nan, np.nan, 1.0, np.nan, np.nan],
+
+                [np.nan, np.nan, np.inf, np.nan],
+                [np.nan, np.nan, np.inf, np.nan, np.nan],
+
+                [np.nan, np.nan, np.nan, np.inf],
+                np.arange(11),
+            ]:
+            with self.subTest(data=input_data):
+                S = pd.Series(input_data)
+
+                result_ref = test_impl(S)
+                result = hpat_func(S)
+                np.testing.assert_equal(result, result_ref)
 
     @skip_sdc_jit("Series.max() any parameters unsupported")
     def test_series_max_param(self):

sdc/tests/tests_perf/test_perf_numpy.py

@@ -113,6 +113,16 @@ def _test_case(self, cases, name, total_data_length, data_num=1, input_data=test
         CE(type_='Numba', code='np.sum(data)', jitted=True),
         CE(type_='SDC', code='sdc.functions.numpy_like.sum(data)', jitted=True),
     ], usecase_params='data'),
+    TC(name='nanmin', size=[10 ** 7], call_expr=[
+        CE(type_='Python', code='np.nanmin(data)', jitted=False),
+        CE(type_='Numba', code='np.nanmin(data)', jitted=True),
+        CE(type_='SDC', code='sdc.functions.numpy_like.nanmin(data)', jitted=True),
+    ], usecase_params='data'),
+    TC(name='nanmax', size=[10 ** 7], call_expr=[
+        CE(type_='Python', code='np.nanmax(data)', jitted=False),
+        CE(type_='Numba', code='np.nanmax(data)', jitted=True),
+        CE(type_='SDC', code='sdc.functions.numpy_like.nanmax(data)', jitted=True),
+    ], usecase_params='data'),
 ]
 
 generate_test_cases(cases, TestFunctions, 'function')

sdc/tests/tests_perf/test_perf_series.py

@@ -115,8 +115,8 @@ def _test_case(self, pyfunc, name, total_data_length, data_num=1, input_data=tes
     TC(name='max', size=[10 ** 8], params='skipna=False'),
     TC(name='mean', size=[10 ** 8]),
     TC(name='median', size=[10 ** 8]),
-    TC(name='min', size=[10 ** 8]),
-    TC(name='min', size=[10 ** 7], params='skipna=False'),
+    TC(name='min', size=[10 ** 8], params='skipna=True'),
+    TC(name='min', size=[10 ** 8], params='skipna=False'),
     TC(name='mod', size=[10 ** 7], params='other', data_num=2),
     TC(name='mul', size=[10 ** 7], params='other', data_num=2),
     TC(name='ndim', size=[10 ** 7], call_expr='data.ndim', usecase_params='data'),