Merge branch 'PolyMathOrg:master' into describe

Author: Joshua-Dias-Barreto

src/DataFrame-IO-Tests/DataFrameCsvReaderTest.class.st

@@ -106,10 +106,10 @@ DataFrameCsvReaderTest >> testReadFromString [
 	actualDataFrame := DataFrameCsvReader new readFromString: TestCsvStrings commaCsvString.
 
 	self
-		assertCollection: actualDataFrame columns first
+		assertCollection: actualDataFrame asArrayOfColumns first
 		hasSameElements: #('1:10 am' '1:30 am' '1:50 am' '2:10 am' '2:30 am').
 	self
-		assertCollection: actualDataFrame columns last
+		assertCollection: actualDataFrame asArrayOfColumns last
 		hasSameElements: #('rain' 'rain' 'snow' '-' 'rain')
 ]
 

src/DataFrame-IO/DataFrameTypeDetector.class.st

@@ -146,7 +146,7 @@ DataFrameTypeDetector >> detectColumnTypeAndConvert: anArray [
 { #category : #'public API' }
 DataFrameTypeDetector >> detectTypesAndConvert: aDataFrame [
 
-	aDataFrame columns with: aDataFrame columnNames do: [ :column :columnName |
+	aDataFrame asArrayOfColumns with: aDataFrame columnNames do: [ :column :columnName |
 		| thisColumnType |
 		"Get the user given column type for this column name and if it wasn't
 			 given then use the default type detection"

src/DataFrame-Tests/DataFrameTest.class.st

@@ -758,11 +758,10 @@ DataFrameTest >> testColumnTransformNotFound [
 DataFrameTest >> testColumns [
 
 	| expectedCollection |
-
-	expectedCollection := #(
-		(Barcelona Dubai London)
-	   (1.609 2.789 8.788)
-	   (true true false)).
+	expectedCollection := {
+		                      (df columnAt: 1).
+		                      (df columnAt: 2).
+		                      (df columnAt: 3) } asArray.
 
 	self assert: df columns equals: expectedCollection
 ]
@@ -2845,6 +2844,36 @@ DataFrameTest >> testNumberOfNils [
 	self assert: df numberOfNils equals: expected
 ]
 
+{ #category : #tests }
+DataFrameTest >> testNumericalColumnNames [
+
+	| dataFrame expected |
+	dataFrame := DataFrame withRows:
+		             #( #( Male 21 Argentina 94 ) #( Female 20 France 97 )
+		                #( Female 22 Spain 95 ) #( Male 24 Portugal 99 ) ).
+	dataFrame columnNames: #( Gender Age Country Score ).
+
+	expected := #( Age Score ) asOrderedCollection.
+
+	self assert: dataFrame numericalColumnNames equals: expected
+]
+
+{ #category : #tests }
+DataFrameTest >> testNumericalColumns [
+
+	| dataFrame expectedCollection |
+	dataFrame := DataFrame withRows:
+		             #( #( Male 21 Argentina 94 ) #( Female 20 France 97 )
+		                #( Female 22 Spain 95 ) #( Male 24 Portugal 99 ) ).
+	dataFrame columnNames: #( Gender Age Country Score ).
+
+	expectedCollection := {
+		                      (dataFrame columnAt: 2).
+		                      (dataFrame columnAt: 4) } asArray.
+
+	self assert: dataFrame numericalColumns equals: expectedCollection
+]
+
 { #category : #splitjoin }
 DataFrameTest >> testOuterJoin [
 	| df2 expected |
@@ -3889,6 +3918,30 @@ DataFrameTest >> testReplaceNilsWithMode [
 	self assert: df replaceNilsWithMode equals: expected
 ]
 
+{ #category : #tests }
+DataFrameTest >> testReplaceNilsWithNextRowValue [
+
+	|expected|
+
+	df := DataFrame withRows: #(
+		(7 1 nil)
+   		(8 nil 3)
+   		(nil 2 nil)).
+
+	df rowNames: #(A B C).
+	df columnNames: #(Column1 Column2 Column3).
+
+	expected := DataFrame withRows: #(
+		(7 1 3)
+   		(8 2 3)
+   		(nil 2 nil)).
+
+	expected rowNames: #(A B C).
+	expected columnNames: #(Column1 Column2 Column3).
+
+	self assert: (df replaceNilsWithNextRowValue) equals: expected
+]
+
 { #category : #replacing }
 DataFrameTest >> testReplaceNilsWithPreviousRowValue [
 

src/DataFrame-Tests/DataSeriesTest.class.st

@@ -874,10 +874,11 @@ DataSeriesTest >> testCategoricalIsNotNumerical [
 
 { #category : #'tests - categorical' }
 DataSeriesTest >> testCategoricalUniqueValues [
+
 	| aSeries expected actual |
-	aSeries := DataSeries withKeys: #(a b c d e) values: #(z y y z x).
-	expected := #(x y z).
-	actual := aSeries uniqueValues.
+	aSeries := DataSeries withKeys: #( a b c d e ) values: #( z y y z x ).
+	expected := #( x y z ).
+	actual := aSeries removeDuplicates.
 	self assert: actual equals: expected
 ]
 
@@ -1587,6 +1588,42 @@ DataSeriesTest >> testMathTan [
 	self assert: a tan closeTo: b
 ]
 
+{ #category : #'tests - statistics' }
+DataSeriesTest >> testMax [
+
+	self assert: #( 1 2 3 4 ) asDataSeries max equals: 4
+]
+
+{ #category : #'tests - statistics' }
+DataSeriesTest >> testMaxWithNils [
+
+	self assert: #( 1 nil 3 nil ) asDataSeries max equals: 3
+]
+
+{ #category : #tests }
+DataSeriesTest >> testMedian [
+
+	self assert: #( 1 2 3 4 5 ) asDataSeries median equals: 3
+]
+
+{ #category : #tests }
+DataSeriesTest >> testMedianWithNils [
+
+	self assert: #( 1 2 nil 3 4 nil 5 ) asDataSeries median equals: 3
+]
+
+{ #category : #'tests - statistics' }
+DataSeriesTest >> testMin [
+
+	self assert: #( 1 2 3 4 ) asDataSeries min equals: 1
+]
+
+{ #category : #'tests - statistics' }
+DataSeriesTest >> testMinWithNils [
+
+	self assert: #( nil 2 nil 4 ) asDataSeries min equals: 2
+]
+
 { #category : #'tests - creation' }
 DataSeriesTest >> testNewFrom [
 
@@ -1680,6 +1717,20 @@ DataSeriesTest >> testRemoveAtIndex [
 	self assert: series equals: expected
 ]
 
+{ #category : #tests }
+DataSeriesTest >> testRemoveDuplicates [
+
+	| dataseries expected |
+	dataseries := DataSeries
+		              withKeys: #( 'A' 'B' 'C' 'D' 'E' )
+		              values: #( 7 1 1 1 3 )
+		              name: series.
+
+	expected := #( 7 1 3 ).
+
+	self assert: dataseries removeDuplicates equals: expected
+]
+
 { #category : #'tests - removing' }
 DataSeriesTest >> testRemoveNils [
 
@@ -2161,6 +2212,18 @@ DataSeriesTest >> testStatsZerothQuartileEqualsMin [
 	self assert: series zerothQuartile equals: series min
 ]
 
+{ #category : #'tests - statistics' }
+DataSeriesTest >> testStdev [
+
+	self assert: #( 1 2 3 ) asDataSeries stdev equals: 1
+]
+
+{ #category : #tests }
+DataSeriesTest >> testStdevWithNils [
+
+	self assert: #( 1 nil 2 nil 3 ) asDataSeries stdev equals: 1
+]
+
 { #category : #'tests - arithmetic' }
 DataSeriesTest >> testSum [
 

src/DataFrame/DataFrame.class.st

@@ -543,7 +543,7 @@ DataFrame >> average [
 { #category : #'data-types' }
 DataFrame >> calculateDataTypes [
 
-	self columns doWithIndex: [ :column :i | self dataTypes at: (self columnNames at: i) put: column calculateDataType ]
+	self asArrayOfColumns doWithIndex: [ :column :i | self dataTypes at: (self columnNames at: i) put: column calculateDataType ]
 ]
 
 { #category : #comparing }
@@ -722,10 +722,9 @@ DataFrame >> columnNames: aCollection [
 
 { #category : #accessing }
 DataFrame >> columns [
-
 	"Returns a collection of all columns"
 
-	^ self asArrayOfColumns
+	^ (1 to: self numberOfColumns) collect: [ :j | self columnAt: j ]
 ]
 
 { #category : #accessing }
@@ -1098,7 +1097,7 @@ DataFrame >> hasNils [
 	"Returns true if there is atleast one nil value in the data frame. Returns false if there are no nil values in the dataframe"
 
 	| arrayOfColumns |
-	arrayOfColumns := self columns.
+	arrayOfColumns := self asArrayOfColumns.
 	1 to: self numberOfColumns do: [ :column |
 		1 to: self numberOfRows do: [ :row |
 		((arrayOfColumns at: column) at: row) ifNil: [ ^ true ] ] ].
@@ -1442,7 +1441,7 @@ DataFrame >> normalized [
 		'DataFrame will remove the dependency over normalization in the next version. You can use pharo-ai/data-preprocessing project to normalize your DataFrame and even more!'.
 	normalizers := (1 to: self anyOne size) collect: [ :e | self class defaultNormalizerClass new ].
 
-	normalizedColumns := self columns with: normalizers collect: [ :col :normalizer | col normalizedUsing: normalizer ].
+	normalizedColumns := self asArrayOfColumns with: normalizers collect: [ :col :normalizer | col normalizedUsing: normalizer ].
 
 	^ self class withColumns: normalizedColumns columnNames: self columnNames
 ]
@@ -1473,6 +1472,22 @@ DataFrame >> numberOfRows [
 	^ contents numberOfRows
 ]
 
+{ #category : #accessing }
+DataFrame >> numericalColumnNames [
+	"Returns the names of all numerical columns of the dataframe"
+
+	^ self columnNames select: [ :columnName |
+		  (self dataTypes at: columnName) includesBehavior: Number ]
+]
+
+{ #category : #accessing }
+DataFrame >> numericalColumns [
+	"Returns all numerical columns of the dataframe"
+
+	^ self columns select: [ :column |
+		  (self dataTypes at: column name) includesBehavior: Number ]
+]
+
 { #category : #splitjoin }
 DataFrame >> outerJoin: aDataFrame [
 	"Performs outer join on aDataFrame with rowNames as keys"
@@ -1838,6 +1853,18 @@ DataFrame >> replaceNilsWithMode [
 		modeOfColumn := nil ]
 ]
 
+{ #category : #replacing }
+DataFrame >> replaceNilsWithNextRowValue [
+
+	| value numberOfRows |
+	numberOfRows := self numberOfRows.
+	1 to: self numberOfColumns do: [ :i |
+		self numberOfRows to: 1 by: -1 do: [ :j |
+			j < numberOfRows ifTrue: [
+				(self at: j at: i) ifNil: [ self at: j at: i put: value ] ].
+			value := self at: j at: i ] ]
+]
+
 { #category : #replacing }
 DataFrame >> replaceNilsWithPreviousRowValue [
 	"Replaces all nil values of a data frame with the previous non-nil value of the column in which it is present"