Added filter support on 2D numerical columns (mask 2D)

Fixed Merger to set 2D masked values to NaN + fixed CLI inputs/args in HISTORY
DataModel improvements

Author: bourgesl

src/main/java/fr/jmmc/oitools/OIFitsProcessor.java

@@ -38,7 +38,6 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.logging.Level;
@@ -590,10 +589,10 @@ public final void getNumericalColumnsNames(final Set<String> columnNames) {
      * Return the column descriptors (standard then derived) representing
      * numerical values (double arrays)
      *
-     * @return list of numerical column descriptors
+     * @param columnDescList list to store numerical column descriptors
+     * @return given columnDescList reference
      */
-    public final List<ColumnMeta> getNumericalColumnsDescs() {
-        final ArrayList<ColumnMeta> columnDescList = new ArrayList<ColumnMeta>();
+    public final ArrayList<ColumnMeta> getNumericalColumnsDescs(final ArrayList<ColumnMeta> columnDescList) {
         // Standard columns:
         for (Map.Entry<String, ColumnMeta> entry : getColumnsDesc().entrySet()) {
             final ColumnMeta meta = entry.getValue();
@@ -621,10 +620,10 @@ public final List<ColumnMeta> getNumericalColumnsDescs() {
      * Return the column descriptors (standard then derived) representing
      * CHAR/SHORT/INT values (arrays) except "CORRINDX_..." columns
      *
-     * @return list of other column descriptors
+     * @param columnDescList list to store other column descriptors
+     * @return given columnDescList reference
      */
-    public final List<ColumnMeta> getOtherColumnsDescs() {
-        final ArrayList<ColumnMeta> columnDescList = new ArrayList<ColumnMeta>();
+    public final ArrayList<ColumnMeta> getOtherColumnsDescs(final ArrayList<ColumnMeta> columnDescList) {
         // Standard columns:
         for (Map.Entry<String, ColumnMeta> entry : getColumnsDesc().entrySet()) {
             final ColumnMeta meta = entry.getValue();

src/main/java/fr/jmmc/oitools/fits/FitsTable.java

@@ -44,6 +44,8 @@ public class ColumnMeta extends CellMeta {
     private final int repeat;
     /** optional column name storing error values (may be null) */
     private final String errName;
+    /** optional column name storing data values (may be null) for the error column */
+    private String dataName = null;
     /** optional data range (may be null) */
     private final DataRange dataRange;
     /** optional column alias ie alternate name (may be null) */
@@ -226,13 +228,27 @@ public int getRepeat() {
 
     /* ---  Error relationships --- */
     /**
-     * Return the optional column name storing error values (may be null)
      * @return optional column name storing error values (may be null)
      */
     public final String getErrorColumnName() {
         return errName;
     }
 
+    /**
+     * @return optional column name storing error values (may be null)
+     */
+    public String getDataColumnName() {
+        return dataName;
+    }
+
+    /**
+     * Define the optional column name storing data values (may be null) for the error column
+     * @param dataName optional column name storing data values (may be null) for the error column
+     */
+    public void setDataColumnName(final String dataName) {
+        this.dataName = dataName;
+    }
+
     /* ---  Data range --- */
     /**
      * Return the optional data range (may be null)

src/main/java/fr/jmmc/oitools/meta/ColumnMeta.java

@@ -20,6 +20,8 @@
 package fr.jmmc.oitools.meta;
 
 import fr.jmmc.oitools.model.OIAbstractData;
+import java.util.LinkedHashSet;
+import java.util.Set;
 
 /**
  * This specific ColumnMeta overrides the getRepeat() method to use the OIWaveLength.getNWave() method
@@ -33,6 +35,8 @@ public final class WaveColumnMeta extends ArrayColumnMeta {
     private final OIAbstractData oiAbsData;
     /** optional expression to compute derived columns */
     private String expression;
+    /** JEL resolved used variables by the optional expression */
+    private Set<String> relatedColumnNames = null;
 
     /**
      * ColumnMeta class constructor
@@ -216,4 +220,16 @@ public int getRepeat() {
     public String getExpression() {
         return expression;
     }
+
+    public boolean hasRelatedColumnNames() {
+        return (this.relatedColumnNames != null) && !this.relatedColumnNames.isEmpty();
+    }
+
+    public Set<String> getRelatedColumnNames() {
+        if (this.relatedColumnNames == null) {
+            this.relatedColumnNames = new LinkedHashSet<>(8);
+        }
+        return relatedColumnNames;
+    }
+
 }

src/main/java/fr/jmmc/oitools/meta/WaveColumnMeta.java

@@ -96,6 +96,7 @@ public static synchronized DataModel getInstance(final OIFitsStandard version) {
 
     public static DataModel getInstance(final Collection<OIData> oiDatas) {
         if (oiDatas == null || oiDatas.isEmpty()) {
+            // logger.log(Level.SEVERE, "getInstance: using default :", new Throwable());
             return getInstance();
         }
         return new DataModel(oiDatas, false);
@@ -429,6 +430,9 @@ private static void dumpColumn(final ColumnMeta column, final StringBuilder sb)
         if (column.getErrorColumnName() != null) {
             sb.append("    <errorcolumn>").append(column.getErrorColumnName()).append("</errorcolumn>\n");
         }
+        if (column.getDataColumnName() != null) {
+            sb.append("    <datacolumn>").append(column.getDataColumnName()).append("</datacolumn>\n");
+        }
         if (column.getDataRange() != null) {
             sb.append("    <datarange>\n");
             final DataRange range = column.getDataRange();
@@ -597,9 +601,11 @@ private DataModel(final Collection<OIData> oiDatas, final boolean shared) {
     }
 
     private void reset() {
+        allColumnNames = null;
         allNumColumnNames = null;
         allNumColumnNames1D = null;
         allNumColumnNames2D = null;
+        allOtherColumnNames = null;
     }
 
     public void refresh() {
@@ -712,9 +718,14 @@ private static Set<String> getOtherColumnNames(final Collection<OIData> oiDatas)
     private static void getAllNumericalColumnNames(final Collection<OIData> oiDatas, final String extName,
                                                    final boolean all, final boolean is2D,
                                                    final Set<String> columnNames) {
+
+        final ArrayList<ColumnMeta> columnsDescCollection = new ArrayList<ColumnMeta>();
+
         for (final OIData oiData : oiDatas) {
             if (extName.equals(oiData.getExtName())) {
-                for (final ColumnMeta colMeta : oiData.getNumericalColumnsDescs()) {
+                columnsDescCollection.clear();
+
+                for (final ColumnMeta colMeta : oiData.getNumericalColumnsDescs(columnsDescCollection)) {
                     if (colMeta != null) {
                         if (colMeta instanceof WaveColumnMeta) {
                             if (all || is2D) {
@@ -733,9 +744,14 @@ private static void getAllNumericalColumnNames(final Collection<OIData> oiDatas,
 
     private static void getOtherColumnNames(final Collection<OIData> oiDatas, final String extName,
                                             final Set<String> columnNames) {
+
+        final ArrayList<ColumnMeta> columnsDescCollection = new ArrayList<ColumnMeta>();
+
         for (final OIData oiData : oiDatas) {
             if (extName.equals(oiData.getExtName())) {
-                for (final ColumnMeta meta : oiData.getOtherColumnsDescs()) {
+                columnsDescCollection.clear();
+
+                for (final ColumnMeta meta : oiData.getOtherColumnsDescs(columnsDescCollection)) {
                     if (meta != null) {
                         if (logger.isLoggable(Level.FINE)) {
                             logger.log(Level.FINE, "Column[{0}] type = {1}",
@@ -772,7 +788,7 @@ public static void main(String[] unused) {
             final boolean isNumeric2D = dm.isNumericalColumn2D(name);
             final boolean isOther = dm.isOtherColumn(name);
 
-            logger.log(Level.WARNING, "column[{0}] : isNumeric={1} isNumeric1D={2} isNumeric2D={3} isOther={4}",
+            logger.log(Level.WARNING, "column[{0}]:\tisNumeric={1}\tisNumeric1D={2}\tisNumeric2D={3}\tisOther={4}",
                     new Object[]{name, isNumeric, isNumeric1D, isNumeric2D, isOther});
         }
     }

src/main/java/fr/jmmc/oitools/model/DataModel.java

@@ -120,8 +120,9 @@ public boolean accept(final int rowIndex, final int colIndex) {
     public void setAccept(final int rowIndex, final int colIndex, final boolean value) {
         if (!vect1D) {
             bitSet.set(getBitIndex(rowIndex, colIndex), value);
+        } else {
+            throw new IllegalArgumentException("Incompatible mask dimensions (2D) !");
         }
-        throw new IllegalArgumentException("Incompatible mask dimensions (2D) !");
     }
 
     private int getBitIndex(final int rowIndex, final int colIndex) {
@@ -132,8 +133,12 @@ public int cardinality() {
         return this.bitSet.cardinality();
     }
 
-    public boolean isFull() {
-        return (this == FULL);
+    public static boolean isFull(final IndexMask mask) {
+        return (mask == FULL);
+    }
+
+    public static boolean isNotFull(final IndexMask mask) {
+        return (mask != null) && (mask != FULL);
     }
 
     @Override
@@ -166,4 +171,17 @@ public BitSet getBitSet() {
         return bitSet;
     }
 
+    public int getIndexNone() {
+        if (!vect1D) {
+            return nbCols - 2;
+        }
+        return -1;
+    }
+
+    public int getIndexFull() {
+        if (!vect1D) {
+            return nbCols - 1;
+        }
+        return -1;
+    }
 }

src/main/java/fr/jmmc/oitools/model/IndexMask.java

@@ -20,15 +20,16 @@
 package fr.jmmc.oitools.model;
 
 import fr.jmmc.oitools.meta.ColumnMeta;
+import fr.jmmc.oitools.meta.WaveColumnMeta;
 import static fr.jmmc.oitools.model.ModelBase.UNDEFINED_DBL;
 import gnu.jel.CompilationException;
 import gnu.jel.CompiledExpression;
 import gnu.jel.DVMap;
 import gnu.jel.Evaluator;
 import gnu.jel.Library;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -37,7 +38,7 @@
  * ExpressionEvaluator implementation based on gnu JEL
  * @author grosje
  */
-public class JELEval extends ExpressionEvaluator {
+public final class JELEval extends ExpressionEvaluator {
 
     /** logger */
     private final static Logger _logger = Logger.getLogger(JELEval.class.getName());
@@ -56,8 +57,7 @@ public JELEval() {
      */
     @Override
     public double[][] eval(final OIData oiData, final String colNameEval,
-                           final String expression, final boolean testOnly)
-            throws RuntimeException {
+                           final String expression, final boolean testOnly) throws RuntimeException {
 
         if (_logger.isLoggable(Level.FINE)) {
             _logger.log(Level.FINE, "eval: column= {0} expression= {1}", new Object[]{colNameEval, expression});
@@ -73,7 +73,7 @@ public double[][] eval(final OIData oiData, final String colNameEval,
 
         // Prepare inputs:
         // all numerical names (original + derived):
-        final List<ColumnMeta> columnsDescCollection = oiData.getNumericalColumnsDescs();
+        final ArrayList<ColumnMeta> columnsDescCollection = oiData.getNumericalColumnsDescs(new ArrayList<ColumnMeta>());
 
         // Create input column names:
         int n = 0;
@@ -111,11 +111,13 @@ public double[][] eval(final OIData oiData, final String colNameEval,
             final Library lib = new Library(staticLib, dynamicLib, null, resolver, null);
 
             // Math.random():
+            // may throw CompilationException or Throwable:
             lib.markStateDependent("random", null);
 
             // Compile expression
-            _logger.fine("Compilation ...");
+            _logger.fine("Compiling expression ...");
 
+            // may throw CompilationException or Throwable:
             final CompiledExpression expr_c = Evaluator.compile(expression, lib);
 
             // Retrieve values:
@@ -126,6 +128,22 @@ public double[][] eval(final OIData oiData, final String colNameEval,
                 _logger.log(Level.FINE, "usedNames: {0}", usedNames);
             }
 
+            // Update related column names for the dynamic column:
+            final ColumnMeta colMetaExpr = oiData.getColumnDerivedDesc(colNameEval);
+
+            if (colMetaExpr instanceof WaveColumnMeta) {
+                if (_logger.isLoggable(Level.FINE)) {
+                    _logger.log(Level.FINE, "colMetaExpr: {0}", colMetaExpr);
+                }
+                final Set<String> relatedColumnNames = ((WaveColumnMeta) colMetaExpr).getRelatedColumnNames();
+                relatedColumnNames.clear();
+                relatedColumnNames.addAll(usedNames);
+
+                if (_logger.isLoggable(Level.FINE)) {
+                    _logger.log(Level.FINE, "relatedColumnNames: {0}", relatedColumnNames);
+                }
+            }
+
             // Create the input column array:
             // warning: array must have the same size and indices as the input column names (jelNames):
             n = 0;
@@ -181,6 +199,7 @@ public double[][] eval(final OIData oiData, final String colNameEval,
                 for (int j = 0; j < nWaves; j++) {
                     varProvider.j = j;
 
+                    // may throw Throwable:
                     row[j] = expr_c.evaluate_double(context);
                 }
             }