ensmallen lbfgs improve

Author: gauravmanmode

src/optimagic/algorithms.py

@@ -294,7 +294,6 @@ class BoundedGradientBasedLocalScalarAlgorithms(AlgoSelection):
     nlopt_slsqp: Type[NloptSLSQP] = NloptSLSQP
     nlopt_tnewton: Type[NloptTNewton] = NloptTNewton
     nlopt_var: Type[NloptVAR] = NloptVAR
-    ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS
     scipy_lbfgsb: Type[ScipyLBFGSB] = ScipyLBFGSB
     scipy_slsqp: Type[ScipySLSQP] = ScipySLSQP
     scipy_truncated_newton: Type[ScipyTruncatedNewton] = ScipyTruncatedNewton
@@ -849,7 +848,6 @@ class BoundedGradientBasedLocalAlgorithms(AlgoSelection):
     nlopt_slsqp: Type[NloptSLSQP] = NloptSLSQP
     nlopt_tnewton: Type[NloptTNewton] = NloptTNewton
     nlopt_var: Type[NloptVAR] = NloptVAR
-    ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS
     scipy_lbfgsb: Type[ScipyLBFGSB] = ScipyLBFGSB
     scipy_ls_dogbox: Type[ScipyLSDogbox] = ScipyLSDogbox
     scipy_ls_trf: Type[ScipyLSTRF] = ScipyLSTRF
@@ -967,7 +965,6 @@ class BoundedGradientBasedScalarAlgorithms(AlgoSelection):
     nlopt_slsqp: Type[NloptSLSQP] = NloptSLSQP
     nlopt_tnewton: Type[NloptTNewton] = NloptTNewton
     nlopt_var: Type[NloptVAR] = NloptVAR
-    ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS
     scipy_basinhopping: Type[ScipyBasinhopping] = ScipyBasinhopping
     scipy_dual_annealing: Type[ScipyDualAnnealing] = ScipyDualAnnealing
     scipy_lbfgsb: Type[ScipyLBFGSB] = ScipyLBFGSB
@@ -1692,7 +1689,6 @@ class BoundedLocalScalarAlgorithms(AlgoSelection):
     nlopt_sbplx: Type[NloptSbplx] = NloptSbplx
     nlopt_tnewton: Type[NloptTNewton] = NloptTNewton
     nlopt_var: Type[NloptVAR] = NloptVAR
-    ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS
     scipy_lbfgsb: Type[ScipyLBFGSB] = ScipyLBFGSB
     scipy_neldermead: Type[ScipyNelderMead] = ScipyNelderMead
     scipy_powell: Type[ScipyPowell] = ScipyPowell
@@ -1999,7 +1995,6 @@ class BoundedGradientBasedAlgorithms(AlgoSelection):
     nlopt_slsqp: Type[NloptSLSQP] = NloptSLSQP
     nlopt_tnewton: Type[NloptTNewton] = NloptTNewton
     nlopt_var: Type[NloptVAR] = NloptVAR
-    ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS
     scipy_basinhopping: Type[ScipyBasinhopping] = ScipyBasinhopping
     scipy_dual_annealing: Type[ScipyDualAnnealing] = ScipyDualAnnealing
     scipy_lbfgsb: Type[ScipyLBFGSB] = ScipyLBFGSB
@@ -2601,7 +2596,6 @@ class BoundedLocalAlgorithms(AlgoSelection):
     nlopt_tnewton: Type[NloptTNewton] = NloptTNewton
     nlopt_var: Type[NloptVAR] = NloptVAR
     pounders: Type[Pounders] = Pounders
-    ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS
     scipy_lbfgsb: Type[ScipyLBFGSB] = ScipyLBFGSB
     scipy_ls_dogbox: Type[ScipyLSDogbox] = ScipyLSDogbox
     scipy_ls_trf: Type[ScipyLSTRF] = ScipyLSTRF
@@ -2837,7 +2831,6 @@ class BoundedScalarAlgorithms(AlgoSelection):
     nlopt_sbplx: Type[NloptSbplx] = NloptSbplx
     nlopt_tnewton: Type[NloptTNewton] = NloptTNewton
     nlopt_var: Type[NloptVAR] = NloptVAR
-    ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS
     pygmo_bee_colony: Type[PygmoBeeColony] = PygmoBeeColony
     pygmo_cmaes: Type[PygmoCmaes] = PygmoCmaes
     pygmo_compass_search: Type[PygmoCompassSearch] = PygmoCompassSearch
@@ -3349,7 +3342,6 @@ class BoundedAlgorithms(AlgoSelection):
     nlopt_tnewton: Type[NloptTNewton] = NloptTNewton
     nlopt_var: Type[NloptVAR] = NloptVAR
     pounders: Type[Pounders] = Pounders
-    ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS
     pygmo_bee_colony: Type[PygmoBeeColony] = PygmoBeeColony
     pygmo_cmaes: Type[PygmoCmaes] = PygmoCmaes
     pygmo_compass_search: Type[PygmoCompassSearch] = PygmoCompassSearch

src/optimagic/optimizers/pyensmallen_optimizers.py

@@ -8,16 +8,28 @@
 
 from optimagic import mark
 from optimagic.optimization.algo_options import (
+    CONVERGENCE_FTOL_REL,
+    CONVERGENCE_GTOL_ABS,
     MAX_LINE_SEARCH_STEPS,
     STOPPING_MAXITER,
 )
 from optimagic.optimization.algorithm import Algorithm, InternalOptimizeResult
 from optimagic.optimization.internal_optimization_problem import (
     InternalOptimizationProblem,
 )
-from optimagic.typing import (
-    AggregationLevel,
-)
+from optimagic.typing import AggregationLevel, NonNegativeFloat, PositiveInt
+
+LIMITED_MEMORY_MAX_HISTORY = 10
+"""Number of memory points to be stored (default 10)."""
+MIN_LINE_SEARCH_STEPS = 1e-20
+"""The minimum step of the line search."""
+MAX_LINE_SEARCH_TRIALS = 50
+"""The maximum number of trials for the line search (before giving up)."""
+ARMIJO_CONSTANT = 1e-4
+"""Controls the accuracy of the line search routine for determining the Armijo
+condition."""
+WOLFE_CONDITION = 0.9
+"""Parameter for detecting the Wolfe condition."""
 
 
 @mark.minimizer(
@@ -28,35 +40,49 @@
     needs_jac=True,
     needs_hess=False,
     supports_parallelism=False,
-    supports_bounds=True,
+    supports_bounds=False,
     supports_linear_constraints=False,
     supports_nonlinear_constraints=False,
     disable_history=False,
 )
 @dataclass(frozen=True)
 class EnsmallenLBFGS(Algorithm):
-    stopping_maxiter = STOPPING_MAXITER
-    max_step_for_line_search = MAX_LINE_SEARCH_STEPS
-    # min_step_for_line_search = MIN_LINE_SEARCH_STEPS
+    limited_memory_max_history: PositiveInt = LIMITED_MEMORY_MAX_HISTORY
+    stopping_maxiter: PositiveInt = STOPPING_MAXITER
+    armijo_constant: NonNegativeFloat = ARMIJO_CONSTANT  # needs review
+    wolfe_condition: NonNegativeFloat = WOLFE_CONDITION  # needs review
+    convergence_gtol_abs: NonNegativeFloat = CONVERGENCE_GTOL_ABS
+    convergence_ftol_rel: NonNegativeFloat = CONVERGENCE_FTOL_REL
+    max_line_search_trials: PositiveInt = MAX_LINE_SEARCH_TRIALS
+    min_step_for_line_search: NonNegativeFloat = MIN_LINE_SEARCH_STEPS
+    max_step_for_line_search: NonNegativeFloat = MAX_LINE_SEARCH_STEPS
 
     def _solve_internal_problem(
         self, problem: InternalOptimizationProblem, x0: NDArray[np.float64]
     ) -> InternalOptimizeResult:
-        optimizer = pye.L_BFGS()
-
-        print(type(problem.fun(x0)))
+        optimizer = pye.L_BFGS(
+            numBasis=self.limited_memory_max_history,
+            maxIterations=self.stopping_maxiter,
+            armijoConstant=self.armijo_constant,
+            wolfe=self.wolfe_condition,
+            minGradientNorm=self.convergence_gtol_abs,
+            factr=self.convergence_ftol_rel,
+            maxLineSearchTrials=self.max_line_search_trials,
+            minStep=self.min_step_for_line_search,
+            maxStep=self.max_step_for_line_search,
+        )
 
         def objective_function(
             x: NDArray[np.float64], grad: NDArray[np.float64]
         ) -> np.float64:
             grad[:] = problem.jac(x)
             return np.float64(problem.fun(x))
 
-        raw_res = optimizer.optimize(objective_function, x0)
+        raw = optimizer.optimize(objective_function, x0)
 
         res = InternalOptimizeResult(
-            x=raw_res,
-            fun=problem.fun(raw_res),
+            x=raw,  # only best x is available
+            fun=problem.fun(raw),  # best f(x) value is not available
         )
 
         return res