Include useful context when logging a bad reduction

rnewson · rnewson · commit b2094abd5b38 · 2025-11-10T22:41:11.000Z
diff --git a/share/server/views.js b/share/server/views.js
@@ -16,7 +16,7 @@ var Views = (function() {
 
   var map_results = []; // holds temporary emitted values during doc map
 
-  function runReduce(reduceFuns, keys, values, rereduce) {
+  function runReduce(reduceFuns, keys, values, rereduce, ctx) {
     var code_size = 0;
     for (var i in reduceFuns) {
       var fun_body =  reduceFuns[i];
@@ -42,7 +42,8 @@ var Views = (function() {
       var log_message = [
           "Reduce output must shrink more rapidly:",
           "input size:", input_length,
-          "output size:", reduce_length
+          "output size:", reduce_length,
+          "context:", ctx
       ].join(" ");
       if (State.query_config.reduce_limit === "log") {
           log("reduce_overflow_error: " + log_message);
@@ -65,8 +66,8 @@ var Views = (function() {
       // fatal_error. But by default if they don't do error handling we
       // just eat the exception and carry on.
       //
-      // In this case we abort map processing but don't destroy the 
-      // JavaScript process. If you need to destroy the JavaScript 
+      // In this case we abort map processing but don't destroy the
+      // JavaScript process. If you need to destroy the JavaScript
       // process, throw the error form matched by the block below.
       throw(["error", "map_runtime_error", "function raised 'fatal_error'"]);
     } else if (err[0] == "fatal") {
@@ -93,17 +94,17 @@ var Views = (function() {
       }
       return rv;
     },
-    reduce : function(reduceFuns, kvs) {
+    reduce : function(reduceFuns, kvs, ctx) {
       var keys = new Array(kvs.length);
       var values = new Array(kvs.length);
       for(var i = 0; i < kvs.length; i++) {
           keys[i] = kvs[i][0];
           values[i] = kvs[i][1];
       }
-      runReduce(reduceFuns, keys, values, false);
+      runReduce(reduceFuns, keys, values, false, ctx);
     },
-    rereduce : function(reduceFuns, values) {
-      runReduce(reduceFuns, null, values, true);
+    rereduce : function(reduceFuns, values, ctx) {
+      runReduce(reduceFuns, null, values, true, ctx);
     },
     mapDoc : function(doc) {
       // Compute all the map functions against the document.
diff --git a/src/couch/src/couch_native_process.erl b/src/couch/src/couch_native_process.erl
@@ -183,7 +183,7 @@ run(State, [<<"map_doc">>, Doc]) ->
         State#evstate.funs
     ),
     {State, Resp};
-run(State, [<<"reduce">>, Funs, KVs]) ->
+run(State, [<<"reduce">>, Funs, KVs, _Ctx]) ->
     {Keys, Vals} =
         lists:foldl(
             fun([K, V], {KAcc, VAcc}) ->
@@ -195,7 +195,7 @@ run(State, [<<"reduce">>, Funs, KVs]) ->
     Keys2 = lists:reverse(Keys),
     Vals2 = lists:reverse(Vals),
     {State, catch reduce(State, Funs, Keys2, Vals2, false)};
-run(State, [<<"rereduce">>, Funs, Vals]) ->
+run(State, [<<"rereduce">>, Funs, Vals, _Ctx]) ->
     {State, catch reduce(State, Funs, null, Vals, true)};
 run(#evstate{ddocs = DDocs} = State, [<<"ddoc">>, <<"new">>, DDocId, DDoc]) ->
     DDocs2 = store_ddoc(DDocs, DDocId, DDoc),
diff --git a/src/couch/src/couch_query_servers.erl b/src/couch/src/couch_query_servers.erl
@@ -14,7 +14,7 @@
 
 -export([try_compile/4]).
 -export([start_doc_map/3, map_doc_raw/2, stop_doc_map/1, raw_to_ejson/1]).
--export([reduce/3, rereduce/3, validate_doc_update/6]).
+-export([reduce/3, reduce/4, rereduce/3, rereduce/4, validate_doc_update/6]).
 -export([filter_docs/5]).
 -export([filter_view/4]).
 -export([finalize/2]).
@@ -108,35 +108,43 @@ finalize(<<"_last", _/binary>>, {_K, _Id, V}) ->
 finalize(_RedSrc, Reduction) ->
     {ok, Reduction}.
 
-rereduce(_Lang, [], _ReducedValues) ->
+rereduce(Lang, RedSrcs, _ReducedValues) ->
+    rereduce(Lang, RedSrcs, _ReducedValues, <<>>).
+
+rereduce(_Lang, [], _ReducedValues, _Ctx) ->
     {ok, []};
-rereduce(Lang, RedSrcs, ReducedValues) ->
+rereduce(Lang, RedSrcs, ReducedValues, Ctx) ->
     Grouped = group_reductions_results(ReducedValues),
     Results = lists:zipwith(
         fun
             (<<"_", _/binary>> = FunSrc, Values) ->
-                {ok, [Result]} = builtin_reduce(rereduce, [FunSrc], [[[], V] || V <- Values], []),
+                {ok, [Result]} = builtin_reduce(
+                    rereduce, [FunSrc], [[[], V] || V <- Values], [], Ctx
+                ),
                 Result;
             (FunSrc, Values) ->
-                os_rereduce(Lang, [FunSrc], Values)
+                os_rereduce(Lang, [FunSrc], Values, Ctx)
         end,
         RedSrcs,
         Grouped
     ),
     {ok, Results}.
 
-reduce(_Lang, [], _KVs) ->
+reduce(Lang, RedSrcs, _KVs) ->
+    reduce(Lang, RedSrcs, _KVs, <<>>).
+
+reduce(_Lang, [], _KVs, _Ctx) ->
     {ok, []};
-reduce(Lang, RedSrcs, KVs) ->
+reduce(Lang, RedSrcs, KVs, Ctx) ->
     {OsRedSrcs, BuiltinReds} = lists:partition(
         fun
             (<<"_", _/binary>>) -> false;
             (_OsFun) -> true
         end,
         RedSrcs
     ),
-    {ok, OsResults} = os_reduce(Lang, OsRedSrcs, KVs),
-    {ok, BuiltinResults} = builtin_reduce(reduce, BuiltinReds, KVs, []),
+    {ok, OsResults} = os_reduce(Lang, OsRedSrcs, KVs, Ctx),
+    {ok, BuiltinResults} = builtin_reduce(reduce, BuiltinReds, KVs, [], Ctx),
     recombine_reduce_results(RedSrcs, OsResults, BuiltinResults, []).
 
 recombine_reduce_results([], [], [], Acc) ->
@@ -146,12 +154,12 @@ recombine_reduce_results([<<"_", _/binary>> | RedSrcs], OsResults, [BRes | Built
 recombine_reduce_results([_OsFun | RedSrcs], [OsR | OsResults], BuiltinResults, Acc) ->
     recombine_reduce_results(RedSrcs, OsResults, BuiltinResults, [OsR | Acc]).
 
-os_reduce(_Lang, [], _KVs) ->
+os_reduce(_Lang, [], _KVs, _Ctx) ->
     {ok, []};
-os_reduce(Lang, OsRedSrcs, KVs) ->
+os_reduce(Lang, OsRedSrcs, KVs, Ctx) ->
     Proc = get_os_process(Lang),
     OsResults =
-        try proc_prompt(Proc, [<<"reduce">>, OsRedSrcs, KVs]) of
+        try proc_prompt(Proc, [<<"reduce">>, OsRedSrcs, KVs, Ctx]) of
             [true, Reductions] -> Reductions
         catch
             throw:{reduce_overflow_error, Msg} ->
@@ -161,11 +169,11 @@ os_reduce(Lang, OsRedSrcs, KVs) ->
         end,
     {ok, OsResults}.
 
-os_rereduce(Lang, OsRedSrcs, KVs) ->
+os_rereduce(Lang, OsRedSrcs, KVs, Ctx) ->
     case get_overflow_error(KVs) of
         undefined ->
             Proc = get_os_process(Lang),
-            try proc_prompt(Proc, [<<"rereduce">>, OsRedSrcs, KVs]) of
+            try proc_prompt(Proc, [<<"rereduce">>, OsRedSrcs, KVs, Ctx]) of
                 [true, [Reduction]] -> Reduction
             catch
                 throw:{reduce_overflow_error, Msg} ->
@@ -184,36 +192,36 @@ get_overflow_error([{[{reduce_overflow_error, _}]} = Error | _]) ->
 get_overflow_error([_ | Rest]) ->
     get_overflow_error(Rest).
 
-builtin_reduce(_Re, [], _KVs, Acc) ->
+builtin_reduce(_Re, [], _KVs, Acc, _Ctx) ->
     {ok, lists:reverse(Acc)};
-builtin_reduce(Re, [<<"_sum", _/binary>> | BuiltinReds], KVs, Acc) ->
+builtin_reduce(Re, [<<"_sum", _/binary>> | BuiltinReds], KVs, Acc, Ctx) ->
     Sum = builtin_sum_rows(KVs, 0),
-    Red = check_sum_overflow(?term_size(KVs), ?term_size(Sum), Sum),
-    builtin_reduce(Re, BuiltinReds, KVs, [Red | Acc]);
-builtin_reduce(reduce, [<<"_count", _/binary>> | BuiltinReds], KVs, Acc) ->
+    Red = check_sum_overflow(?term_size(KVs), ?term_size(Sum), Sum, Ctx),
+    builtin_reduce(Re, BuiltinReds, KVs, [Red | Acc], Ctx);
+builtin_reduce(reduce, [<<"_count", _/binary>> | BuiltinReds], KVs, Acc, Ctx) ->
     Count = length(KVs),
-    builtin_reduce(reduce, BuiltinReds, KVs, [Count | Acc]);
-builtin_reduce(rereduce, [<<"_count", _/binary>> | BuiltinReds], KVs, Acc) ->
+    builtin_reduce(reduce, BuiltinReds, KVs, [Count | Acc], Ctx);
+builtin_reduce(rereduce, [<<"_count", _/binary>> | BuiltinReds], KVs, Acc, Ctx) ->
     Count = builtin_sum_rows(KVs, 0),
-    builtin_reduce(rereduce, BuiltinReds, KVs, [Count | Acc]);
-builtin_reduce(Re, [<<"_stats", _/binary>> | BuiltinReds], KVs, Acc) ->
+    builtin_reduce(rereduce, BuiltinReds, KVs, [Count | Acc], Ctx);
+builtin_reduce(Re, [<<"_stats", _/binary>> | BuiltinReds], KVs, Acc, Ctx) ->
     Stats = builtin_stats(Re, KVs),
-    builtin_reduce(Re, BuiltinReds, KVs, [Stats | Acc]);
-builtin_reduce(Re, [<<"_approx_count_distinct", _/binary>> | BuiltinReds], KVs, Acc) ->
+    builtin_reduce(Re, BuiltinReds, KVs, [Stats | Acc], Ctx);
+builtin_reduce(Re, [<<"_approx_count_distinct", _/binary>> | BuiltinReds], KVs, Acc, Ctx) ->
     Distinct = approx_count_distinct(Re, KVs),
-    builtin_reduce(Re, BuiltinReds, KVs, [Distinct | Acc]);
-builtin_reduce(Re, [<<"_top_", N/binary>> | BuiltinReds], KVs, Acc) ->
+    builtin_reduce(Re, BuiltinReds, KVs, [Distinct | Acc], Ctx);
+builtin_reduce(Re, [<<"_top_", N/binary>> | BuiltinReds], KVs, Acc, Ctx) ->
     Top = builtin_rank_n(Re, fun rank_fun_top/2, binary_to_integer(N), KVs),
-    builtin_reduce(Re, BuiltinReds, KVs, [Top | Acc]);
-builtin_reduce(Re, [<<"_bottom_", N/binary>> | BuiltinReds], KVs, Acc) ->
+    builtin_reduce(Re, BuiltinReds, KVs, [Top | Acc], Ctx);
+builtin_reduce(Re, [<<"_bottom_", N/binary>> | BuiltinReds], KVs, Acc, Ctx) ->
     Bottom = builtin_rank_n(Re, fun rank_fun_bottom/2, binary_to_integer(N), KVs),
-    builtin_reduce(Re, BuiltinReds, KVs, [Bottom | Acc]);
-builtin_reduce(Re, [<<"_first", _/binary>> | BuiltinReds], KVs, Acc) ->
+    builtin_reduce(Re, BuiltinReds, KVs, [Bottom | Acc], Ctx);
+builtin_reduce(Re, [<<"_first", _/binary>> | BuiltinReds], KVs, Acc, Ctx) ->
     First = builtin_first_last(Re, fun builtin_cmp_first/2, KVs),
-    builtin_reduce(Re, BuiltinReds, KVs, [First | Acc]);
-builtin_reduce(Re, [<<"_last", _/binary>> | BuiltinReds], KVs, Acc) ->
+    builtin_reduce(Re, BuiltinReds, KVs, [First | Acc], Ctx);
+builtin_reduce(Re, [<<"_last", _/binary>> | BuiltinReds], KVs, Acc, Ctx) ->
     Last = builtin_first_last(Re, fun builtin_cmp_last/2, KVs),
-    builtin_reduce(Re, BuiltinReds, KVs, [Last | Acc]).
+    builtin_reduce(Re, BuiltinReds, KVs, [Last | Acc], Ctx).
 
 builtin_sum_rows([], Acc) ->
     Acc;
@@ -278,28 +286,29 @@ sum_arrays([X | Xs], [Y | Ys]) when is_number(X), is_number(Y) ->
 sum_arrays(Else, _) ->
     throw_sum_error(Else).
 
-check_sum_overflow(InSize, OutSize, Sum) ->
+check_sum_overflow(InSize, OutSize, Sum, Ctx) ->
     Overflowed = OutSize > reduce_limit_threshold() andalso OutSize * reduce_limit_ratio() > InSize,
     case config:get("query_server_config", "reduce_limit", "true") of
         "true" when Overflowed ->
-            Msg = log_sum_overflow(InSize, OutSize),
+            Msg = log_sum_overflow(InSize, OutSize, Ctx),
             {[
                 {<<"error">>, <<"builtin_reduce_error">>},
                 {<<"reason">>, Msg}
             ]};
         "log" when Overflowed ->
-            log_sum_overflow(InSize, OutSize),
+            log_sum_overflow(InSize, OutSize, Ctx),
             Sum;
         _ ->
             Sum
     end.
 
-log_sum_overflow(InSize, OutSize) ->
+log_sum_overflow(InSize, OutSize, Ctx) ->
     Fmt =
         "Reduce output must shrink more rapidly: "
         "input size: ~b "
-        "output size: ~b",
-    Msg = iolist_to_binary(io_lib:format(Fmt, [InSize, OutSize])),
+        "output size: ~b "
+        "context: ~s",
+    Msg = iolist_to_binary(io_lib:format(Fmt, [InSize, OutSize, Ctx])),
     couch_log:error(Msg, []),
     Msg.
 
diff --git a/src/fabric/include/fabric.hrl b/src/fabric/include/fabric.hrl
@@ -12,6 +12,8 @@
 
 -record(collector, {
     db_name=nil,
+    ddoc_name=nil,
+    view_name=nil,
     query_args,
     callback,
     counters,
diff --git a/src/fabric/src/fabric_view.erl b/src/fabric/src/fabric_view.erl
@@ -285,7 +285,14 @@ get_next_row(#collector{reducer = RedSrc} = State0) when RedSrc =/= undefined ->
                 Records
             ),
             Wrapped = [[fabric_view_row:get_value(R)] || R <- Records],
-            {ok, [Reduced]} = couch_query_servers:rereduce(Lang, [RedSrc], Wrapped),
+            ReduceCtx = ?l2b(
+                io_lib:format("~s/~s/_view/~s", [
+                    State0#collector.db_name,
+                    State0#collector.ddoc_name,
+                    State0#collector.view_name
+                ])
+            ),
+            {ok, [Reduced]} = couch_query_servers:rereduce(Lang, [RedSrc], Wrapped, ReduceCtx),
             {ok, Finalized} = couch_query_servers:finalize(RedSrc, Reduced),
             State = State0#collector{keys = RestKeys, rows = RowDict, counters = Counters},
             ViewRow = fabric_view_row:from_props(
@@ -852,7 +859,7 @@ t_get_next_row_reduce(_) ->
     Row2 = {view_row, #{key => Key, id => reduced, value => finalized}},
     meck:expect(rexi, stream_ack, ['_'], meck:val(ok)),
     meck:expect(
-        couch_query_servers, rereduce, [Language, [reducer], Values], meck:val({ok, [reduced]})
+        couch_query_servers, rereduce, [Language, [reducer], Values, '_'], meck:val({ok, [reduced]})
     ),
     meck:expect(couch_query_servers, finalize, [reducer, reduced], meck:val({ok, finalized})),
     ?assertEqual({Row1, State3}, get_next_row(State1)),
diff --git a/src/fabric/src/fabric_view_reduce.erl b/src/fabric/src/fabric_view_reduce.erl
@@ -17,6 +17,7 @@
 -include_lib("fabric/include/fabric.hrl").
 -include_lib("mem3/include/mem3.hrl").
 -include_lib("couch_mrview/include/couch_mrview.hrl").
+-include_lib("couch/include/couch_db.hrl").
 
 go(DbName, GroupId, View, Args, Callback, Acc0, VInfo) when is_binary(GroupId) ->
     {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", GroupId/binary>>, []),
@@ -50,7 +51,7 @@ go(Db, DDoc, VName, Args, Callback, Acc, VInfo) ->
                 Callback({error, insufficient_storage}, Acc);
             {ok, Workers} ->
                 try
-                    go2(DbName, Workers, VInfo, CoordArgs, Callback, Acc)
+                    go2(DbName, DDoc#doc.id, VName, Workers, VInfo, CoordArgs, Callback, Acc)
                 after
                     fabric_streams:cleanup(Workers)
                 end;
@@ -64,7 +65,7 @@ go(Db, DDoc, VName, Args, Callback, Acc, VInfo) ->
         rexi_monitor:stop(RexiMon)
     end.
 
-go2(DbName, Workers, {red, {_, Lang, View}, _} = VInfo, Args, Callback, Acc0) ->
+go2(DbName, DDocId, VName, Workers, {red, {_, Lang, View}, _} = VInfo, Args, Callback, Acc0) ->
     #mrargs{limit = Limit, skip = Skip, keys = Keys, update_seq = UpdateSeq} = Args,
     RedSrc = couch_mrview_util:extract_view_reduce(VInfo),
     OsProc =
@@ -74,6 +75,8 @@ go2(DbName, Workers, {red, {_, Lang, View}, _} = VInfo, Args, Callback, Acc0) ->
         end,
     State = #collector{
         db_name = DbName,
+        ddoc_name = DDocId,
+        view_name = VName,
         query_args = Args,
         callback = Callback,
         counters = fabric_dict:init(Workers, 0),
