% Licensed under the Apache License, Version 2.0 (the "License"); you may not
% use this file except in compliance with the License. You may obtain a copy of
% the License at
%
%   http://www.apache.org/licenses/LICENSE-2.0
%
% Unless required by applicable law or agreed to in writing, software
% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
% License for the specific language governing permissions and limitations under
% the License.

-module(couch_set_view_updater).

-export([update/5]).
% Exported for unit tests only.
-export([convert_back_index_kvs_to_binary/2]).

-include("couch_db.hrl").
-include_lib("couch_set_view/include/couch_set_view.hrl").

-define(QUEUE_BASE_ITEMS, 2000).
-define(QUEUE_BASE_SIZE, 256 * 1024).
-define(MIN_WRITER_NUM_ITEMS, 1000).
-define(MIN_WRITER_BATCH_SIZE, 100 * 1024).
-define(CHECKPOINT_WRITE_INTERVAL, 5000000).
-define(MAX_SORT_BUFFER_SIZE, 1048576).
-define(MAX_SORT_NUM_FILES, 16).

% Same as in couch_btree.erl
-define(KEY_BITS,       12).
-define(MAX_KEY_SIZE,   ((1 bsl ?KEY_BITS) - 1)).

-define(replace(L, K, V), lists:keystore(K, 1, L, {K, V})).

-record(writer_acc, {
    parent,
    owner,
    group,
    log_fd = nil,
    write_queue,
    initial_build,
    view_empty_kvs,
    kvs = [],
    kvs_size = 0,
    kvs_length = 0,
    state = updating_active,
    final_batch = false,
    max_seqs,
    replicas_transferred = [],
    cleanup_kv_count = 0,
    cleanup_time = 0,
    inserted_kvs = 0,
    deleted_kvs = 0,
    inserted_ids = 0,
    deleted_ids = 0,
    merge_buffers = nil,
    tmp_dir = nil,
    sort_files = nil,
    sort_file_workers = [],
    write_queue_size
}).


-spec update(pid(), #set_view_group{},
             partition_seqs(), string() | 'nil', string()) -> no_return().
update(Owner, Group, CurSeqs, LogFilePath, TmpDir) ->
    #set_view_group{
        set_name = SetName,
        type = Type,
        name = DDocId
    } = Group,
    ActiveParts = couch_set_view_util:decode_bitmask(?set_abitmask(Group)),
    PassiveParts = couch_set_view_util:decode_bitmask(?set_pbitmask(Group)),
    NumChanges = couch_set_view_util:missing_changes_count(CurSeqs, ?set_seqs(Group)),

    process_flag(trap_exit, true),

    BeforeEnterTs = os:timestamp(),
    Parent = self(),
    BarrierEntryPid = spawn_link(fun() ->
        DDocIds = couch_set_view_util:get_ddoc_ids_with_sig(SetName, Group),
        couch_task_status:add_task([
            {type, blocked_indexer},
            {set, SetName},
            {signature, ?l2b(couch_util:to_hex(Group#set_view_group.sig))},
            {design_documents, DDocIds},
            {indexer_type, Type}
        ]),
        case Type of
        main ->
            ok = couch_index_barrier:enter(couch_main_index_barrier, Parent);
        replica ->
            ok = couch_index_barrier:enter(couch_replica_index_barrier, Parent)
        end,
        Parent ! {done, self(), (timer:now_diff(os:timestamp(), BeforeEnterTs) / 1000000)},
        receive shutdown -> ok end
    end),

    BlockedTime = receive
    {done, BarrierEntryPid, Duration} ->
        Duration;
    {'EXIT', _, Reason} ->
        exit({updater_error, Reason})
    end,

    CleanupParts = couch_set_view_util:decode_bitmask(?set_cbitmask(Group)),
    InitialBuild = couch_set_view_util:is_group_empty(Group),
    ?LOG_INFO("Updater for set view `~s`, ~s group `~s` started~n"
              "Active partitions:    ~w~n"
              "Passive partitions:   ~w~n"
              "Cleanup partitions:   ~w~n"
              "Replicas to transfer: ~w~n"
              "Pending transition:   ~n"
              "    active:           ~w~n"
              "    passive:          ~w~n"
              "    unindexable:      ~w~n"
              "Initial build:        ~s~n",
              [SetName, Type, DDocId,
               ActiveParts,
               PassiveParts,
               CleanupParts,
               ?set_replicas_on_transfer(Group),
               ?pending_transition_active(?set_pending_transition(Group)),
               ?pending_transition_passive(?set_pending_transition(Group)),
               ?pending_transition_unindexable(?set_pending_transition(Group)),
               InitialBuild
              ]),

    WriterAcc0 = #writer_acc{
        parent = self(),
        owner = Owner,
        group = Group,
        initial_build = InitialBuild,
        max_seqs = CurSeqs,
        tmp_dir = TmpDir
    },
    update(WriterAcc0, ActiveParts, PassiveParts, BlockedTime, BarrierEntryPid, NumChanges, LogFilePath).


update(WriterAcc, ActiveParts, PassiveParts, BlockedTime, BarrierEntryPid, NumChanges, LogFilePath) ->
    #writer_acc{
        owner = Owner,
        group = Group
    } = WriterAcc,
    #set_view_group{
        set_name = SetName,
        type = Type,
        name = DDocId,
        sig = GroupSig,
        views = Views
    } = Group,

    StartTime = os:timestamp(),

    MapQueueOptions = [{max_size, ?QUEUE_BASE_SIZE}, {max_items, ?QUEUE_BASE_ITEMS}],
    WriteQueueOptions = case WriterAcc#writer_acc.initial_build of
    true ->
        [{max_size, ?MAX_SORT_BUFFER_SIZE div 2},
         {max_items, infinity}];
    false ->
        [{max_size, ?QUEUE_BASE_SIZE * (length(Views) + 1)},
         {max_items, ?QUEUE_BASE_ITEMS * (length(Views) + 1)}]
    end,
    {ok, MapQueue} = couch_work_queue:new(MapQueueOptions),
    {ok, WriteQueue} = couch_work_queue:new(WriteQueueOptions),

    Mapper = spawn_link(fun() ->
        try
            couch_set_view_mapreduce:start_map_context(Group),
            try
                do_maps(Group, MapQueue, WriteQueue)
            after
                couch_set_view_mapreduce:end_map_context()
            end
        catch _:Error ->
            Stacktrace = erlang:get_stacktrace(),
            ?LOG_ERROR("Set view `~s`, ~s group `~s`, mapper error~n"
                "error:      ~p~n"
                "stacktrace: ~p~n",
                [SetName, Type, DDocId, Error, Stacktrace]),
            exit(Error)
        end
    end),

    Parent = self(),
    unlink(BarrierEntryPid),
    Writer = spawn_link(fun() ->
        DDocIds = couch_set_view_util:get_ddoc_ids_with_sig(SetName, Group),
        couch_task_status:add_task([
            {type, indexer},
            {set, SetName},
            {signature, ?l2b(couch_util:to_hex(GroupSig))},
            {design_documents, DDocIds},
            {indexer_type, Type},
            {progress, 0},
            {changes_done, 0},
            {initial_build, WriterAcc#writer_acc.initial_build},
            {total_changes, NumChanges}
        ]),
        BarrierEntryPid ! shutdown,
        couch_task_status:set_update_frequency(5000),

        ViewEmptyKVs = [{View, []} || View <- Group#set_view_group.views],
        WriterAcc2 = init_view_merge_params(WriterAcc#writer_acc{
            parent = Parent,
            group = Group,
            write_queue = WriteQueue,
            view_empty_kvs = ViewEmptyKVs,
            log_fd = open_log_file(LogFilePath),
            write_queue_size = couch_util:get_value(max_size, WriteQueueOptions)
        }),
        delete_prev_sort_files(WriterAcc2),
        ok = couch_set_view_util:open_raw_read_fd(Group),
        try
            couch_set_view_mapreduce:start_reduce_context(Group),
            try
                FinalWriterAcc = do_writes(WriterAcc2),
                case FinalWriterAcc#writer_acc.log_fd of
                nil ->
                    ok;
                _ ->
                    ok = file:close(FinalWriterAcc#writer_acc.log_fd)
                end,
                Parent ! {writer_finished, FinalWriterAcc}
            after
                couch_set_view_mapreduce:end_reduce_context(Group)
            end
        catch _:Error ->
            Stacktrace = erlang:get_stacktrace(),
            ?LOG_ERROR("Set view `~s`, ~s group `~s`, writer error~n"
                "error:      ~p~n"
                "stacktrace: ~p~n",
                [SetName, Type, DDocId, Error, Stacktrace]),
            exit(Error)
        after
            ok = couch_set_view_util:close_raw_read_fd(Group)
        end
    end),

    DocLoader = spawn_link(fun() ->
        try
            load_changes(Owner, Parent, Group, MapQueue, Writer, ActiveParts, PassiveParts)
        catch _:Error ->
            Stacktrace = erlang:get_stacktrace(),
            ?LOG_ERROR("Set view `~s`, ~s group `~s`, doc loader error~n"
                "error:      ~p~n"
                "stacktrace: ~p~n",
                [SetName, Type, DDocId, Error, Stacktrace]),
            exit(Error)
        end
    end),

    Result = wait_result_loop(StartTime, DocLoader, Mapper, Writer, BlockedTime, Group),
    case Type of
    main ->
        ok = couch_index_barrier:leave(couch_main_index_barrier);
    replica ->
        ok = couch_index_barrier:leave(couch_replica_index_barrier)
    end,
    case Result of
    {updater_finished, #set_view_updater_result{group = NewGroup}} ->
        ?LOG_DEBUG("Updater for ~s set view group `~s`, set `~s`, writer finished:~n"
                   "  start seqs: ~w~n"
                   "  end seqs:   ~w~n",
                   [Type, DDocId, SetName, ?set_seqs(Group), ?set_seqs(NewGroup)]);
    _ ->
        ok
    end,
    exit(Result).


wait_result_loop(StartTime, DocLoader, Mapper, Writer, BlockedTime, OldGroup) ->
    #set_view_group{set_name = SetName, name = DDocId, type = Type} = OldGroup,
    receive
    {writer_finished, WriterAcc} ->
        Result = #set_view_updater_result{
            group = WriterAcc#writer_acc.group,
            indexing_time = timer:now_diff(os:timestamp(), StartTime) / 1000000,
            blocked_time = BlockedTime,
            state = WriterAcc#writer_acc.state,
            cleanup_kv_count = WriterAcc#writer_acc.cleanup_kv_count,
            cleanup_time = WriterAcc#writer_acc.cleanup_time,
            inserted_ids = WriterAcc#writer_acc.inserted_ids,
            deleted_ids = WriterAcc#writer_acc.deleted_ids,
            inserted_kvs = WriterAcc#writer_acc.inserted_kvs,
            deleted_kvs = WriterAcc#writer_acc.deleted_kvs
        },
        {updater_finished, Result};
    {log_new_changes, Pid, Ref, LogFilePath} ->
        Writer ! {log_new_changes, self(), LogFilePath},
        ?LOG_INFO("Set view `~s`, ~s group `~s`, updater received log "
                  "request from compactor ~p, ref ~p, writer ~p",
                   [SetName, Type, DDocId, Pid, Ref, Writer]),
        erlang:put(log_request, {Pid, Ref}),
        wait_result_loop(StartTime, DocLoader, Mapper, Writer, BlockedTime, OldGroup);
    {log_started, Writer, GroupSnapshot} ->
        ?LOG_INFO("Set view `~s`, ~s group `~s`, updater received log started "
                  "from writer ~p", [SetName, Type, DDocId, Writer]),
        {Pid, Ref} = erlang:erase(log_request),
        Pid ! {Ref, {ok, GroupSnapshot}},
        wait_result_loop(StartTime, DocLoader, Mapper, Writer, BlockedTime, OldGroup);
    {'EXIT', _, Reason} when Reason =/= normal ->
        couch_util:shutdown_sync(DocLoader),
        couch_util:shutdown_sync(Mapper),
        couch_util:shutdown_sync(Writer),
        {updater_error, Reason}
    end.


load_changes(Owner, Updater, Group, MapQueue, Writer, ActiveParts, PassiveParts) ->
    #set_view_group{
        set_name = SetName,
        name = DDocId,
        type = GroupType,
        index_header = #set_view_index_header{seqs = SinceSeqs}
    } = Group,

    FoldFun = fun(PartId, {AccCount, AccSeqs}) ->
        case couch_set_view_util:has_part_seq(PartId, ?set_unindexable_seqs(Group)) of
        true ->
            {AccCount, AccSeqs};
        false ->
            Db = case couch_db:open_int(?dbname(SetName, PartId), []) of
            {ok, PartDb} ->
                PartDb;
            Error ->
                ErrorMsg = io_lib:format("Updater error opening database `~s': ~w",
                                         [?dbname(SetName, PartId), Error]),
                throw({error, iolist_to_binary(ErrorMsg)})
            end,
            try
                Since = couch_util:get_value(PartId, SinceSeqs),
                ChangesWrapper = fun(DocInfo, _, AccCount2) ->
                    load_doc(Db, PartId, DocInfo, MapQueue, Group),
                    {ok, AccCount2 + 1}
                end,
                {ok, _, AccCount3} = couch_db:fast_reads(Db, fun() ->
                    couch_db:enum_docs_since(Db, Since, ChangesWrapper, AccCount, [])
                end),
                {AccCount3, orddict:store(PartId, Db#db.update_seq, AccSeqs)}
            after
                ok = couch_db:close(Db)
            end
        end
    end,

    notify_owner(Owner, {state, updating_active}, Updater),
    try
        case ActiveParts of
        [] ->
            ActiveChangesCount = 0,
            MaxSeqs = orddict:new();
        _ ->
            ?LOG_INFO("Updater reading changes from active partitions to "
                      "update ~s set view group `~s` from set `~s`",
                      [GroupType, DDocId, SetName]),
            {ActiveChangesCount, MaxSeqs} = lists:foldl(
                FoldFun, {0, orddict:new()}, ActiveParts)
        end,
        case PassiveParts of
        [] ->
            FinalChangesCount = ActiveChangesCount,
            MaxSeqs2 = MaxSeqs;
        _ ->
            ?LOG_INFO("Updater reading changes from passive partitions to "
                      "update ~s set view group `~s` from set `~s`",
                      [GroupType, DDocId, SetName]),
            {FinalChangesCount, MaxSeqs2} = lists:foldl(
                FoldFun, {ActiveChangesCount, MaxSeqs}, PassiveParts)
        end,
        ?LOG_INFO("Updater for ~s set view group `~s`, set `~s`, read a total of ~p changes",
                  [GroupType, DDocId, SetName, FinalChangesCount]),
        ?LOG_DEBUG("Updater for ~s set view group `~s`, set `~s`, max partition seqs found:~n~w",
                  [GroupType, DDocId, SetName, MaxSeqs2])
    catch throw:stop ->
        Writer ! stop
    end,
    couch_work_queue:close(MapQueue).


notify_owner(Owner, Msg, UpdaterPid) ->
    Owner ! {updater_info, UpdaterPid, Msg}.


load_doc(Db, PartitionId, DocInfo, MapQueue, Group) ->
    #doc_info{id=DocId, local_seq=Seq, deleted=Deleted} = DocInfo,
    case DocId of
    <<?DESIGN_DOC_PREFIX, _/binary>> ->
        ok;
    _ ->
        if Deleted ->
            couch_work_queue:queue(MapQueue, {Seq, #doc{id=DocId, deleted=true}, PartitionId});
        true ->
            case couch_util:validate_utf8(DocId) of
            true ->
                {ok, Doc} = couch_db:open_doc_int(Db, DocInfo, []),
                couch_work_queue:queue(MapQueue, {Seq, Doc, PartitionId});
            false ->
                #set_view_group{
                   set_name = SetName,
                   name = DDocId,
                   type = GroupType
               } = Group,
                % If the id isn't utf8 (memcached allows it), then log an error
                % message and skip the doc. Send it through the queue anyway
                % so we record the high seq num in case there are a bunch of
                % these at the end, we want to keep track of the high seq and
                % not reprocess again.
                ?LOG_MAPREDUCE_ERROR("Bucket `~s`, ~s group `~s`, skipping "
                    "document with non-utf8 id. Doc id bytes: ~w",
                    [SetName, GroupType, DDocId, ?b2l(DocId)]),
                couch_work_queue:queue(MapQueue, {Seq, #doc{id=DocId, deleted=true}, PartitionId})
            end
        end
    end.


do_maps(Group, MapQueue, WriteQueue) ->
    case couch_work_queue:dequeue(MapQueue) of
    closed ->
        couch_work_queue:close(WriteQueue);
    {ok, Queue, _QueueSize} ->
        Items = lists:foldr(
            fun({Seq, #doc{id = Id, deleted = true}, PartitionId}, Acc) ->
                Item = {Seq, Id, PartitionId, []},
                [Item | Acc];
            ({Seq, #doc{id = Id, deleted = false} = Doc, PartitionId}, Acc) ->
                try
                    {ok, Result} = couch_set_view_mapreduce:map(Doc),
                    Item = {Seq, Id, PartitionId, Result},
                    [Item | Acc]
                catch _:{error, Reason} ->
                    #set_view_group{
                        set_name = SetName,
                        name = DDocId,
                        type = Type
                    } = Group,
                    ErrorMsg = "Bucket `~s`, ~s group `~s`, error mapping document `~s`: ~s",
                    Args = [SetName, Type, DDocId, Id, couch_util:to_binary(Reason)],
                    ?LOG_MAPREDUCE_ERROR(ErrorMsg, Args),
                    [{Seq, Id, PartitionId, []} | Acc]
                end
            end,
            [], Queue),
        ok = couch_work_queue:queue(WriteQueue, Items),
        do_maps(Group, MapQueue, WriteQueue)
    end.


do_writes(Acc) ->
    #writer_acc{
        kvs = Kvs,
        kvs_size = KvsSize,
        kvs_length = KvsLength,
        write_queue = WriteQueue
    } = Acc,
    Acc2 = maybe_open_log_file(Acc),
    case couch_work_queue:dequeue(WriteQueue) of
    closed ->
        FinalAcc = flush_writes(Acc2#writer_acc{final_batch = true}),
        write_header(FinalAcc#writer_acc.group, false),
        FinalAcc;
    {ok, Queue0, QueueSize} ->
        Queue = lists:flatten(Queue0),
        Acc3 = maybe_open_log_file(Acc2),
        Kvs2 = Kvs ++ Queue,
        KvsSize2 = KvsSize + QueueSize,
        KvsLength2 = KvsLength + length(Queue),
        Acc4 = Acc3#writer_acc{
            kvs = Kvs2,
            kvs_size = KvsSize2,
            kvs_length = KvsLength2
        },
        case should_flush_writes(Acc4) of
        true ->
            Acc5 = flush_writes(Acc4),
            Acc6 = Acc5#writer_acc{kvs = [], kvs_size = 0, kvs_length = 0};
        false ->
            Acc6 = Acc4
        end,
        do_writes(Acc6)
    end.


should_flush_writes(#writer_acc{initial_build = true} = Acc) ->
    #writer_acc{kvs_size = KvsSize, write_queue_size = Wqsz} = Acc,
    KvsSize >= Wqsz;
should_flush_writes(#writer_acc{initial_build = false} = Acc) ->
    #writer_acc{kvs_size = KvsSize, kvs_length = KvsLength} = Acc,
    (KvsSize >= ?MIN_WRITER_BATCH_SIZE) orelse (KvsLength >= ?MIN_WRITER_NUM_ITEMS).


flush_writes(#writer_acc{kvs = [], initial_build = false} = Acc) ->
    {Acc2, ReplicasTransferred} = update_transferred_replicas(Acc, []),
    case ReplicasTransferred of
    true ->
        checkpoint(Acc2, true);
    false ->
        ok
    end,
    Acc2;

flush_writes(#writer_acc{initial_build = false} = Acc) ->
    #writer_acc{
        kvs = Kvs,
        view_empty_kvs = ViewEmptyKVs,
        group = Group,
        parent = Parent,
        owner = Owner
    } = Acc,
    {ViewKVs, DocIdViewIdKeys, PartIdSeqs} = process_map_results(Kvs, ViewEmptyKVs, orddict:new()),
    Acc2 = write_changes(Acc, ViewKVs, DocIdViewIdKeys, PartIdSeqs),
    {Acc3, ReplicasTransferred} = update_transferred_replicas(Acc2, PartIdSeqs),
    update_task(Acc#writer_acc.kvs_length),
    case (Acc3#writer_acc.state =:= updating_active) andalso
        lists:any(fun({PartId, _}) ->
            ((1 bsl PartId) band ?set_pbitmask(Group) =/= 0)
        end, PartIdSeqs) of
    true ->
        checkpoint(Acc3, false),
        notify_owner(Owner, {state, updating_passive}, Parent),
        Acc3#writer_acc{state = updating_passive};
    false when ReplicasTransferred ->
        checkpoint(Acc3, true),
        Acc3;
    false ->
        case ?set_cbitmask(Acc3#writer_acc.group) /= ?set_cbitmask(Group) of
        true ->
            checkpoint(Acc3, false);
        false ->
            maybe_checkpoint(Acc3)
        end,
        Acc3
    end;

flush_writes(#writer_acc{initial_build = true} = WriterAcc) ->
    #writer_acc{
        kvs = Kvs,
        kvs_length = KvsLength,
        view_empty_kvs = ViewEmptyKVs,
        merge_buffers = Buffers,
        group = Group,
        final_batch = IsFinalBatch,
        max_seqs = MaxSeqs
    } = WriterAcc,
    #set_view_group{
        id_btree = IdBtree,
        set_name = SetName,
        type = Type,
        name = DDocId,
        fd = GroupFd
    } = Group,
    {ViewKVs, DocIdViewIdKeys, MaxSeqs2} = process_map_results(Kvs, ViewEmptyKVs, MaxSeqs),
    {IdBuffer, IdBufferSize} = dict:fetch(ids_index, Buffers),
    {NewIdBuffer0, NewIdBufferSize} = lists:foldr(
        fun({_DocId, {_PartId, []}}, Acc) ->
            Acc;
        (Kv, {AccBuf, AccSize}) ->
            [{KeyBin, ValBin}] = convert_back_index_kvs_to_binary([Kv], []),
            KvBin = [<<(byte_size(KeyBin)):16>>, KeyBin, ValBin],
            KvBinSize = iolist_size(KvBin),
            {[[<<KvBinSize:32>>, KvBin] | AccBuf], AccSize + KvBinSize}
        end,
        {IdBuffer, IdBufferSize},
        DocIdViewIdKeys),
    NewIdBuffer = case (NewIdBufferSize >= ?MAX_SORT_BUFFER_SIZE) orelse IsFinalBatch of
    true ->
        IdLessFun = fun([_, [_, A, _]], [_, [_, B, _]]) -> (IdBtree#btree.less)(A, B) end,
        lists:sort(IdLessFun, NewIdBuffer0);
    false ->
        NewIdBuffer0
    end,
    {NewBuffers0, InsertKVCount} = lists:foldl(
        fun({#set_view{id_num = Id}, KvList}, {AccBuffers, AccCount}) ->
            {Buf, BufSize} = dict:fetch(Id, AccBuffers),
            {NewBuf, NewBufSize, AccCount3} = lists:foldr(
                fun({KeyBin, ValBin}, {AccBuf, AccBufSize, AccCount2}) ->
                    KvBin = [<<(byte_size(KeyBin)):16>>, KeyBin, ValBin],
                    KvBinSize = iolist_size(KvBin),
                    AccBuf2 = [[<<KvBinSize:32>>, KvBin] | AccBuf],
                    {AccBuf2, AccBufSize + KvBinSize, AccCount2 + 1}
                end,
                {Buf, BufSize, AccCount},
                convert_primary_index_kvs_to_binary(KvList, Group, [])),
            {dict:store(Id, {NewBuf, NewBufSize}, AccBuffers), AccCount3}
        end,
        {dict:store(ids_index, {NewIdBuffer, NewIdBufferSize}, Buffers), 0},
        ViewKVs),

    NewBuffers = lists:foldl(
        fun(#set_view{id_num = Id, btree = Bt}, AccBuffers) ->
            {Buf, BufSize} = dict:fetch(Id, AccBuffers),
            case (BufSize >= ?MAX_SORT_BUFFER_SIZE) orelse IsFinalBatch of
            true ->
                ViewLessFun = fun([_, [_, A, _]], [_, [_, B, _]]) -> (Bt#btree.less)(A, B) end,
                Buf2 = lists:sort(ViewLessFun, Buf),
                dict:store(Id, {Buf2, BufSize}, AccBuffers);
            false ->
                AccBuffers
            end
        end,
        NewBuffers0,
        Group#set_view_group.views),

    {NewBuffers2, NewSortFiles2, SortFileWorkers2} =
        maybe_flush_merge_buffers(NewBuffers, WriterAcc),
    update_task(KvsLength),
    case IsFinalBatch of
    false ->
        WriterAcc#writer_acc{
            merge_buffers = NewBuffers2,
            sort_files = NewSortFiles2,
            sort_file_workers = SortFileWorkers2,
            max_seqs = MaxSeqs2,
            inserted_kvs = WriterAcc#writer_acc.inserted_kvs + InsertKVCount,
            inserted_ids = WriterAcc#writer_acc.inserted_ids + length(DocIdViewIdKeys)
        };
    true ->
        ?LOG_INFO("Updater for set view `~s`, ~s group `~s`, performing final "
                  "btree build phase", [SetName, Type, DDocId]),
        wait_for_workers(SortFileWorkers2),
        [{_, IdsSortedFile}] = dict:fetch(ids_index, NewSortFiles2),
        {ok, NewIdBtreeRoot} = couch_btree_copy:from_sorted_file(
            IdBtree, IdsSortedFile, GroupFd, fun file_sorter_format_function/1),
        NewIdBtree = IdBtree#btree{root = NewIdBtreeRoot},
        ok = file2:delete(IdsSortedFile),
        NewViews = lists:map(
            fun(#set_view{id_num = Id, btree = Bt} = View) ->
               [{_, KvSortedFile}] = dict:fetch(Id, NewSortFiles2),
               {ok, NewBtRoot} = couch_btree_copy:from_sorted_file(
                   Bt, KvSortedFile, GroupFd, fun file_sorter_format_function/1),
               ok = file2:delete(KvSortedFile),
               View#set_view{
                   btree = Bt#btree{root = NewBtRoot}
               }
            end,
            Group#set_view_group.views),
        Header = Group#set_view_group.index_header,
        NewHeader = Header#set_view_index_header{
            id_btree_state = couch_btree:get_state(NewIdBtree),
            view_states = [couch_btree:get_state(V#set_view.btree) || V <- NewViews],
            seqs = MaxSeqs2
        },
        update_task(1),
        FinalAcc = WriterAcc#writer_acc{
            sort_files = nil,
            sort_file_workers = [],
            max_seqs = MaxSeqs2,
            inserted_kvs = WriterAcc#writer_acc.inserted_kvs + InsertKVCount,
            inserted_ids = WriterAcc#writer_acc.inserted_ids + length(DocIdViewIdKeys),
            group = Group#set_view_group{
                id_btree = NewIdBtree,
                views = NewViews,
                index_header = NewHeader
            }
        },
        write_header(FinalAcc#writer_acc.group, false),
        ok = couch_file:flush(GroupFd),
        FinalAcc
    end.


process_map_results(Kvs, ViewEmptyKVs, PartSeqs) ->
    lists:foldl(
        fun({Seq, DocId, PartId, []}, {ViewKVsAcc, DocIdViewIdKeysAcc, PartIdSeqs}) ->
            PartIdSeqs2 = update_part_seq(Seq, PartId, PartIdSeqs),
            {ViewKVsAcc, [{DocId, {PartId, []}} | DocIdViewIdKeysAcc], PartIdSeqs2};
        ({Seq, DocId, PartId, QueryResults}, {ViewKVsAcc, DocIdViewIdKeysAcc, PartIdSeqs}) ->
            {NewViewKVs, NewViewIdKeys} = view_insert_doc_query_results(
                    DocId, PartId, QueryResults, ViewKVsAcc, [], []),
            PartIdSeqs2 = update_part_seq(Seq, PartId, PartIdSeqs),
            {NewViewKVs, [{DocId, {PartId, NewViewIdKeys}} | DocIdViewIdKeysAcc], PartIdSeqs2}
        end,
        {ViewEmptyKVs, [], PartSeqs}, Kvs).


maybe_flush_merge_buffers(BuffersDict, WriterAcc) ->
    #writer_acc{
        view_empty_kvs = ViewEmptyKVs,
        sort_files = SortFilesDict,
        sort_file_workers = Workers,
        tmp_dir = TmpDir,
        group = #set_view_group{id_btree = IdBtree},
        final_batch = IsFinalBatch
    } = WriterAcc,
    ViewInfos = [
        {ids_index, IdBtree#btree.less} |
        [{V#set_view.id_num, (V#set_view.btree)#btree.less} || {V, _} <- ViewEmptyKVs]
    ],
    NumBtrees = length(ViewEmptyKVs) + 1,
    lists:foldl(
        fun({Id, Less}, {AccBuffers, AccFiles, AccWorkers}) ->
            {Buf, BufSize} = dict:fetch(Id, AccBuffers),
            case (BufSize >= ?MAX_SORT_BUFFER_SIZE) orelse IsFinalBatch of
            true ->
                SortFiles = dict:fetch(Id, AccFiles),
                FileName = new_sort_file_name(WriterAcc),
                {ok, Fd} = file2:open(FileName, [raw, append, binary]),
                ok = file:write(Fd, Buf),
                ok = file:close(Fd),
                AccBuffers2 = dict:store(Id, {[], 0}, AccBuffers),
                SortFiles2 = ordsets:add_element({0, FileName}, SortFiles),
                case (ordsets:size(SortFiles2) >= ?MAX_SORT_NUM_FILES) orelse IsFinalBatch of
                true ->
                    LessFun = fun({A, _}, {B, _}) -> Less(A, B) end,
                    MergeSortFile = new_sort_file_name(WriterAcc),
                    {FilesToMerge, NextGen, RestSortFiles} =
                        split_files_to_merge(SortFiles2, IsFinalBatch),
                    case FilesToMerge of
                    [_] ->
                        {AccBuffers2, dict:store(Id, SortFiles2, AccFiles), AccWorkers};
                    [_, _ | _] ->
                        case length(AccWorkers) >= NumBtrees of
                        true ->
                            [OldestWorker | RestWorkersRev] = lists:reverse(AccWorkers),
                            wait_for_workers([OldestWorker]),
                            AccWorkers2 = lists:reverse(RestWorkersRev);
                        false ->
                            AccWorkers2 = AccWorkers
                        end,
                        MergeWorker = spawn_merge_worker(
                            LessFun, TmpDir, AccWorkers2, FilesToMerge, MergeSortFile),
                        SortFiles3 = ordsets:add_element({NextGen, MergeSortFile}, RestSortFiles),
                        {AccBuffers2, dict:store(Id, SortFiles3, AccFiles), [MergeWorker | AccWorkers2]}
                    end;
                false ->
                    {AccBuffers2, dict:store(Id, SortFiles2, AccFiles), AccWorkers}
                end;
            false ->
                {AccBuffers, AccFiles, AccWorkers}
            end
        end,
        {BuffersDict, SortFilesDict, Workers},
        ViewInfos).


update_transferred_replicas(#writer_acc{group = Group} = Acc, _PartIdSeqs) when ?set_replicas_on_transfer(Group) =:= [] ->
    {Acc, false};
update_transferred_replicas(Acc, PartIdSeqs) ->
    #writer_acc{
        group = #set_view_group{index_header = Header} = Group,
        max_seqs = MaxSeqs,
        replicas_transferred = RepsTransferred,
        final_batch = FinalBatch
    } = Acc,
    RepsTransferred2 = lists:foldl(
        fun({PartId, Seq}, A) ->
            case lists:member(PartId, ?set_replicas_on_transfer(Group))
                andalso (Seq >= couch_set_view_util:get_part_seq(PartId, MaxSeqs)) of
            true ->
                ordsets:add_element(PartId, A);
            false ->
                A
            end
        end,
        RepsTransferred, PartIdSeqs),
    % Only update the group's list of replicas on transfer when the updater is finishing
    % or when all the replicas were transferred (indexed). This is to make the cleanup
    % of the replica index much more efficient (less partition state transitions, less
    % cleanup process interruptions/restarts).
    ReplicasOnTransfer = ordsets:subtract(?set_replicas_on_transfer(Group), RepsTransferred2),
    case FinalBatch orelse (ReplicasOnTransfer =:= []) of
    false ->
        {Acc#writer_acc{replicas_transferred = RepsTransferred2}, false};
    true ->
        {Abitmask2, Pbitmask2} = lists:foldl(
            fun(Id, {A, P}) ->
                Mask = 1 bsl Id,
                Mask = ?set_pbitmask(Group) band Mask,
                0 = ?set_abitmask(Group) band Mask,
                {A bor Mask, P bxor Mask}
            end,
            {?set_abitmask(Group), ?set_pbitmask(Group)},
            RepsTransferred2),
        Group2 = Group#set_view_group{
            index_header = Header#set_view_index_header{
                abitmask = Abitmask2,
                pbitmask = Pbitmask2,
                replicas_on_transfer = ReplicasOnTransfer
            }
        },
        {Acc#writer_acc{group = Group2}, ReplicasOnTransfer /= []}
    end.


update_part_seq(Seq, PartId, Acc) ->
    case couch_set_view_util:find_part_seq(PartId, Acc) of
    {ok, Max} when Max >= Seq ->
        Acc;
    _ ->
        orddict:store(PartId, Seq, Acc)
    end.


view_insert_doc_query_results(_DocId, _PartitionId, [], [], ViewKVsAcc, ViewIdKeysAcc) ->
    {lists:reverse(ViewKVsAcc), lists:reverse(ViewIdKeysAcc)};
view_insert_doc_query_results(DocId, PartitionId, [ResultKVs | RestResults],
        [{View, KVs} | RestViewKVs], ViewKVsAcc, ViewIdKeysAcc) ->
    % Take any identical keys and combine the values
    {NewKVs, NewViewIdKeysAcc} = lists:foldl(
        fun({Key, Val}, {[{{Key, PrevDocId} = Kd, PrevVal} | AccRest], AccVid}) when PrevDocId =:= DocId ->
            AccKv2 = case PrevVal of
            {PartitionId, {dups, Dups}} ->
                [{Kd, {PartitionId, {dups, [Val | Dups]}}} | AccRest];
            {PartitionId, UserPrevVal} ->
                [{Kd, {PartitionId, {dups, [Val, UserPrevVal]}}} | AccRest]
            end,
            {AccKv2, AccVid};
        ({Key, Val}, {AccKv, AccVid}) ->
            {[{{Key, DocId}, {PartitionId, Val}} | AccKv], [Key | AccVid]}
        end,
        {KVs, []}, lists:sort(ResultKVs)),
    NewViewKVsAcc = [{View, NewKVs} | ViewKVsAcc],
    case NewViewIdKeysAcc of
    [] ->
        NewViewIdKeysAcc2 = ViewIdKeysAcc;
    _ ->
        NewViewIdKeysAcc2 = [{View#set_view.id_num, NewViewIdKeysAcc} | ViewIdKeysAcc]
    end,
    view_insert_doc_query_results(
        DocId, PartitionId, RestResults, RestViewKVs, NewViewKVsAcc, NewViewIdKeysAcc2).


write_changes(WriterAcc, ViewKeyValuesToAdd, DocIdViewIdKeys, PartIdSeqs) ->
    #writer_acc{
        log_fd = LogFd,
        owner = Owner,
        group = Group,
        initial_build = false,
        cleanup_kv_count = CleanupKvCount0,
        cleanup_time = CleanupTime0
    } = WriterAcc,
    #set_view_group{
        id_btree = IdBtree,
        fd = Fd,
        set_name = SetName,
        name = GroupName,
        type = GroupType
    } = Group,

    {AddDocIdViewIdKeys0, RemoveDocIds, LookupDocIds} = lists:foldr(
        fun({DocId, {PartId, [] = _ViewIdKeys}}, {A, B, C}) ->
                BackKey = make_back_index_key(DocId, PartId),
                {A, [BackKey | B], [BackKey | C]};
            ({DocId, {PartId, _ViewIdKeys}} = KvPairs, {A, B, C}) ->
                BackKey = make_back_index_key(DocId, PartId),
                {[KvPairs | A], B, [BackKey | C]}
        end,
        {[], [], []}, DocIdViewIdKeys),
    AddDocIdViewIdKeys = convert_back_index_kvs_to_binary(AddDocIdViewIdKeys0, []),

    CleanupFun = case ?set_cbitmask(Group) of
    0 ->
        nil;
    _ ->
        couch_set_view_util:make_btree_purge_fun(Group)
    end,
    case ?set_cbitmask(Group) of
    0 ->
        IdBtreePurgedKeyCount = 0,
        CleanupStart = 0,
        {ok, LookupResults, IdBtree2} =
            couch_btree:query_modify(IdBtree, LookupDocIds, AddDocIdViewIdKeys, RemoveDocIds);
    _ ->
        CleanupStart = os:timestamp(),
        {ok, LookupResults, {_Go, IdBtreePurgedKeyCount}, IdBtree2} =
            couch_btree:query_modify(
                IdBtree, LookupDocIds, AddDocIdViewIdKeys, RemoveDocIds, CleanupFun, {go, 0})
    end,
    KeysToRemoveByView = lists:foldl(
        fun(LookupResult, KeysToRemoveByViewAcc) ->
            case LookupResult of
            {ok, {<<_Part:16, DocId/binary>>, <<_Part:16, ViewIdKeys/binary>>}} ->
                lists:foldl(
                    fun({ViewId, Keys}, KeysToRemoveByViewAcc2) ->
                        EncodedKeys = [couch_set_view_util:encode_key_docid(Key, DocId) || Key <- Keys],
                        dict:append_list(ViewId, EncodedKeys, KeysToRemoveByViewAcc2)
                    end,
                    KeysToRemoveByViewAcc, couch_set_view_util:parse_view_id_keys(ViewIdKeys));
            {not_found, _} ->
                KeysToRemoveByViewAcc
            end
        end,
        dict:new(), LookupResults),
    ViewKeyValuesToAddBinary = lists:map(
        fun({View, AddKeyValues}) ->
            {View, convert_primary_index_kvs_to_binary(AddKeyValues, Group, [])}
        end,
        ViewKeyValuesToAdd),
    {Views2, {CleanupKvCount, InsertedKvCount, DeletedKvCount}} =
        lists:mapfoldl(fun({View, {_View, AddKeyValues}}, {AccC, AccI, AccD}) ->
            KeysToRemove = couch_util:dict_find(View#set_view.id_num, KeysToRemoveByView, []),
            case ?set_cbitmask(Group) of
            0 ->
                CleanupCount = 0,
                {ok, ViewBtree2} = couch_btree:add_remove(
                    View#set_view.btree, AddKeyValues, KeysToRemove);
            _ ->
                {ok, {_Go2, CleanupCount}, ViewBtree2} = couch_btree:add_remove(
                    View#set_view.btree, AddKeyValues, KeysToRemove, CleanupFun, {go, 0})
            end,
            NewView = View#set_view{btree = ViewBtree2},
            {NewView, {AccC + CleanupCount, AccI + length(AddKeyValues), AccD + length(KeysToRemove)}}
        end,
        {IdBtreePurgedKeyCount, 0, 0}, lists:zip(Group#set_view_group.views, ViewKeyValuesToAddBinary)),

    case ?set_cbitmask(Group) of
    0 ->
        NewCbitmask = 0,
        CleanupTime = 0;
    _ ->
        {ok, <<_IdCount:40, IdBitmap:?MAX_NUM_PARTITIONS>>} = couch_btree:full_reduce(IdBtree2),
        CombinedBitmap = lists:foldl(
            fun(#set_view{btree = Bt}, AccMap) ->
                {ok, <<_ViewCount:40, Bm:?MAX_NUM_PARTITIONS, _/binary>>} = couch_btree:full_reduce(Bt),
                AccMap bor Bm
            end,
            IdBitmap, Views2),
        NewCbitmask = ?set_cbitmask(Group) band CombinedBitmap,
        CleanupTime = timer:now_diff(os:timestamp(), CleanupStart) / 1000000,
        ?LOG_INFO("Updater for set view `~s`, ~s group `~s`, performed cleanup "
            "of ~p key/value pairs in ~.3f seconds",
            [SetName, GroupType, GroupName, CleanupKvCount, CleanupTime])
    end,
    NewSeqs = update_seqs(PartIdSeqs, ?set_seqs(Group)),
    case LogFd of
    nil ->
        ok;
    _ ->
        LogViewsAddRemoveKvs = lists:map(
            fun({#set_view{id_num = ViewId}, AddKeyValues}) ->
                KeysToRemove = couch_util:dict_find(ViewId, KeysToRemoveByView, []),
                {AddKeyValues, KeysToRemove}
            end,
            ViewKeyValuesToAddBinary),
        LogEntry = {NewSeqs, AddDocIdViewIdKeys, RemoveDocIds, LogViewsAddRemoveKvs},
        LogEntryBin = couch_compress:compress(?term_to_bin(LogEntry)),
        ok = file:write(LogFd, [<<(byte_size(LogEntryBin)):32>>, LogEntryBin]),
        {ok, LogEof} = file:position(LogFd, eof),
        ok = gen_server:cast(Owner, {log_eof, LogEof})
    end,
    Header = Group#set_view_group.index_header,
    NewHeader = Header#set_view_index_header{
        id_btree_state = couch_btree:get_state(IdBtree2),
        view_states = [couch_btree:get_state(V#set_view.btree) || V <- Views2],
        seqs = NewSeqs,
        cbitmask = NewCbitmask
    },
    NewGroup = Group#set_view_group{
        views = Views2,
        id_btree = IdBtree2,
        index_header = NewHeader
    },
    couch_file:flush(Fd),
    WriterAcc#writer_acc{
        group = NewGroup,
        cleanup_kv_count = CleanupKvCount0 + CleanupKvCount,
        cleanup_time = CleanupTime0 + CleanupTime,
        inserted_kvs = WriterAcc#writer_acc.inserted_kvs + InsertedKvCount,
        deleted_kvs = WriterAcc#writer_acc.deleted_kvs + DeletedKvCount,
        inserted_ids = WriterAcc#writer_acc.inserted_ids + length(AddDocIdViewIdKeys),
        deleted_ids = WriterAcc#writer_acc.deleted_ids + length(RemoveDocIds)
    }.


update_seqs(PartIdSeqs, Seqs) ->
    orddict:fold(
        fun(PartId, NewSeq, Acc) ->
            OldSeq = couch_util:get_value(PartId, Acc),
            case is_integer(OldSeq) of
            true ->
                ok;
            false ->
                exit({error, <<"Old seq is not an integer.">>, PartId, OldSeq, NewSeq})
            end,
            case NewSeq > OldSeq of
            true ->
                ok;
            false ->
                exit({error, <<"New seq smaller or equal than old seq.">>, PartId, OldSeq, NewSeq})
            end,
            ?replace(Acc, PartId, NewSeq)
        end,
        Seqs, PartIdSeqs).


update_task(NumChanges) ->
    [Changes, Total] = couch_task_status:get([changes_done, total_changes]),
    Changes2 = Changes + NumChanges,
    Total2 = erlang:max(Total, Changes2),
    Progress = (Changes2 * 100) div Total2,
    couch_task_status:update([
        {progress, Progress},
        {changes_done, Changes2},
        {total_changes, Total2}
    ]).


maybe_checkpoint(WriterAcc) ->
    Before = get(last_header_commit_ts),
    Now = os:timestamp(),
    case (Before == undefined) orelse
        (timer:now_diff(Now, Before) >= ?CHECKPOINT_WRITE_INTERVAL) of
    true ->
        checkpoint(WriterAcc, false),
        put(last_header_commit_ts, Now);
    false ->
        #writer_acc{owner = Owner, parent = Parent, group = Group} = WriterAcc,
        Owner ! {partial_update, Parent, Group}
    end.


checkpoint(#writer_acc{owner = Owner, parent = Parent, group = Group}, DoFsync) ->
    #set_view_group{
        set_name = SetName,
        name = DDocId,
        type = Type
    } = Group,
    ?LOG_INFO("Updater checkpointing set view `~s` update for ~s group `~s`",
              [SetName, Type, DDocId]),
    write_header(Group, DoFsync),
    Owner ! {partial_update, Parent, Group}.


write_header(#set_view_group{fd = Fd} = Group, DoFsync) ->
    HeaderBin = couch_set_view_util:group_to_header_bin(Group),
    ok = couch_file:write_header_bin(Fd, HeaderBin),
    case DoFsync of
    true ->
        ok = couch_file:sync(Fd);
    false ->
        ok
    end.


open_log_file(nil) ->
    nil;
open_log_file(Path) when is_list(Path) ->
    {ok, LogFd} = file2:open(Path, [raw, binary, append]),
    LogFd.


maybe_open_log_file(Acc) ->
    receive
    {log_new_changes, Pid, LogFilePath} ->
        case Acc#writer_acc.log_fd of
        nil ->
            ok;
        OldLogFd ->
            % Compactor died and just restarted, close the current
            % log and open a new one.
            file:close(OldLogFd)
        end,
        LogFd = open_log_file(LogFilePath),
        Pid ! {log_started, self(), Acc#writer_acc.group},
        Acc#writer_acc{log_fd = LogFd}
    after 0 ->
        Acc
    end.


init_view_merge_params(#writer_acc{initial_build = false} = WriterAcc) ->
    WriterAcc;
init_view_merge_params(#writer_acc{group = Group} = WriterAcc) ->
    SortFiles = [{View#set_view.id_num, []} || View <- Group#set_view_group.views],
    Buffers = [{View#set_view.id_num, {[], 0}} || View <- Group#set_view_group.views],
    WriterAcc#writer_acc{
        merge_buffers = dict:from_list([{ids_index, {[], 0}} | Buffers]),
        sort_files = dict:from_list([{ids_index, []} | SortFiles])
    }.


new_sort_file_name(#writer_acc{tmp_dir = TmpDir, group = Group}) ->
    couch_set_view_util:new_sort_file_path(TmpDir, Group#set_view_group.sig).


delete_prev_sort_files(#writer_acc{initial_build = false}) ->
    ok;
delete_prev_sort_files(#writer_acc{tmp_dir = TmpDir}) ->
    ok = couch_set_view_util:delete_sort_files(TmpDir).


wait_for_workers(Pids) ->
    ok = lists:foldr(fun(W, ok) ->
        Ref = erlang:monitor(process, W),
        receive
        {'DOWN', Ref, process, W, {sort_worker_died, _} = Reason} ->
            exit(Reason);
        {'DOWN', Ref, process, W, _Reason} ->
            ok
        end
    end, ok, Pids).


convert_primary_index_kvs_to_binary([], _Group, Acc) ->
    lists:reverse(Acc);
convert_primary_index_kvs_to_binary([{{Key, DocId}, {PartId, V0}} | Rest], Group, Acc)->
    V = case V0 of
    {dups, Values} ->
        ValueListBinary = lists:foldl(
            fun(V, Acc2) ->
                <<Acc2/binary, (byte_size(V)):24, V/binary>>
            end,
            <<>>, Values),
        <<PartId:16, ValueListBinary/binary>>;
    _ ->
        <<PartId:16, (byte_size(V0)):24, V0/binary>>
    end,
    KeyBin = couch_set_view_util:encode_key_docid(Key, DocId),
    case byte_size(KeyBin) > ?MAX_KEY_SIZE of
    true ->
        #set_view_group{set_name = SetName, name = DDocId, type = Type} = Group,
        KeyPrefix = lists:sublist(unicode:characters_to_list(Key), 100),
        Error = iolist_to_binary(
            io_lib:format("key emitted for document `~s` is too long: ~s...", [DocId, KeyPrefix])),
        ?LOG_MAPREDUCE_ERROR("Bucket `~s`, ~s group `~s`, ~s", [SetName, Type, DDocId, Error]),
        throw({error, Error});
    false ->
        convert_primary_index_kvs_to_binary(Rest, Group, [{KeyBin, V} | Acc])
    end.


convert_back_index_kvs_to_binary([], Acc)->
    lists:reverse(Acc);
convert_back_index_kvs_to_binary([{DocId, {PartId, ViewIdKeys}} | Rest], Acc) ->
    ViewIdKeysBinary = lists:foldl(
        fun({ViewId, Keys}, Acc2) ->
            KeyListBinary = lists:foldl(
                fun(Key, AccKeys) ->
                    <<AccKeys/binary, (byte_size(Key)):16, Key/binary>>
                end,
                <<>>, Keys),
            <<Acc2/binary, ViewId:8, (length(Keys)):16, KeyListBinary/binary>>
        end,
        <<>>, ViewIdKeys),
    KvBin = {make_back_index_key(DocId, PartId), <<PartId:16, ViewIdKeysBinary/binary>>},
    convert_back_index_kvs_to_binary(Rest, [KvBin | Acc]).


make_back_index_key(DocId, PartId) ->
    <<PartId:16, DocId/binary>>.


file_sorter_format_function(<<KeyLen:16, Key:KeyLen/binary, Value/binary>>) ->
    {Key, Value}.


split_files_to_merge(SortFiles, true) ->
    {LastGen, _} = lists:last(SortFiles),
    {[F || {_, F} <- SortFiles], LastGen + 1, []};

split_files_to_merge([{MinGen, FirstFile} | Rest], false) ->
    {ChosenGen, ToMerge, NotToMerge} =
        split_files_by_gen(MinGen, Rest, ordsets:new(), [FirstFile]),
    {ToMerge, ChosenGen + length(ToMerge), NotToMerge}.


split_files_by_gen(Gen, [{Gen, F} | Rest], AccNot, Acc) ->
    split_files_by_gen(Gen, Rest, AccNot, [F | Acc]);
split_files_by_gen(Gen, [], AccNot, Acc) ->
    {Gen, Acc, AccNot};
split_files_by_gen(Gen, [{Gen2, F} | Rest], AccNot, [FAcc]) when Gen2 > Gen ->
    split_files_by_gen(Gen2, Rest, ordsets:add_element({Gen, FAcc}, AccNot), [F]);
split_files_by_gen(Gen, [{Gen2, _} | _] = Rest, AccNot, Acc) when Gen2 > Gen ->
    {Gen, Acc, ordsets:union(AccNot, Rest)}.


spawn_merge_worker(LessFun, TmpDir, Workers, FilesToMerge, DestFile) ->
    spawn_link(fun() ->
        SortOptions = [
            {order, LessFun},
            {tmpdir, TmpDir},
            {format, fun file_sorter_format_function/1}
        ],
        wait_for_workers(Workers),
        case file_sorter:merge(FilesToMerge, DestFile, SortOptions) of
        ok ->
            ok;
        {error, Reason} ->
            exit({sort_worker_died, Reason})
        end,
        lists:foreach(fun(F) ->
            case file2:delete(F) of
            ok ->
                ok;
            {error, Reason2} ->
                exit({sort_worker_died, Reason2})
            end
        end, FilesToMerge)
    end).
