Add support for keying the cache by revision

This patch splits up the code paths between ddoc_cache and
ddoc_cache_opener a bit more clearly, keeping the gen_server
implementation details "mostly" in one place. The overloaded
open function has been replaced with more explicit function
names. I've left the old open to support old code during the
upgrade process but it should be removed after the next release.

The opener lookup keys are a bit mismatched from the LRU keys so
I've renamed those variables to make it clear when we're talking
about the openers rather than the cache itself. It could be
argued that the revision oblivious doc lookup could reuse the
result of any pending revision specific lookup on the same id
but it seems a bit too optimistic here so I've left them as
separate opener keys.

diff --git a/src/ddoc_cache.erl b/src/ddoc_cache.erl
index e862667..6c54c36 100644 (file)
--- a/src/ddoc_cache.erl
+++ b/src/ddoc_cache.erl
@@ -12,70 +12,60 @@
 
 -module(ddoc_cache).
 
+-include_lib("couch/include/couch_db.hrl").
 
--export([
-    start/0,
-    stop/0,
-    
-    open/2,
-    evict/2
-]).
+-export([start/0, stop/0]).
 
+% public API
+-export([open_doc/2, open_doc/3, open_validation_funs/1, evict/2]).
 
--define(CACHE, ddoc_cache_lru).
--define(OPENER, ddoc_cache_opener).
-
+% deprecated API
+-export([open/2]).
 
 start() ->
     application:start(ddoc_cache).
 
-
 stop() ->
     application:stop(ddoc_cache).
 
+open_doc(DbName, DocId) ->
+    Key = {DbName, DocId, '_'},
+    case ddoc_cache_opener:match_newest(Key) of
+        {ok, _} = Resp ->
+            Resp;
+        missing ->
+            ddoc_cache_opener:open_doc(DbName, DocId);
+        recover ->
+            ddoc_cache_opener:recover_doc(DbName, DocId)
+    end.
 
-open(DbName, validation_funs) ->
-    open({DbName, validation_funs});
-open(DbName, <<"_design/", _/binary>>=DDocId) when is_binary(DbName) ->
-    open({DbName, DDocId});
-open(DbName, DDocId) when is_binary(DDocId) ->
-    open({DbName, <<"_design/", DDocId/binary>>}).
-
-
-open(Key) ->
-    try ets_lru:lookup_d(?CACHE, Key) of
+open_doc(DbName, DocId, RevId) ->
+    Key = {DbName, DocId, RevId},
+    case ddoc_cache_opener:lookup(Key) of
         {ok, _} = Resp ->
             Resp;
-        _ ->
-            case gen_server:call(?OPENER, {open, Key}, infinity) of
-                {open_ok, Resp} ->
-                    Resp;
-                {open_error, throw, Error} ->
-                    throw(Error);
-                {open_error, error, Error} ->
-                    erlang:error(Error);
-                {open_error, exit, Error} ->
-                    exit(Error)
-            end
-    catch
-        error:badarg ->
-            recover(Key)
+        missing ->
+            ddoc_cache_opener:open_doc(DbName, DocId, RevId);
+        recover ->
+            ddoc_cache_opener:recover_doc(DbName, DocId, RevId)
     end.
 
+open_validation_funs(DbName) ->
+    Key = {DbName, validation_funs},
+    case ddoc_cache_opener:lookup(Key) of
+        {ok, _} = Resp ->
+            Resp;
+        missing ->
+            ddoc_cache_opener:open_validation_funs(DbName);
+        recover ->
+            ddoc_cache_opener:recover_validation_funs(DbName)
+    end.
 
 evict(ShardDbName, DDocIds) ->
     DbName = mem3:dbname(ShardDbName),
-    gen_server:cast(?OPENER, {evict, DbName, DDocIds}).
-
+    ddoc_cache_opener:evict_docs(DbName, DDocIds).
 
-recover({DbName, validation_funs}) ->
-    {ok, DDocs} = fabric:design_docs(mem3:dbname(DbName)),
-    Funs = lists:flatmap(fun(DDoc) ->
-        case couch_doc:get_validate_doc_fun(DDoc) of
-            nil -> [];
-            Fun -> [Fun]
-        end
-    end, DDocs),
-    {ok, Funs};
-recover({DbName, DDocId}) ->
-    fabric:open_doc(DbName, DDocId, [ejson_body]).
+open(DbName, validation_funs) ->
+    open_validation_funs(DbName);
+open(DbName, DocId) ->
+    open_doc(DbName, DocId).

diff --git a/src/ddoc_cache_opener.erl b/src/ddoc_cache_opener.erl
index b34359f..780f9dd 100644 (file)
--- a/src/ddoc_cache_opener.erl
+++ b/src/ddoc_cache_opener.erl
@@ -13,38 +13,49 @@
 -module(ddoc_cache_opener).
 -behaviour(gen_server).
 
+-include_lib("couch/include/couch_db.hrl").
 -include_lib("mem3/include/mem3.hrl").
 
--export([
-    start_link/0
-]).
+% worker API
+-export([start_link/0]).
 
+% public API
 -export([
-    open_ddoc/1
+    open_doc/2,
+    open_doc/3,
+    open_validation_funs/1,
+    evict_docs/2,
+    lookup/1,
+    match_newest/1,
+    recover_doc/2,
+    recover_doc/3,
+    recover_validation_funs/1
 ]).
 
+% couch_event listener callback
+-export([handle_db_event/3]).
+
+% gen_server behavior
 -export([
     init/1,
     terminate/2,
-
     handle_call/3,
     handle_cast/2,
     handle_info/2,
-
     code_change/3
 ]).
 
+% sub-process spawn API
 -export([
-    handle_db_event/3
+    fetch_doc_data/1
 ]).
 
-
 -define(CACHE, ddoc_cache_lru).
 -define(OPENING, ddoc_cache_opening).
 
 -type dbname() :: iodata().
 -type docid() :: iodata().
--type revision() :: {integer(), binary()}.
+-type revision() :: {pos_integer(), <<_:128>>}.
 
 -record(opener, {
     key,
@@ -57,10 +68,78 @@
     evictor
 }).
 
-
 start_link() ->
     gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
 
+-spec open_doc(dbname(), docid()) -> {ok, #doc{}}.
+open_doc(DbName, DocId) ->
+    Resp = gen_server:call(?MODULE, {open, {DbName, DocId}}, infinity),
+    handle_open_response(Resp).
+
+-spec open_doc(dbname(), docid(), revision()) -> {ok, #doc{}}.
+open_doc(DbName, DocId, RevId) ->
+    Resp = gen_server:call(?MODULE, {open, {DbName, DocId, RevId}}, infinity),
+    handle_open_response(Resp).
+
+-spec open_validation_funs(dbname()) -> {ok, [fun()]}.
+open_validation_funs(DbName) ->
+    Resp = gen_server:call(?MODULE, {open, {DbName, validation_funs}}, infinity),
+    handle_open_response(Resp).
+
+-spec evict_docs(dbname(), [docid()]) -> ok.
+evict_docs(DbName, DocIds) ->
+    gen_server:cast(?MODULE, {evict, DbName, DocIds}).
+
+lookup(Key) ->
+    try ets_lru:lookup_d(?CACHE, Key) of
+        {ok, _} = Resp ->
+            Resp;
+        _ ->
+            missing
+    catch
+        error:badarg ->
+            recover
+    end.
+
+match_newest(Key) ->
+    try ets_lru:match_object(?CACHE, Key, '_') of
+        [] ->
+            missing;
+        Docs ->
+            Sorted = lists:sort(fun (#doc{revs=L}, #doc{revs=R}) ->
+                L >= R
+            end, Docs),
+            {ok, hd(Sorted)}
+    catch
+        error:badarg ->
+            recover
+    end.
+
+recover_doc(DbName, DDocId) ->
+    fabric:open_doc(DbName, DDocId, []).
+
+recover_doc(DbName, DDocId, RevId) ->
+    {ok, [Resp]} = fabric:open_revs(DbName, DDocId, [RevId], []),
+    Resp.
+
+recover_validation_funs(DbName) ->
+    {ok, DDocs} = fabric:design_docs(mem3:dbname(DbName)),
+    Funs = lists:flatmap(fun(DDoc) ->
+        case couch_doc:get_validate_doc_fun(DDoc) of
+            nil -> [];
+            Fun -> [Fun]
+        end
+    end, DDocs),
+    {ok, Funs}.
+
+handle_db_event(ShardDbName, created, St) ->
+    gen_server:cast(?MODULE, {evict, mem3:dbname(ShardDbName)}),
+    {ok, St};
+handle_db_event(ShardDbName, deleted, St) ->
+    gen_server:cast(?MODULE, {evict, mem3:dbname(ShardDbName)}),
+    {ok, St};
+handle_db_event(_DbName, _Event, St) ->
+    {ok, St}.
 
 init(_) ->
     process_flag(trap_exit, true),
@@ -72,7 +151,6 @@ init(_) ->
         evictor = Evictor
     }}.
 
-
 terminate(_Reason, St) ->
     case is_pid(St#st.evictor) of
         true -> exit(St#st.evictor, kill);
@@ -80,15 +158,14 @@ terminate(_Reason, St) ->
     end,
     ok.
 
-
-handle_call({open, {_DbName, _DDocId}=Key}, From, St) ->
-    case ets:lookup(?OPENING, Key) of
+handle_call({open, OpenerKey}, From, St) ->
+    case ets:lookup(?OPENING, OpenerKey) of
         [#opener{clients=Clients}=O] ->
             ets:insert(?OPENING, O#opener{clients=[From | Clients]}),
             {noreply, St};
         [] ->
-            Pid = spawn_link(?MODULE, open_ddoc, [Key]),
-            ets:insert(?OPENING, #opener{key=Key, pid=Pid, clients=[From]}),
+            Pid = spawn_link(?MODULE, fetch_doc_data, [OpenerKey]),
+            ets:insert(?OPENING, #opener{key=OpenerKey, pid=Pid, clients=[From]}),
             {noreply, St}
     end;
 
@@ -105,39 +182,41 @@ handle_cast({evict, DbName, DDocIds}, St) ->
     {noreply, St};
 
 handle_cast({do_evict, DbName}, St) ->
-    DDocIds = ets_lru:match(?CACHE, {DbName, '$1'}, '_'),
+    DDocIds = lists:flatten(ets_lru:match(?CACHE, {DbName, '$1', '_'}, '_')),
     handle_cast({do_evict, DbName, DDocIds}, St);
 
 handle_cast({do_evict, DbName, DDocIds}, St) ->
     ets_lru:remove(?CACHE, {DbName, validation_funs}),
     lists:foreach(fun(DDocId) ->
-        ets_lru:remove(?CACHE, {DbName, DDocId})
+        RevIds = ets_lru:match(?CACHE, {DbName, DDocId, '$1'}, '_'),
+        lists:foreach(fun([RevId]) ->
+            ets_lru:remove(?CACHE, {DbName, DDocId, RevId})
+        end, RevIds)
     end, DDocIds),
     {noreply, St};
 
 handle_cast(Msg, St) ->
     {stop, {invalid_cast, Msg}, St}.
 
-
 handle_info({'EXIT', Pid, Reason}, #st{evictor=Pid}=St) ->
     couch_log:error("ddoc_cache_opener evictor died ~w", [Reason]),
     {ok, Evictor} = couch_event:link_listener(?MODULE, handle_db_event, nil, [all_dbs]),
     {noreply, St#st{evictor=Evictor}};
 
-handle_info({'EXIT', _Pid, {open_ok, Key, Resp}}, St) ->
-    respond(Key, {open_ok, Resp}),
+handle_info({'EXIT', _Pid, {open_ok, OpenerKey, Resp}}, St) ->
+    respond(OpenerKey, {open_ok, Resp}),
     {noreply, St};
 
-handle_info({'EXIT', _Pid, {open_error, Key, Type, Error}}, St) ->
-    respond(Key, {open_error, Type, Error}),
+handle_info({'EXIT', _Pid, {open_error, OpenerKey, Type, Error}}, St) ->
+    respond(OpenerKey, {open_error, Type, Error}),
     {noreply, St};
 
 handle_info({'EXIT', Pid, Reason}, St) ->
     Pattern = #opener{pid=Pid, _='_'},
     case ets:match_object(?OPENING, Pattern) of
-        [#opener{key=Key, clients=Clients}] ->
+        [#opener{key=OpenerKey, clients=Clients}] ->
             _ = [gen_server:reply(C, {error, Reason}) || C <- Clients],
-            ets:delete(?OPENING, Key),
+            ets:delete(?OPENING, OpenerKey),
             {noreply, St};
         [] ->
             {stop, {unknown_pid_died, {Pid, Reason}}, St}
@@ -146,46 +225,53 @@ handle_info({'EXIT', Pid, Reason}, St) ->
 handle_info(Msg, St) ->
     {stop, {invalid_info, Msg}, St}.
 
-
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
 
-
-handle_db_event(ShardDbName, created, St) ->
-    gen_server:cast(?MODULE, {evict, mem3:dbname(ShardDbName)}),
-    {ok, St};
-handle_db_event(ShardDbName, deleted, St) ->
-    gen_server:cast(?MODULE, {evict, mem3:dbname(ShardDbName)}),
-    {ok, St};
-handle_db_event(_DbName, _Event, St) ->
-    {ok, St}.
-
-
--spec open_ddoc({dbname(), validation_funs | docid()}) -> no_return().
-open_ddoc({DbName, validation_funs}=Key) ->
-    {ok, DDocs} = fabric:design_docs(mem3:dbname(DbName)),
-    Funs = lists:flatmap(fun(DDoc) ->
-        case couch_doc:get_validate_doc_fun(DDoc) of
-            nil -> [];
-            Fun -> [Fun]
-        end
-    end, DDocs),
-    ok = ets_lru:insert(?CACHE, {DbName, validation_funs}, Funs),
-    exit({open_ok, Key, {ok, Funs}});
-open_ddoc({DbName, DDocId}=Key) ->
-    try fabric:open_doc(DbName, DDocId, [ejson_body]) of
+-spec fetch_doc_data({dbname(), validation_funs}) -> no_return();
+                    ({dbname(), docid()}) -> no_return();
+                    ({dbname(), docid(), revision()}) -> no_return().
+fetch_doc_data({DbName, validation_funs}) ->
+    OpenerKey = {DbName, validation_funs},
+    {ok, Funs} = recover_validation_funs(DbName),
+    ok = ets_lru:insert(?CACHE, OpenerKey, Funs),
+    exit({open_ok, OpenerKey, {ok, Funs}});
+fetch_doc_data({DbName, DocId}) ->
+    OpenerKey = {DbName, DocId},
+    try recover_doc(DbName, DocId) of
         {ok, Doc} ->
-            ok = ets_lru:insert(?CACHE, {DbName, DDocId}, Doc),
-            exit({open_ok, Key, {ok, Doc}});
+            {RevCount, [RevHash| _]} = Doc#doc.revs,
+            RevId = {RevCount, RevHash},
+            ok = ets_lru:insert(?CACHE, {DbName, DocId, RevId}, Doc),
+            exit({open_ok, OpenerKey, {ok, Doc}});
         Else ->
-            exit({open_ok, Key, Else})
+            exit({open_ok, OpenerKey, Else})
     catch
         Type:Reason ->
-            exit({open_error, Key, Type, Reason})
+            exit({open_error, OpenerKey, Type, Reason})
+    end;
+fetch_doc_data({DbName, DocId, RevId}) ->
+    OpenerKey = {DbName, DocId, RevId},
+    try recover_doc(DbName, DocId, RevId) of
+        {ok, Doc} ->
+            ok = ets_lru:insert(?CACHE, {DbName, DocId, RevId}, Doc),
+            exit({open_ok, OpenerKey, {ok, Doc}});
+        Else ->
+            exit({open_ok, OpenerKey, Else})
+    catch
+        Type:Reason ->
+            exit({open_error, OpenerKey, Type, Reason})
     end.
 
+handle_open_response(Resp) ->
+    case Resp of
+        {open_ok, Value} -> Value;
+        {open_error, throw, Error} -> throw(Error);
+        {open_error, error, Error} -> erlang:error(Error);
+        {open_error, exit, Error} -> exit(Error)
+    end.
 
-respond(Key, Resp) ->
-    [#opener{clients=Clients}] = ets:lookup(?OPENING, Key),
+respond(OpenerKey, Resp) ->
+    [#opener{clients=Clients}] = ets:lookup(?OPENING, OpenerKey),
     _ = [gen_server:reply(C, Resp) || C <- Clients],
-    ets:delete(?OPENING, Key).
+    ets:delete(?OPENING, OpenerKey).