File 3493-Optimize-ets-operations.patch of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 3493-Optimize-ets-operations.patch of Package erlang

From 03c893ef92bbc9cafed48941c2c419580147c1f6 Mon Sep 17 00:00:00 2001
From: Raimo Niskanen <raimo@erlang.org>
Date: Fri, 9 Apr 2021 13:04:59 +0200
Subject: [PATCH 3/4] Optimize ets operations

Use match specifications and ets:select_delete/2 to traverse and
delete in one pass.  Reduce the number of passes for alloc_entry
through one pass to get all time stamps, calculate deletion
criteria, then one pass to delete.
---
 lib/kernel/src/inet_db.erl | 173 +++++++++++++++++++++++--------------
 1 file changed, 107 insertions(+), 66 deletions(-)

diff --git a/lib/kernel/src/inet_db.erl b/lib/kernel/src/inet_db.erl
index 7cf95dda8a..47f6258c81 100644
--- a/lib/kernel/src/inet_db.erl
+++ b/lib/kernel/src/inet_db.erl
@@ -1193,7 +1193,7 @@ handle_cast(_Msg, State) ->
 -spec handle_info(term(), state()) -> {'noreply', state()}.
 
 handle_info(refresh_timeout, State) ->
-    do_refresh_cache(State#state.cache),
+    _ = delete_expired(State#state.cache, times()),
     {noreply, State#state{cache_timer = init_timer()}};
 
 handle_info(_Info, State) ->
@@ -1598,7 +1598,7 @@ is_reqname(clear_cache) -> true;
 is_reqname(clear_hosts) -> true;
 is_reqname(_) -> false.
 
-%% Add a resource record to the cache if there is space left.
+%% Add a resource record to the cache if there is a cache.
 %% If the cache is full this function first deletes old entries,
 %% i.e. entries with the oldest access time.
 %%
@@ -1643,7 +1643,20 @@ times() ->
 
 %% ETS match expressions
 %%
--compile({inline, [dns_rr_match/3, dns_rr_match/4]}).
+-compile(
+   {inline,
+    [dns_rr_match_tm_ttl_cnt/3, dns_rr_match_cnt/1,
+     dns_rr_match/3, dns_rr_match/4]}).
+%%
+dns_rr_match_tm_ttl_cnt(TM, TTL, Cnt) ->
+    #dns_rr{
+       domain = '_', class = '_', type = '_', data = '_',
+       cnt = Cnt, tm = TM, ttl = TTL, bm = '_', func = '_'}.
+dns_rr_match_cnt(Cnt) ->
+    #dns_rr{
+       domain = '_', class = '_', type = '_', data = '_',
+       cnt = Cnt, tm = '_', ttl = '_', bm = '_', func = '_'}.
+%%
 dns_rr_match(Domain, Class, Type) ->
     #dns_rr{
        domain = Domain, class = Class, type = Type, data = '_',
@@ -1656,6 +1669,7 @@ dns_rr_match(Domain, Class, Type, Data) ->
 
 %% RR creation
 -compile({inline, [dns_rr_add/5]}).
+%%
 dns_rr_add(Domain, Class, Type, TTL, Data) ->
     #dns_rr{
        domain = Domain, class = Class, type = Type,
@@ -1798,81 +1812,98 @@ cache_refresh() ->
     end.
 
 %% Delete all entries with expired TTL.
-%% Returns the access time of the entry with the oldest access time
-%% in the cache.
-do_refresh_cache(CacheDb) ->
-    Now = times(),
-    [OldestT | RRsToDelete] =
-        ets:foldl(
-          fun (#dns_rr{tm = TM, ttl = TTL} = RR, [T | RRs])
-                when TM + TTL < Now -> % Expired - delete it
-                  [T, RR | RRs];
-              (#dns_rr{cnt = C}, [T | RRs])
-                when C < T -> % Older than previously oldest - use this cnt
-                  [C | RRs];
-              (#dns_rr{}, T_RRs) -> % Keep - do not delete
-                  T_RRs
-          end, [Now], CacheDb),
-    _ = [ets:delete_object(CacheDb, RR) || RR <- RRsToDelete],
-    OldestT.
+%% Returns the number of deleted entries.
+%%
+delete_expired(CacheDb, TM) ->
+    ets:select_delete(
+      CacheDb,
+      [{dns_rr_match_tm_ttl_cnt('$1', '$2', '_'), [],
+        %% Delete all with tm + ttl < TM
+        [{'<', {'+', '$1', '$2'}, {const, TM}}]}]).
 
 
 %% -------------------------------------------------------------------
 %% Allocate room for a new entry in the cache.
+%%
 %% Deletes entries with expired TTL and all entries with latest
-%% access time older than
-%% trunc((TM - OldestTM) * 0.3) + OldestTM from the cache if it
-%% is full. Does not delete more than 10% of the entries in the cache
+%% access time older than trunc((TM - OldestTM) / 3) + OldestTM
+%% from the cache if it is full.
+%%
+%% Does not delete more than 1/10 of the entries in the cache
 %% though, unless they there deleted due to expired TTL.
-%% Returns: true if space for a new entry otherwise false.
+%% Returns: true if space for a new entry otherwise false
+%% (true if we have a cache since we always make room for new).
 %% -------------------------------------------------------------------
 alloc_entry(Db, CacheDb, TM) ->
-    CurSize = ets:info(CacheDb, size),
-    case ets:lookup_element(Db, cache_size, 2) of
-	Size when Size =< CurSize, Size > 0 ->
-	    alloc_entry(CacheDb, CurSize, TM, trunc(Size * 0.1) + 1);
-	Size when Size =< 0 ->
+    Size = ets:lookup_element(Db, cache_size, 2),
+    if
+	Size =< 0 ->
 	    false;
-	_Size ->
-	    true
-    end.
-
-alloc_entry(CacheDb, OldSize, TM, N) ->
-    OldestTM = do_refresh_cache(CacheDb),     % Delete timedout entries
-    case ets:info(CacheDb, size) of
-	OldSize ->
-	    %% No entrys timedout
-	    delete_n_oldest(CacheDb, TM, OldestTM, N);
-	_ ->
-	    true
+        true ->
+            CurSize = ets:info(CacheDb, size),
+            if
+                Size =< CurSize ->
+                    N = ((Size - 1) div 10) + 1,
+                    _ = delete_oldest(CacheDb, TM, N),
+                    true;
+                true ->
+                    true
+            end
     end.
 
-delete_n_oldest(CacheDb, TM, OldestTM, N) ->
-    DelTM = trunc((TM - OldestTM) * 0.3) + OldestTM,
-    delete_older(CacheDb, DelTM, N) =/= 0.
-
-%% Delete entries with latest access time older than TM.
-%% Delete max N number of entries.
-%% Return the number of deleted entries.
+%% This deletion should always give some room since
+%% it removes a percentage of the oldest entries.
+%%
+%% Fetch all cnt times, sort them, calculate a limit
+%% as the earliest of the time 1/3 from the oldest to now,
+%% and the 1/10 oldest entry,.
+%%
+%% Delete all entries with a cnt time older than that,
+%% and all expired (tm + ttl < now).
 %%
-delete_older(CacheDb, TM, N) ->
-    RRsToDelete =
-        ets:foldl(
-          fun (#dns_rr{cnt = C} = RR, RRs)
-              when C =< TM -> % Candidate for deletion
-                  [RR | RRs];
-              (#dns_rr{}, RRs) ->
-                  RRs
-          end, [], CacheDb),
-    N - delete_older_n(CacheDb, lists:keysort(#dns_rr.cnt, RRsToDelete), N).
-
-delete_older_n(_CacheDb, RRs, 0) when is_list(RRs) ->
-    0;
-delete_older_n(_CacheDb, [], N) ->
-    N;
-delete_older_n(CacheDb, [RR | RRs], N) ->
-    ets:delete_object(CacheDb, RR),
-    delete_older_n(CacheDb, RRs, N - 1).
+delete_oldest(CacheDb, TM, N) ->
+    case
+        lists:sort(
+          ets:select(
+            CacheDb,
+            %% All cnt vals
+            [{dns_rr_match_cnt('$1'), [], ['$1']}]))
+        %% That could be space optimized by using ets:select/3
+        %% with a limit, and storing the returned times in
+        %% gb_sets with size limitation of N.  Then we would
+        %% never have to sort the whole list and find
+        %% the N:th element, but instead take the smallest
+        %% and largest elements from gb_sets.
+        %%
+        %% The size of the whole list is, however, already
+        %% much smaller than all table entries, so is is
+        %% unclear how much of an improvement that would be.
+        %%
+        %% Note that since gb_sets does not store duplicate
+        %% times, that will not work nicely if there are
+        %% many duplicate times, which is not unlikely
+        %% given the second resolution.  Therefore it is
+        %% possible that gb_trees and storing the number
+        %% of occurences for a cnt time might be needed,
+        %% so insertion gets more complicated and slower,
+        %% and we need our own concept of set size.
+        %%
+    of
+        [] -> % Empty table, this should not happen,
+            0;
+        [OldestTM | _] = TMs ->
+            DelTM_A = ((TM - OldestTM) div 3) + OldestTM,
+            DelTM_B = lists_nth(N, TMs, DelTM_A), % N:th cnt time
+            DelTM = min(DelTM_A, DelTM_B),
+            %%
+            ets:select_delete(
+              CacheDb,
+              [{dns_rr_match_tm_ttl_cnt('$1', '$2', '$3'), [],
+                %% RRs with cnt =< DelTM or tm + ttl < TM
+                [{'orelse',
+                  {'=<', '$3', {const, DelTM}},
+                  {'<', {'+', '$1', '$2'}, {const, TM}}}]}])
+    end.
 
 
 %% as lists:delete/2, but delete all exact matches
@@ -1893,3 +1924,13 @@ lists_keydelete(K, N, [T|Ts]) when element(N, T) =:= K ->
     lists_keydelete(K, N, Ts);
 lists_keydelete(K, N, [X|Ts]) ->
     [X|lists_keydelete(K, N, Ts)].
+
+%% as lists:nth/2 but return Default for out of bounds
+lists_nth(0, List, Default) when is_list(List) ->
+    Default;
+lists_nth(1, [H | _], _Default) ->
+    H;
+lists_nth(_N, [], Default) ->
+    Default;
+lists_nth(N, [_ | T], Default) ->
+    lists_nth(N - 1, T, Default).
-- 
2.26.2

Places

File 3493-Optimize-ets-operations.patch of Package erlang

Places