Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
home:Ledest:erlang:23
erlang
2873-Implement-rand-bytes-1-and-rand-bytes_s-2....
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File 2873-Implement-rand-bytes-1-and-rand-bytes_s-2.patch of Package erlang
From c0fbb37cd4e8220f8d331eeb3eac540fd7f20470 Mon Sep 17 00:00:00 2001 From: Raimo Niskanen <raimo@erlang.org> Date: Fri, 7 Aug 2020 17:12:07 +0200 Subject: [PATCH 3/6] Implement rand:bytes/1 and rand:bytes_s/2 --- lib/stdlib/doc/src/rand.xml | 29 +++++ lib/stdlib/src/rand.erl | 57 ++++++++++ lib/stdlib/test/rand_SUITE.erl | 197 ++++++++++++++++++++++++++------- 3 files changed, 245 insertions(+), 38 deletions(-) diff --git a/lib/stdlib/doc/src/rand.xml b/lib/stdlib/doc/src/rand.xml index 0eed1c2e49..9e3c628920 100644 --- a/lib/stdlib/doc/src/rand.xml +++ b/lib/stdlib/doc/src/rand.xml @@ -350,6 +350,35 @@ tests. We suggest to use a sign test to extract a random Boolean value.</pre> </datatypes> <funcs> + <func> + <name name="bytes" arity="1" since="OTP 24.0"/> + <fsummary>Return a random binary.</fsummary> + <desc><marker id="bytes-1"/> + <p> + Returns, for a specified integer <c><anno>N</anno> >= 0</c>, + a <c>binary()</c> with that number of random bytes. + Generates as many random numbers as required using + the selected algorithm to compose the binary, + and updates the state in the process dictionary accordingly. + </p> + </desc> + </func> + + <func> + <name name="bytes_s" arity="2" since="OTP 24.0"/> + <fsummary>Return a random binary.</fsummary> + <desc><marker id="bytes-1"/> + <p> + Returns, for a specified integer <c><anno>N</anno> >= 0</c> + and a state, a <c>binary()</c> with that number of random bytes, + and a new state. + Generates as many random numbers as required using + the selected algorithm to compose the binary, + and the new state. + </p> + </desc> + </func> + <func> <name name="export_seed" arity="0" since="OTP 18.0"/> <fsummary>Export the random number generation state.</fsummary> diff --git a/lib/stdlib/src/rand.erl b/lib/stdlib/src/rand.erl index 1de78a23ac..d5906d3730 100644 --- a/lib/stdlib/src/rand.erl +++ b/lib/stdlib/src/rand.erl @@ -31,6 +31,7 @@ export_seed/0, export_seed_s/1, uniform/0, uniform/1, uniform_s/1, uniform_s/2, uniform_real/0, uniform_real_s/1, + bytes/1, bytes_s/2, jump/0, jump/1, normal/0, normal/2, normal_s/1, normal_s/3 ]). @@ -539,6 +540,62 @@ uniform_real_s(#{max:=_} = AlgHandler, Next, M0, BitNo, R0) -> {V1, R1} = Next(R0), uniform_real_s(AlgHandler, Next, M0, BitNo, R1, ?MASK(56, V1), 56). + +%% bytes/1: given a number N, +%% returns a random binary with N bytes + +-spec bytes(N :: non_neg_integer()) -> Bytes :: binary(). +bytes(N) -> + {Bytes, State} = bytes_s(N, seed_get()), + _ = seed_put(State), + Bytes. + + +%% bytes_s/2: given a number N and a state, +%% returns a random binary with N bytes and a new state + +-spec bytes_s(N :: non_neg_integer(), State :: state()) -> + {Bytes :: binary(), NewState :: state()}. +bytes_s(N, {#{bits:=Bits, next:=Next} = AlgHandler, R}) + when is_integer(N), 0 =< N -> + WeakLowBits = maps:get(weak_low_bits, AlgHandler, 0), + bytes_r(N, AlgHandler, Next, R, Bits, WeakLowBits); +bytes_s(N, {#{max:=Mask, next:=Next} = AlgHandler, R}) + when is_integer(N), 0 =< N, ?MASK(58) =< Mask -> + %% Old spec - assume 58 bits and 2 weak low bits + %% giving 56 bits i.e precisely 7 bytes per generated number + Bits = 58, + WeakLowBits = 2, + bytes_r(N, AlgHandler, Next, R, Bits, WeakLowBits). + +%% N: Number of bytes to generate +%% Bits: Number of bits in the generated word +%% WeakLowBits: Number of low bits in the generated word +%% to waste due to poor quality +bytes_r(N, AlgHandler, Next, R, Bits, WeakLowBits) -> + %% We use whole bytes from each generator word, + %% GoodBytes: that number of bytes + GoodBytes = (Bits - WeakLowBits) bsr 3, + GoodBits = GoodBytes bsl 3, + %% Shift: how many bits of each generator word to waste + %% by shifting right - we use the bits from the big end + Shift = Bits - GoodBits, + bytes_r(N, AlgHandler, Next, R, <<>>, GoodBytes, GoodBits, Shift). +%% +bytes_r(N0, AlgHandler, Next, R0, Bytes0, GoodBytes, GoodBits, Shift) + when GoodBytes < N0 -> + {V, R1} = Next(R0), + Bytes1 = <<Bytes0/binary, (V bsr Shift):GoodBits>>, + N1 = N0 - GoodBytes, + bytes_r(N1, AlgHandler, Next, R1, Bytes1, GoodBytes, GoodBits, Shift); +bytes_r(N, AlgHandler, Next, R0, Bytes, _GoodBytes, GoodBits, _Shift) -> + {V, R1} = Next(R0), + Bits = N bsl 3, + %% Use the big end bits + Shift = GoodBits - Bits, + {<<Bytes/binary, (V bsr Shift):Bits>>, {AlgHandler, R1}}. + + %% jump/1: given a state, jump/1 %% returns a new state which is equivalent to that %% after a large number of call defined for each algorithm. diff --git a/lib/stdlib/test/rand_SUITE.erl b/lib/stdlib/test/rand_SUITE.erl index 230020857f..207cff00e5 100644 --- a/lib/stdlib/test/rand_SUITE.erl +++ b/lib/stdlib/test/rand_SUITE.erl @@ -33,6 +33,7 @@ suite() -> all() -> [seed, interval_int, interval_float, + bytes_count, api_eq, reference, {group, basic_stats}, @@ -45,7 +46,7 @@ all() -> groups() -> [{basic_stats, [parallel], - [basic_stats_uniform_1, basic_stats_uniform_2, + [basic_stats_uniform_1, basic_stats_uniform_2, basic_stats_bytes, basic_stats_standard_normal]}, {distr_stats, [parallel], [stats_standard_normal_box_muller, @@ -81,6 +82,17 @@ test() -> algs() -> [exsss, exrop, exsp, exs1024s, exs64, exsplus, exs1024, exro928ss]. +crypto_support() -> + try crypto:strong_rand_bytes(1) of + <<_>> -> + ok + catch + error : low_entropy -> + low_entropy; + error : undef -> + no_crypto + end. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Test that seed and seed_s and export_seed/0 is working. @@ -164,7 +176,9 @@ api_eq_1(S00) -> V1 = rand:uniform(1000000), {V2, S2} = rand:normal_s(S1), V2 = rand:normal(), - S2 + B3 = rand:bytes(64), + {B3, S3} = rand:bytes_s(64, S2), + S3 end, S1 = lists:foldl(Check, S00, lists:seq(1, 200)), S1 = get(rand_seed), @@ -251,6 +265,25 @@ interval_float_1(N) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Check that bytes/1 and bytes_s/2 generates +%% the right number of bytes + +bytes_count(Config) when is_list(Config) -> + Algs = [default|algs()], + Counts = lists:seq(0, 255), + [begin + _ = rand:seed(Alg), + [begin + ExportState = rand:export_seed(), + B = rand:bytes(N), + {B, _NewState} = rand:bytes_s(N, rand:seed_s(ExportState)), + N = byte_size(B) + end || N <- Counts] + end || Alg <- Algs], + ok. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + %% Check if each algorithm generates the proper sequence. reference(Config) when is_list(Config) -> [reference_1(Alg) || Alg <- algs()], @@ -327,6 +360,13 @@ basic_stats_uniform_2(Config) when is_list(Config) -> || Alg <- [default|algs()]], ok. +basic_stats_bytes(Config) when is_list(Config) -> + ct:timetrap({minutes,15}), %% valgrind needs a lot of time + [basic_bytes( + ?LOOP div 100, rand:seed_s(Alg), 0, array:new(256, [{default, 0}])) + || Alg <- [default|algs()]], + ok. + basic_stats_standard_normal(Config) when is_list(Config) -> ct:timetrap({minutes,6}), %% valgrind needs a lot of time io:format("Testing standard normal~n",[]), @@ -357,6 +397,7 @@ basic_stats_normal(Config) when is_list(Config) -> end, IntendedMeanVariancePairs). + basic_uniform_1(N, S0, Sum, A0) when N > 0 -> {X,S} = case N band 1 of @@ -369,39 +410,78 @@ basic_uniform_1(N, S0, Sum, A0) when N > 0 -> A = array:set(I, 1+array:get(I,A0), A0), basic_uniform_1(N-1, S, Sum+X, A); basic_uniform_1(0, {#{type:=Alg}, _}, Sum, A) -> - AverN = Sum / ?LOOP, - io:format("~.12w: Average: ~.4f~n", [Alg, AverN]), + Loop = ?LOOP, + AverExp = 1.0 / 2, + Buckets = 100, Counters = array:to_list(A), Min = lists:min(Counters), Max = lists:max(Counters), - io:format("~.12w: Min: ~p Max: ~p~n", [Alg, Min, Max]), - - %% Verify that the basic statistics are ok - %% be gentle we don't want to see to many failing tests - abs(0.5 - AverN) < 0.005 orelse ct:fail({average, Alg, AverN}), - abs(?LOOP div 100 - Min) < 1000 orelse ct:fail({min, Alg, Min}), - abs(?LOOP div 100 - Max) < 1000 orelse ct:fail({max, Alg, Max}), - ok. + basic_verify(Alg, Loop, Sum, AverExp, Buckets, Min, Max). basic_uniform_2(N, S0, Sum, A0) when N > 0 -> {X,S} = rand:uniform_s(100, S0), A = array:set(X-1, 1+array:get(X-1,A0), A0), basic_uniform_2(N-1, S, Sum+X, A); basic_uniform_2(0, {#{type:=Alg}, _}, Sum, A) -> - AverN = Sum / ?LOOP, - io:format("~.12w: Average: ~.4f~n", [Alg, AverN]), + Loop = ?LOOP, + AverExp = ((100 - 1) / 2) + 1, + Buckets = 100, Counters = tl(array:to_list(A)), Min = lists:min(Counters), Max = lists:max(Counters), - io:format("~.12w: Min: ~p Max: ~p~n", [Alg, Min, Max]), + basic_verify(Alg, Loop, Sum, AverExp, Buckets, Min, Max). + +basic_bytes(N, S0, Sum0, A0) when N > 0 -> + ByteSize = 100, + {Bin,S} = rand:bytes_s(ByteSize, S0), + {Sum,A} = basic_bytes_incr(Bin, Sum0, A0), + basic_bytes(N-1, S, Sum, A); +basic_bytes(0, {#{type:=Alg}, _}, Sum, A) -> + ByteSize = 100, + Loop = (?LOOP * ByteSize) div 100, + Buckets = 256, + AverExp = (Buckets - 1) / 2, + Counters = array:to_list(A), + Min = lists:min(Counters), + Max = lists:max(Counters), + basic_verify(Alg, Loop, Sum, AverExp, Buckets, Min, Max). +basic_bytes_incr(Bin, Sum, A) -> + basic_bytes_incr(Bin, Sum, A, 0). +%% +basic_bytes_incr(Bin, Sum, A, N) -> + case Bin of + <<_:N/binary, B, _/binary>> -> + basic_bytes_incr( + Bin, Sum+B, array:set(B, array:get(B, A)+1, A), N+1); + <<_/binary>> -> + {Sum,A} + end. + +basic_verify(Alg, Loop, Sum, AverExp, Buckets, Min, Max) -> + AverDiff = AverExp * 0.01, + Aver = Sum / Loop, + io:format( + "~.12w: Expected Average: ~.4f, Allowed Diff: ~.4f, Average: ~.4f~n", + [Alg, AverExp, AverDiff, Aver]), + %% + CountExp = Loop / Buckets, + CountDiff = CountExp * 0.1, + io:format( + "~.12w: Expected Count: ~p, Allowed Diff: ~p, Min: ~p, Max: ~p~n", + [Alg, CountExp, CountDiff, Min, Max]), + %% %% Verify that the basic statistics are ok %% be gentle we don't want to see to many failing tests - abs(50.5 - AverN) < 0.5 orelse ct:fail({average, Alg, AverN}), - abs(?LOOP div 100 - Min) < 1000 orelse ct:fail({min, Alg, Min}), - abs(?LOOP div 100 - Max) < 1000 orelse ct:fail({max, Alg, Max}), + abs(Aver - AverExp) < AverDiff orelse + ct:fail({average, Alg, Aver, AverExp, AverDiff}), + abs(Min - CountExp) < CountDiff orelse + ct:fail({min, Alg, Min, CountExp, CountDiff}), + abs(Max - CountExp) < CountDiff orelse + ct:fail({max, Alg, Max, CountExp, CountDiff}), ok. + basic_normal_1(N, IntendedMean, IntendedVariance, S0, StandardSum, StandardSq) when N > 0 -> {X,S} = normal_s(IntendedMean, IntendedVariance, S0), % We now shape X into a standard normal distribution (in case it wasn't already) @@ -417,6 +497,7 @@ basic_normal_1(0, _IntendedMean, _IntendedVariance, {#{type:=Alg}, _}, StandardS StandardStdDev = math:sqrt(StandardVariance), io:format("~.12w: Standardised Average: ~7.4f, Standardised StdDev ~6.4f~n", [Alg, StandardMean, StandardStdDev]), + %% %% Verify that the basic statistics are ok %% be gentle we don't want to see to many failing tests abs(StandardMean) < 0.005 orelse ct:fail({average, Alg, StandardMean}), @@ -779,8 +860,8 @@ rand_state(Gen) -> %% Test that the user can write algorithms. plugin(Config) when is_list(Config) -> - try crypto:strong_rand_bytes(1) of - <<_>> -> + case crypto_support() of + ok -> _ = lists:foldl( fun(_, S0) -> {V1, S1} = rand:uniform_s(10000, S0), @@ -789,12 +870,9 @@ plugin(Config) when is_list(Config) -> true = is_float(V2), S2 end, crypto64_seed(), lists:seq(1, 200)), - ok - catch - error:low_entropy -> - {skip,low_entropy}; - error:undef -> - {skip,no_crypto} + ok; + Problem -> + {skip,Problem} end. %% Test implementation @@ -855,16 +933,19 @@ measure(Config) -> {(X), (St)} when is_float(X) -> St end). +-define(CHECK_BYTE_SIZE(Gen, Size, Bin, St), + case (Gen) of + {(Bin), (St)} when byte_size(Bin) =:= (Size) -> + St + end). do_measure(_Config) -> Algs = - algs() ++ - try crypto:strong_rand_bytes(1) of - <<_>> -> - [crypto64, crypto_cache, crypto_aes, crypto] - catch - error:low_entropy -> []; - error:undef -> [] + case crypto_support() of + ok -> + algs() ++ [crypto64, crypto_cache, crypto_aes, crypto]; + _ -> + algs() end, %% ct:pal("~nRNG uniform integer range 10000 performance~n",[]), @@ -1021,6 +1102,27 @@ do_measure(_Config) -> end, Algs), %% + ByteSize = 16, % At about 100 bytes crypto_bytes breaks even to exsss + ct:pal("~nRNG ~w bytes performance~n",[ByteSize]), + _ = + measure_1( + fun (_) -> ByteSize end, + fun (State, Size, Mod) -> + measure_loop( + fun (St0) -> + ?CHECK_BYTE_SIZE( + Mod:bytes_s(Size, St0), Size, + Bin, St1) + end, + State) + end, + case crypto_support() of + ok -> + Algs ++ [crypto_bytes, crypto_bytes_cached]; + _ -> + Algs + end), + %% ct:pal("~nRNG uniform float performance~n",[]), _ = measure_1( @@ -1116,6 +1218,10 @@ measure_1(RangeFun, Fun, Alg, TMark) -> crypto_aes, crypto:strong_rand_bytes(256))}; random -> {random, random:seed(os:timestamp()), get(random_seed)}; + crypto_bytes -> + {?MODULE, ignored_state}; + crypto_bytes_cached -> + {?MODULE, <<>>}; _ -> {rand, rand:seed_s(Alg)} end, @@ -1139,6 +1245,21 @@ measure_1(RangeFun, Fun, Alg, TMark) -> {Pid, Msg} -> Msg end. +%% Comparison algorithm for rand:bytes_s/2 vs. crypto:strong_rand_bytes/1 +bytes_s(N, Cache) when is_binary(Cache) -> + %% crypto_bytes_cached + case Cache of + <<Bytes:N/binary, Rest/binary>> -> + {Bytes, Rest}; + <<Part/binary>> -> + <<Bytes:N/binary, Rest/binary>> = + <<Part/binary, (crypto:strong_rand_bytes(N * 16))/binary>>, + {Bytes, Rest} + end; +bytes_s(N, ignored_state = St) -> + %% crypto_bytes + {crypto:strong_rand_bytes(N), St}. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% The jump sequence tests has two parts %% for those with the functional API (jump/1) @@ -1268,16 +1389,16 @@ short_jump(Config) when is_list(Config) -> fun ({Alg,AlgState}) -> {Alg,rand:exro928_jump_2pow20(AlgState)} end), - try crypto:strong_rand_bytes(1) of - _ -> + case crypto_support() of + ok -> short_jump( crypto:rand_seed_alg_s(crypto_aes, integer_to_list(Seed)), fun ({Alg,AlgState}) -> {Alg,crypto:rand_plugin_aes_jump_2pow20(AlgState)} end), - ok - catch error:undef -> - {skip,no_crypto} + ok; + Problem -> + {skip,Problem} end. short_jump({#{bits := Bits},_} = State_0, Jump2Pow20) -> -- 2.26.2
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
