Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
home:Ledest:erlang:24
erlang
0286-dialyzer-Parallelize-remote-types-processi...
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File 0286-dialyzer-Parallelize-remote-types-processing-in-cont.patch of Package erlang
From 12f0567f701a2ade95b528cc64e69713583e19ba Mon Sep 17 00:00:00 2001 From: Hans Bolinder <hasse@erlang.org> Date: Sun, 1 Aug 2021 09:09:44 +0200 Subject: [PATCH 6/7] dialyzer: Parallelize remote types processing in contracts This has effect if many processes can run i parallel. --- lib/dialyzer/src/dialyzer_codeserver.erl | 19 ++-- lib/dialyzer/src/dialyzer_contracts.erl | 108 ++++++++++++++++------ lib/dialyzer/src/dialyzer_coordinator.erl | 31 +++++-- lib/dialyzer/src/dialyzer_utils.erl | 2 +- lib/dialyzer/src/dialyzer_worker.erl | 12 ++- lib/dialyzer/src/erl_types.erl | 15 +-- 6 files changed, 134 insertions(+), 53 deletions(-) diff --git a/lib/dialyzer/src/dialyzer_codeserver.erl b/lib/dialyzer/src/dialyzer_codeserver.erl index 0472e2c128..b25ea791d2 100644 --- a/lib/dialyzer/src/dialyzer_codeserver.erl +++ b/lib/dialyzer/src/dialyzer_codeserver.erl @@ -29,7 +29,7 @@ finalize_records/1, get_contracts/1, get_callbacks/1, - get_exported_types/1, + get_exported_types_table/1, extract_exported_types/1, get_exports/1, get_records_table/1, @@ -132,12 +132,15 @@ new() -> CodeOptions = [compressed, public, {read_concurrency, true}], Code = ets:new(dialyzer_codeserver_code, CodeOptions), ReadOptions = [compressed, {read_concurrency, true}], - [Contracts, Callbacks, Records, ExportedTypes] = + [Records, ExportedTypes] = [ets:new(Name, ReadOptions) || - Name <- [dialyzer_codeserver_contracts, - dialyzer_codeserver_callbacks, - dialyzer_codeserver_records, + Name <- [dialyzer_codeserver_records, dialyzer_codeserver_exported_types]], + ReadWriteOptions = [public | ReadOptions], + [Contracts, Callbacks] = + [ets:new(Name, ReadWriteOptions) || + Name <- [dialyzer_codeserver_contracts, + dialyzer_codeserver_callbacks]], TempOptions = [public, {write_concurrency, true}], [Exports, FunMetaInfo, TempExportedTypes, TempRecords, TempContracts, TempCallbacks] = @@ -211,10 +214,10 @@ insert_fun_meta_info(List, #codeserver{fun_meta_info = FunMetaInfo} = CS) -> is_exported(MFA, #codeserver{exports = Exports}) -> ets_set_is_element(MFA, Exports). --spec get_exported_types(codeserver()) -> sets:set(mfa()). +-spec get_exported_types_table(codeserver()) -> map_ets(). -get_exported_types(#codeserver{exported_types = ExpTypes}) -> - ets_set_to_set(ExpTypes). +get_exported_types_table(#codeserver{exported_types = ExpTypes}) -> + ExpTypes. -spec extract_exported_types(codeserver()) -> {codeserver(), set_ets()}. diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl index 5adcc78a0e..e434c64e4e 100644 --- a/lib/dialyzer/src/dialyzer_contracts.erl +++ b/lib/dialyzer/src/dialyzer_contracts.erl @@ -27,12 +27,23 @@ process_contract_remote_types/1, store_tmp_contract/6]). --export_type([file_contract/0, plt_contracts/0]). +%% For dialyzer_worker. +-export([process_contract_remote_types_module/2]). + +-export_type([file_contract/0, plt_contracts/0, + contract_remote_types_init_data/0, + contract_remote_types_result/0]). %%----------------------------------------------------------------------- -include("dialyzer.hrl"). +-type ext_types_message() :: {pid(), 'ext_types', + {mfa(), {file:filename(), erl_anno:location()}}} + | {'error', io_lib:chars()}. +-type contract_remote_types_init_data() :: dialyzer_codeserver:codeserver(). +-type contract_remote_types_result() :: [ext_types_message()]. + %%----------------------------------------------------------------------- %% Types used in other parts of the system below %%----------------------------------------------------------------------- @@ -143,35 +154,76 @@ sequence([H|T], Delimiter) -> H ++ Delimiter ++ sequence(T, Delimiter). dialyzer_codeserver:codeserver(). process_contract_remote_types(CodeServer) -> - Mods = dialyzer_codeserver:all_temp_modules(CodeServer), + case dialyzer_codeserver:all_temp_modules(CodeServer) of + [] -> + CodeServer; + Mods -> + %% CodeServer is updated by each worker, but is still valid + %% after updates. Workers call + %% process_contract_remote_types_module/2 below. + Return = + dialyzer_coordinator:parallel_job(contract_remote_types, + Mods, + _InitData=CodeServer, + _Timing=none), + %% We need to pass on messages and thrown errors from erl_types: + _ = [self() ! {self(), ext_types, ExtType} || + {_, ext_types, ExtType} <- Return], + case [Error || {error, _} = Error <- Return] of + [] -> + dialyzer_codeserver:finalize_contracts(CodeServer); + [Error | _] -> + throw(Error) + end + end. + +-spec process_contract_remote_types_module(module(), + dialyzer_codeserver:codeserver()) -> [ext_types_message()]. + +process_contract_remote_types_module(ModuleName, CodeServer) -> RecordTable = dialyzer_codeserver:get_records_table(CodeServer), - ExpTypes = dialyzer_codeserver:get_exported_types(CodeServer), - ModuleFun = - fun(ModuleName) -> - ContractFun = - fun({MFA, {File, TmpContract, Xtra}}, C0) -> - #tmp_contract{contract_funs = CFuns, forms = Forms} = TmpContract, - {NewCs, C2} = lists:mapfoldl(fun(CFun, C1) -> - CFun(ExpTypes, RecordTable, C1) - end, C0, CFuns), - Args = general_domain(NewCs), - Contract = #contract{contracts = NewCs, args = Args, forms = Forms}, - {{MFA, {File, Contract, Xtra}}, C2} - end, - Cache = erl_types:cache__new(), - {ContractMap, CallbackMap} = - dialyzer_codeserver:get_temp_contracts(ModuleName, CodeServer), - {NewContractList, Cache1} = - lists:mapfoldl(ContractFun, Cache, maps:to_list(ContractMap)), - {NewCallbackList, _NewCache} = - lists:mapfoldl(ContractFun, Cache1, maps:to_list(CallbackMap)), - dialyzer_codeserver:store_contracts(ModuleName, - maps:from_list(NewContractList), - maps:from_list(NewCallbackList), - CodeServer) + ExpTypes = dialyzer_codeserver:get_exported_types_table(CodeServer), + ContractFun = + fun({MFA, {File, TmpContract, Xtra}}, C0) -> + #tmp_contract{contract_funs = CFuns, forms = Forms} = TmpContract, + {NewCs, C2} = lists:mapfoldl(fun(CFun, C1) -> + CFun(ExpTypes, RecordTable, C1) + end, C0, CFuns), + Args = general_domain(NewCs), + Contract = #contract{contracts = NewCs, args = Args, forms = Forms}, + {{MFA, {File, Contract, Xtra}}, C2} end, - lists:foreach(ModuleFun, Mods), - dialyzer_codeserver:finalize_contracts(CodeServer). + Cache = erl_types:cache__new(), + {ContractMap, CallbackMap} = + dialyzer_codeserver:get_temp_contracts(ModuleName, CodeServer), + try + {NewContractList, Cache1} = + lists:mapfoldl(ContractFun, Cache, maps:to_list(ContractMap)), + {NewCallbackList, _NewCache} = + lists:mapfoldl(ContractFun, Cache1, maps:to_list(CallbackMap)), + _NewCodeServer = + dialyzer_codeserver:store_contracts(ModuleName, + maps:from_list(NewContractList), + maps:from_list(NewCallbackList), + CodeServer), + rcv_ext_types() + catch + throw:{error, _}=Error -> + [Error] ++ rcv_ext_types() + end. + +rcv_ext_types() -> + Self = self(), + Self ! {Self, done}, + rcv_ext_types(Self, []). + +rcv_ext_types(Self, ExtTypes) -> + receive + {Self, ext_types, _} = ExtType -> + rcv_ext_types(Self, [ExtType | ExtTypes]); + {Self, done} -> + lists:usort(ExtTypes) + end. -type fun_types() :: orddict:orddict(label(), erl_types:type_table()). diff --git a/lib/dialyzer/src/dialyzer_coordinator.erl b/lib/dialyzer/src/dialyzer_coordinator.erl index 7c1bc1de5a..f8a078364b 100644 --- a/lib/dialyzer/src/dialyzer_coordinator.erl +++ b/lib/dialyzer/src/dialyzer_coordinator.erl @@ -43,34 +43,44 @@ -type timing() :: dialyzer_timing:timing_server(). -type scc() :: [mfa_or_funlbl()]. --type mode() :: 'typesig' | 'dataflow' | 'compile' | 'warnings'. +-type mode() :: 'typesig' | 'dataflow' | 'compile' | 'warnings' | + 'contract_remote_types'. -type compile_job() :: file:filename(). -type typesig_job() :: scc(). -type dataflow_job() :: module(). -type warnings_job() :: module(). +-type contract_remote_types_job() :: module(). --type job() :: compile_job() | typesig_job() | dataflow_job() | warnings_job(). +-type job() :: compile_job() | typesig_job() | dataflow_job() | + warnings_job() | contract_remote_types_job(). -type compile_init_data() :: dialyzer_analysis_callgraph:compile_init_data(). -type typesig_init_data() :: dialyzer_succ_typings:typesig_init_data(). -type dataflow_init_data() :: dialyzer_succ_typings:dataflow_init_data(). -type warnings_init_data() :: dialyzer_succ_typings:warnings_init_data(). +-type contract_remote_types_init_data() :: + dialyzer_contracts:contract_remote_types_init_data(). -type compile_result() :: dialyzer_analysis_callgraph:compile_result(). -type typesig_result() :: [mfa_or_funlbl()]. -type dataflow_result() :: [mfa_or_funlbl()]. -type warnings_result() :: [dial_warning()]. +-type contract_remote_types_result() :: + dialyzer_contracts:contract_remote_types_result(). -type init_data() :: compile_init_data() | typesig_init_data() | - dataflow_init_data() | warnings_init_data(). + dataflow_init_data() | warnings_init_data() | + contract_remote_types_init_data(). -type result() :: compile_result() | typesig_result() | - dataflow_result() | warnings_result(). + dataflow_result() | warnings_result() | + contract_remote_types_result(). -type job_result() :: dialyzer_analysis_callgraph:one_file_mid_error() | dialyzer_analysis_callgraph:one_file_result_ok() | - typesig_result() | dataflow_result() | warnings_result(). + typesig_result() | dataflow_result() | + warnings_result() | contract_remote_types_result(). -record(state, {mode :: mode(), active = 0 :: integer(), @@ -94,7 +104,10 @@ ('dataflow', [dataflow_job()], dataflow_init_data(), timing()) -> dataflow_result(); ('warnings', [warnings_job()], warnings_init_data(), - timing()) -> warnings_result(). + timing()) -> warnings_result(); + ('contract_remote_types', [contract_remote_types_job()], + contract_remote_types_init_data(), timing()) -> + contract_remote_types_result(). parallel_job(Mode, Jobs, InitData, Timing) -> State = spawn_jobs(Mode, Jobs, InitData, Timing), @@ -158,7 +171,9 @@ collect_result(#state{mode = Mode, active = Active, result = Result, ets:delete(SCCtoPID), NewResult; 'warnings' -> - NewResult + NewResult; + 'contract_remote_types' -> + NewResult end; N -> case TypesigOrDataflow of @@ -187,6 +202,8 @@ update_result(Mode, InitData, Job, Data, Result) -> X when X =:= 'typesig'; X =:= 'dataflow' -> dialyzer_succ_typings:lookup_names(Data, InitData) ++ Result; 'warnings' -> + Data ++ Result; + 'contract_remote_types' -> Data ++ Result end. diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl index c4ac96c9dc..e3a3976366 100644 --- a/lib/dialyzer/src/dialyzer_utils.erl +++ b/lib/dialyzer/src/dialyzer_utils.erl @@ -233,7 +233,7 @@ get_record_fields([], _RecDict, Acc) -> %% The field types are cached. Used during analysis when handling records. process_record_remote_types(CServer) -> - ExpTypes = dialyzer_codeserver:get_exported_types(CServer), + ExpTypes = dialyzer_codeserver:get_exported_types_table(CServer), Mods = dialyzer_codeserver:all_temp_modules(CServer), process_opaque_types0(Mods, CServer, ExpTypes), VarTable = erl_types:var_table__new(), diff --git a/lib/dialyzer/src/dialyzer_worker.erl b/lib/dialyzer/src/dialyzer_worker.erl index af0f2e9e08..64de596da7 100644 --- a/lib/dialyzer/src/dialyzer_worker.erl +++ b/lib/dialyzer/src/dialyzer_worker.erl @@ -66,7 +66,8 @@ init(#state{job = SCC, mode = Mode, init_data = InitData, DependsOn = [{Pid, erlang:monitor(process, Pid)} || Pid <- Pids], loop(updating, State#state{depends_on = DependsOn}); init(#state{mode = Mode} = State) when - Mode =:= 'compile'; Mode =:= 'warnings' -> + Mode =:= 'compile'; Mode =:= 'warnings'; + Mode =:= 'contract_remote_types' -> loop(running, State). loop(updating, #state{mode = Mode} = State) when @@ -83,6 +84,11 @@ loop(waiting, #state{mode = Mode} = State) when ?debug("~w: Wait: ~p\n", [self(), State#state.job]), NewState = wait_for_success_typings(State), loop(updating, NewState); +loop(running, #state{mode = 'contract_remote_types'} = State) -> + request_activation(State), + ?debug("~w: Remote types: ~p\n", [self(), State#state.job]), + Result = do_work(State), + report_to_coordinator(Result, State); loop(running, #state{mode = 'compile'} = State) -> request_activation(State), ?debug("Compile: ~s\n",[State#state.job]), @@ -127,7 +133,9 @@ request_activation(#state{coordinator = Coordinator}) -> do_work(#state{mode = Mode, job = Job, init_data = InitData}) -> case Mode of typesig -> dialyzer_succ_typings:find_succ_types_for_scc(Job, InitData); - dataflow -> dialyzer_succ_typings:refine_one_module(Job, InitData) + dataflow -> dialyzer_succ_typings:refine_one_module(Job, InitData); + contract_remote_types -> + dialyzer_contracts:process_contract_remote_types_module(Job, InitData) end. report_to_coordinator(Result, #state{job = Job, coordinator = Coordinator}) -> diff --git a/lib/dialyzer/src/erl_types.erl b/lib/dialyzer/src/erl_types.erl index e381bfb64e..e218d2b051 100644 --- a/lib/dialyzer/src/erl_types.erl +++ b/lib/dialyzer/src/erl_types.erl @@ -4663,6 +4663,7 @@ mod_name(Mod, Name) -> [erl_type()], type_names()}. -type mod_type_table() :: ets:tid(). -type mod_records() :: dict:dict(module(), type_table()). +-type exported_type_table() :: ets:tid(). -record(cache, { types = maps:new() :: #{cache_key() => {erl_type(), expand_limit()}}, @@ -4671,7 +4672,7 @@ mod_name(Mod, Name) -> -opaque cache() :: #cache{}. --spec t_from_form(parse_form(), sets:set(mfa()), site(), mod_type_table(), +-spec t_from_form(parse_form(), exported_type_table(), site(), mod_type_table(), var_table(), cache()) -> {erl_type(), cache()}. t_from_form(Form, ExpTypes, Site, RecDict, VarTab, Cache) -> @@ -4696,12 +4697,12 @@ t_from_form_without_remote(Form, Site, TypeTable) -> -type expand_depth() :: integer(). -record(from_form, {site :: site(), - xtypes :: sets:set(mfa()) | 'replace_by_none', + xtypes :: exported_type_table() | 'replace_by_none', mrecs :: 'undefined' | mod_type_table(), vtab :: var_table(), tnames :: type_names()}). --spec t_from_form_check_remote(parse_form(), sets:set(mfa()), site(), +-spec t_from_form_check_remote(parse_form(), exported_type_table(), site(), mod_type_table()) -> 'ok'. t_from_form_check_remote(Form, ExpTypes, Site, RecDict) -> State = #from_form{site = Site, @@ -4722,7 +4723,7 @@ t_from_form_check_remote(Form, ExpTypes, Site, RecDict) -> %% types balanced (unions will otherwise collapse to any()) by limiting %% the depth the same way as t_limit/2 does. --spec t_from_form1(parse_form(), sets:set(mfa()) | 'replace_by_none', +-spec t_from_form1(parse_form(), exported_type_table() | 'replace_by_none', site(), 'undefined' | mod_type_table(), var_table(), cache()) -> {erl_type(), cache()}. @@ -5074,7 +5075,7 @@ remote_from_form(Anno, RemMod, Name, Args, S, D, L, C) -> self() ! {self(), ext_types, ext_types_message(MFA, Anno, Site)}, {t_any(), L, C}; {RemDict, C1} -> - case sets:is_element(MFA, ET) of + case ets:member(ET, MFA) of true -> RemType = {type, MFA}, case can_unfold_more(RemType, TypeNames) of @@ -5345,7 +5346,7 @@ recur_limit(Fun, D, L, TypeName, TypeNames) -> Fun(D, L) end. --spec t_check_record_fields(parse_form(), sets:set(mfa()), site(), +-spec t_check_record_fields(parse_form(), exported_type_table(), site(), mod_type_table(), var_table(), cache()) -> cache(). t_check_record_fields(Form, ExpTypes, Site, RecDict, VarTable, Cache) -> @@ -5576,7 +5577,7 @@ t_form_to_string({type, _Anno, Name, []} = T) -> V = var_table__new(), C = cache__new(), State = #from_form{site = Site, - xtypes = sets:new(), + xtypes = replace_by_none, mrecs = 'undefined', vtab = V, tnames = []}, -- 2.31.1
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
