source/interprocedural/fixpointAnalysis.ml (674 lines of code) (raw):
(*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*)
open Core
open Ast
open Pyre
open Statement
module Kind = AnalysisKind
let initialize_configuration kind ~static_analysis_configuration =
let (AnalysisResult.Analysis { analysis; _ }) = AnalysisResult.get_abstract_analysis kind in
let module Analysis = (val analysis) in
Analysis.initialize_configuration ~static_analysis_configuration
(* Initialize models for the given analysis.
* For the taint analysis, this parses taint stubs into models and queries. *)
let initialize_models kind ~scheduler ~static_analysis_configuration ~environment ~callables ~stubs =
let (AnalysisResult.Analysis { analysis; kind = storable_kind }) =
AnalysisResult.get_abstract_analysis kind
in
let module Analysis = (val analysis) in
let specialize_models { AnalysisResult.initial_models = analysis_initial_models; skip_overrides } =
let to_model_t model =
let pkg = AnalysisResult.Pkg { kind = ModelPart storable_kind; value = model } in
{ AnalysisResult.models = Kind.Map.add kind pkg Kind.Map.empty; is_obscure = false }
in
{
AnalysisResult.initial_models = analysis_initial_models |> Target.Map.map ~f:to_model_t;
skip_overrides;
}
in
Analysis.initialize_models
~static_analysis_configuration
~scheduler
~environment
~callables
~stubs
|> specialize_models
module Testing = struct
(* This signature for record_initial_models is unreachable in production, we only have initial
models for callable_t. But as a hack the integration tests sometimes directly store initial
models on non-callable targets *)
let record_initial_models ~targets ~stubs models =
let record_models models =
let add_model_to_memory ~key:call_target ~data:model =
FixpointState.add_predefined FixpointState.Epoch.initial call_target model
in
Target.Map.iteri models ~f:add_model_to_memory
in
(* Augment models with initial inferred and obscure models *)
let add_missing_initial_models models =
targets
|> List.filter ~f:(fun target -> not (Target.Map.mem models target))
|> List.fold ~init:models ~f:(fun models target ->
Target.Map.set models ~key:target ~data:AnalysisResult.empty_model)
in
let add_missing_obscure_models models =
(stubs :> Target.t list)
|> List.filter ~f:(fun target -> not (Target.Map.mem models target))
|> List.fold ~init:models ~f:(fun models target ->
Target.Map.set models ~key:target ~data:AnalysisResult.obscure_model)
in
models |> add_missing_initial_models |> add_missing_obscure_models |> record_models
end
(* Save initial models in the shared memory. *)
let record_initial_models ~callables ~stubs models =
Testing.record_initial_models ~targets:(callables :> Target.t list) ~stubs models
let analysis_failed step ~exn callable ~message =
let callable = (callable :> Target.t) in
Log.error
"%s in step %s while analyzing %s.\nException %s\nBacktrace: %s"
message
(FixpointState.show_step step)
(Target.show callable)
(Exn.to_string exn)
(Printexc.get_backtrace ());
raise exn
let get_empty_model (type a) (kind : < model : a ; .. > AnalysisResult.storable_kind) : a =
(* Existentially abstract unspecified analysis data *)
let (AnalysisResult.ModelPart k1) = AnalysisResult.ModelPart kind in
let module Analysis = (val AnalysisResult.get_analysis k1) in
Analysis.empty_model
let get_obscure_models abstract_analysis =
let models =
let (AnalysisResult.Analysis { kind; analysis }) = abstract_analysis in
let module Analysis = (val analysis) in
let obscure_model = Analysis.obscure_model in
Kind.Map.add
(Kind.abstract kind)
(AnalysisResult.Pkg { kind = ModelPart kind; value = obscure_model })
Kind.Map.empty
in
AnalysisResult.{ models; is_obscure = true }
let non_fixpoint_witness
(type a)
(kind : < model : a ; .. > AnalysisResult.storable_kind)
~(iteration : int)
~(previous : a)
~(next : a)
=
(* Existentially abstract unspecified analysis data *)
let (AnalysisResult.ModelPart k1) = AnalysisResult.ModelPart kind in
let module Analysis = (val AnalysisResult.get_analysis k1) in
if Analysis.reached_fixpoint ~iteration ~previous ~next then
None
else
Some ()
(* Key is witness of non-fixpoint. *)
(* Find witnesses where next </= previous. *)
let non_fixpoint_witness ~iteration _kind left right =
let open AnalysisResult in
match left, right with
| None, None -> failwith "impossible"
| None, Some (Pkg { kind = ModelPart kind; value = next }) ->
let empty = get_empty_model kind in
non_fixpoint_witness kind ~iteration ~previous:empty ~next
| Some (Pkg { kind = ModelPart kind; value = previous }), None ->
let empty = get_empty_model kind in
non_fixpoint_witness kind ~iteration ~previous ~next:empty
| ( Some (Pkg { kind = ModelPart k1; value = previous }),
Some (Pkg { kind = ModelPart k2; value = next }) ) -> (
match Kind.are_equal k1 k2 with
| Kind.Equal -> non_fixpoint_witness k1 ~iteration ~previous ~next
| Kind.Distinct -> failwith "Wrong kind matched up in fixpoint test.")
let reached_fixpoint_model_only ~iteration ~previous ~next =
Kind.Map.merge (non_fixpoint_witness ~iteration) previous next |> Kind.Map.is_empty
let join_models ~iteration left right =
let open AnalysisResult in
let join_pkg
_
(Pkg { kind = ModelPart left_kind; value = left })
(Pkg { kind = ModelPart right_kind; value = right })
=
match Kind.are_equal left_kind right_kind with
| Kind.Equal ->
let module Analysis = (val get_analysis left_kind) in
Some (Pkg { kind = ModelPart left_kind; value = Analysis.join ~iteration left right })
| Kind.Distinct -> failwith "Wrong kind matched up in join."
in
if phys_equal left right then
left
else
Kind.Map.union join_pkg left right
let widen_models ~iteration ~previous ~next =
let open AnalysisResult in
let widen_pkg
_
(Pkg { kind = ModelPart k1; value = previous })
(Pkg { kind = ModelPart k2; value = next })
=
match Kind.are_equal k1 k2 with
| Kind.Equal ->
let module Analysis = (val AnalysisResult.get_analysis k1) in
Some (Pkg { kind = ModelPart k1; value = Analysis.widen ~iteration ~previous ~next })
| Kind.Distinct -> failwith "Wrong kind matched up in widen."
in
if phys_equal previous next then
previous
else
Kind.Map.union widen_pkg previous next
let explain_non_fixpoint ~iteration ~previous ~next =
let open AnalysisResult in
let witnesses = Kind.Map.merge (non_fixpoint_witness ~iteration) previous next in
let print_witness key () =
Log.log ~section:`Interprocedural "%s is non-fixpoint" (Kind.show key)
in
Kind.Map.iter print_witness witnesses
let show_models models =
let open AnalysisResult in
(* list them to make the type system do its work *)
let to_string (abstract_kind, Pkg { kind = ModelPart kind; value = model }) =
let module Analysis = (val get_analysis kind) in
Format.sprintf "%s: %s" (Kind.show abstract_kind) (Analysis.show_call_model model)
in
Kind.Map.bindings models |> List.map ~f:to_string |> String.concat ~sep:"\n"
let widen ~iteration ~previous ~next =
let verify_widen ~iteration ~previous ~next =
let result = widen_models ~iteration ~previous ~next in
if not (reached_fixpoint_model_only ~iteration ~previous:result ~next:previous) then (
Log.error
"WIDEN DOES NOT RESPECT JOIN: previous = %s\nwiden = %s\n"
(show_models previous)
(show_models result);
explain_non_fixpoint ~iteration ~previous:result ~next:previous);
if not (reached_fixpoint_model_only ~iteration ~previous:result ~next) then (
Log.error
"WIDEN DOES NOT RESPECT JOIN: next = %s\nwiden = %s\n"
(show_models next)
(show_models result);
explain_non_fixpoint ~iteration ~previous:result ~next);
result
in
if phys_equal previous next then
previous
else
verify_widen ~iteration ~previous ~next
let reached_fixpoint
~iteration
~previous:{ AnalysisResult.models = previous_model; is_obscure = previous_obscure }
~next:{ AnalysisResult.models = next_model; is_obscure = next_obscure }
=
if (not previous_obscure) && next_obscure then
false
else
reached_fixpoint_model_only ~iteration ~previous:previous_model ~next:next_model
let widen
~iteration
~previous:{ AnalysisResult.models = previous_model; is_obscure = previous_obscure }
~next:{ AnalysisResult.models = next_model; is_obscure = next_obscure }
=
AnalysisResult.
{
is_obscure = previous_obscure || next_obscure;
models = widen ~iteration ~previous:previous_model ~next:next_model;
}
let widen_if_necessary step callable ~old_model ~new_model result =
(* Check if we've reached a fixed point *)
if reached_fixpoint ~iteration:step.FixpointState.iteration ~previous:old_model ~next:new_model
then (
Log.log
~section:`Interprocedural
"Reached fixpoint for %a\n%a"
Target.pretty_print
callable
AnalysisResult.pp_model_t
old_model;
FixpointState.{ is_partial = false; model = old_model; result })
else
let model = widen ~iteration:step.FixpointState.iteration ~previous:old_model ~next:new_model in
Log.log
~section:`Interprocedural
"Widened fixpoint for %a\nold: %anew: %a\nwidened: %a"
Target.pretty_print
callable
AnalysisResult.pp_model_t
old_model
AnalysisResult.pp_model_t
new_model
AnalysisResult.pp_model_t
model;
FixpointState.{ is_partial = true; model; result }
let analyze_define
step
abstract_analysis
callable
environment
qualifier
({ Node.value = { Define.signature = { name; _ }; _ }; _ } as define)
=
let () = Log.log ~section:`Interprocedural "Analyzing %a" Target.pp_callable_t callable in
let old_model =
match FixpointState.get_old_model callable with
| Some model ->
let () =
Log.log
~section:`Interprocedural
"Analyzing %a, with initial model %a"
Target.pp_callable_t
callable
AnalysisResult.pp_model_t
model
in
model
| None ->
Format.asprintf "No initial model found for %a" Target.pretty_print callable |> failwith
in
let models, results =
try
let (AnalysisResult.Analysis { AnalysisResult.kind; analysis }) = abstract_analysis in
let open AnalysisResult in
let abstract_kind = Kind.abstract kind in
let module Analysis = (val analysis) in
let existing = AnalysisResult.get (ModelPart kind) old_model.models in
let result, model = Analysis.analyze ~callable ~environment ~qualifier ~define ~existing in
( Kind.Map.add abstract_kind (Pkg { kind = ModelPart kind; value = model }) Kind.Map.empty,
Kind.Map.add abstract_kind (Pkg { kind = ResultPart kind; value = result }) Kind.Map.empty )
with
| Analysis.ClassHierarchy.Untracked annotation ->
Log.log
~section:`Info
"Could not generate model for `%a` due to invalid annotation `%s`"
Reference.pp
name
annotation;
AnalysisResult.Kind.Map.empty, AnalysisResult.Kind.Map.empty
| Sys.Break as exn -> analysis_failed step ~exn ~message:"Hit Ctrl+C" callable
| _ as exn -> analysis_failed step ~exn ~message:"Analysis failed" callable
in
let new_model = { AnalysisResult.models; is_obscure = false } in
widen_if_necessary step callable ~old_model ~new_model results
let strip_for_callsite model =
let open AnalysisResult in
(* list them to make the type system do its work *)
let strip abstract_kind (Pkg { kind = ModelPart kind; value = model }) models =
let module Analysis = (val get_analysis kind) in
let model = Analysis.strip_for_callsite model in
Kind.Map.add abstract_kind (Pkg { kind = ModelPart kind; value = model }) models
in
let models = Kind.Map.fold strip model.models Kind.Map.empty in
{ model with models }
let analyze_overrides ({ FixpointState.iteration; _ } as step) callable =
let timer = Timer.start () in
let overrides =
DependencyGraphSharedMemory.get_overriding_types
~member:(Target.get_override_reference callable)
|> Option.value ~default:[]
|> List.map ~f:(fun at_type -> Target.create_derived_override callable ~at_type)
in
let model =
let lookup_and_join ({ AnalysisResult.is_obscure; models } as result) override =
match FixpointState.get_model override with
| None ->
Log.log
~section:`Interprocedural
"During override analysis, can't find model for %a"
Target.pretty_print
override;
result
| Some model ->
let model = strip_for_callsite model in
{
is_obscure = is_obscure || model.is_obscure;
models = join_models ~iteration models model.models;
}
in
let direct_model =
FixpointState.get_model (Target.get_corresponding_method callable)
|> Option.value ~default:AnalysisResult.empty_model
|> strip_for_callsite
in
List.fold overrides ~f:lookup_and_join ~init:direct_model
in
let old_model =
match FixpointState.get_old_model callable with
| Some model -> model
| None ->
Format.asprintf "No initial model found for %a" Target.pretty_print callable |> failwith
in
let state =
widen_if_necessary step callable ~old_model ~new_model:model AnalysisResult.empty_result
in
Statistics.performance
~randomly_log_every:1000
~always_log_time_threshold:1.0 (* Seconds *)
~name:"Override analysis"
~section:`Interprocedural
~normals:["callable", callable |> Target.get_override_reference |> Reference.show]
~timer
();
state
let callables_to_dump =
(* Set of defines we'll print models for at the end of the analysis. *)
ref Target.Set.empty
let analyze_callable analysis step callable environment =
let resolution = Analysis.TypeEnvironment.ReadOnly.global_resolution environment in
let () =
(* Verify invariants *)
let open FixpointState in
match get_meta_data callable with
| None -> ()
| Some { step = { epoch; _ }; _ } when epoch <> step.epoch ->
let message =
Format.sprintf
"Fixpoint inconsistency: callable %s analyzed during epoch %s, but stored metadata \
from epoch %s"
(Target.show callable)
(Epoch.show step.epoch)
(Epoch.show epoch)
in
Log.error "%s" message;
failwith message
| _ -> ()
in
match callable with
| #Target.callable_t as callable -> (
match Target.get_module_and_definition callable ~resolution with
| None ->
let () = Log.error "Found no definition for %s" (Target.show callable) in
let () =
if not (FixpointState.is_initial_iteration step) then (
let message =
Format.sprintf
"Fixpoint inconsistency: Target %s without body analyzed past initial step: %s"
(Target.show callable)
(FixpointState.show_step step)
in
Log.error "%s" message;
failwith message)
in
FixpointState.
{
is_partial = false;
model = get_obscure_models analysis;
result = AnalysisResult.empty_result;
}
| Some (qualifier, ({ Node.value; _ } as define)) ->
if Define.dump value then
callables_to_dump := Target.Set.add callable !callables_to_dump;
analyze_define step analysis callable environment qualifier define)
| #Target.override_t as callable -> analyze_overrides step callable
| #Target.object_t as path ->
Format.asprintf "Found object %a in fixpoint analysis" Target.pp path |> failwith
type expensive_callable = {
time_to_analyze_in_ms: int;
callable: Target.t;
}
type result = {
callables_processed: int;
expensive_callables: expensive_callable list;
callables_to_dump: Target.Set.t;
}
(* Called on a worker with a set of targets to analyze. *)
let one_analysis_pass ~analysis ~step ~environment ~callables =
let analysis = AnalysisResult.get_abstract_analysis analysis in
let analyze_and_cache expensive_callables callable =
let timer = Timer.start () in
let result = analyze_callable analysis step callable environment in
FixpointState.add step callable result;
(* Log outliers. *)
let time_to_analyze_in_ms = Timer.stop_in_ms timer in
if time_to_analyze_in_ms > 500 then begin
Statistics.performance
~name:"static analysis of expensive callable"
~timer
~section:`Interprocedural
~normals:["callable", Target.show callable]
();
{ time_to_analyze_in_ms; callable } :: expensive_callables
end
else
expensive_callables
in
let expensive_callables = List.fold callables ~f:analyze_and_cache ~init:[] in
{
callables_processed = List.length callables;
callables_to_dump = !callables_to_dump;
expensive_callables;
}
let get_callable_dependents ~dependencies callable =
Target.Map.find dependencies callable |> Option.value ~default:[]
let compute_callables_to_reanalyze
step
previous_batch
~dependencies
~filtered_callables
~all_callables
=
let open FixpointState in
let reanalyze_caller caller accumulator = Target.Set.add caller accumulator in
let might_change_if_reanalyzed =
List.fold previous_batch ~init:Target.Set.empty ~f:(fun accumulator callable ->
if not (get_is_partial callable) then
accumulator
else
(* c must be re-analyzed next iteration because its result has changed, and therefore its
callers must also be reanalyzed. *)
let callers = get_callable_dependents ~dependencies callable in
List.fold
callers
~init:(Target.Set.add callable accumulator)
~f:(fun accumulator caller -> reanalyze_caller caller accumulator))
in
(* Filter the original list in order to preserve topological sort order. *)
let callables_to_reanalyze =
List.filter all_callables ~f:(fun callable ->
Target.Set.mem callable might_change_if_reanalyzed)
in
let () =
if List.length callables_to_reanalyze <> Target.Set.cardinal might_change_if_reanalyzed then
let missing =
Target.Set.diff might_change_if_reanalyzed (Target.Set.of_list callables_to_reanalyze)
in
let check_missing callable =
match get_meta_data callable with
| None -> () (* okay, caller is in a later epoch *)
| Some _ when Target.Set.mem callable filtered_callables ->
(* This is fine - even though this function was called, it was filtered from the
analysis beforehand. *)
Log.warning
"%s was omitted due to being explicitly filtered from the analysis."
(Target.show callable)
| Some { step = { epoch; _ }; _ } when epoch = Epoch.predefined -> ()
| Some meta ->
let message =
Format.sprintf
"Re-analysis in iteration %d determined to analyze %s but it is not part of epoch \
%s (meta: %s)"
step.iteration
(Target.show callable)
(Epoch.show step.epoch)
(meta_data_to_string meta)
in
Log.error "%s" message;
failwith message
in
Target.Set.iter check_missing missing
in
callables_to_reanalyze
let compute_fixpoint
~scheduler
~environment
~analysis
~dependencies
~filtered_callables
~all_callables
epoch
=
(* Start iteration > 0 is to avoid a useless special 0 iteration for mega components. *)
let max_iterations = 100 in
let rec iterate ~iteration callables_to_analyze =
let number_of_callables = List.length callables_to_analyze in
let () =
let witnesses =
if number_of_callables <= 6 then
String.concat ~sep:", " (List.map ~f:Target.show callables_to_analyze)
else
"..."
in
Log.info "Iteration #%d. %d callables [%s]" iteration number_of_callables witnesses
in
if number_of_callables = 0 then (* Fixpoint. *)
iteration
else if iteration >= max_iterations then (
let max_to_show = 15 in
let bucket =
callables_to_analyze |> List.map ~f:Target.show |> List.sort ~compare:String.compare
in
let bucket_len = List.length bucket in
let message =
Format.sprintf
"For a bucket of %d callables: %s%s"
bucket_len
(String.concat ~sep:", " (List.take bucket max_to_show))
(if bucket_len > max_to_show then "..." else "")
in
Log.error "%s" message;
failwith message)
else
let timer = Timer.start () in
let step = FixpointState.{ epoch; iteration } in
let old_batch = FixpointState.KeySet.of_list callables_to_analyze in
let reduce left right =
let callables_processed = left.callables_processed + right.callables_processed in
let () =
Log.log
~section:`Progress
"Processed %d of %d callables"
callables_processed
number_of_callables
in
{
callables_processed;
callables_to_dump = Target.Set.union left.callables_to_dump right.callables_to_dump;
expensive_callables = List.rev_append left.expensive_callables right.expensive_callables;
}
in
let () =
FixpointState.oldify old_batch;
let {
callables_to_dump = iteration_callables_to_dump;
expensive_callables = iteration_expensive_callables;
_;
}
=
Scheduler.map_reduce
scheduler
~policy:(Scheduler.Policy.legacy_fixed_chunk_size 1000)
~map:(fun _ callables -> one_analysis_pass ~analysis ~step ~environment ~callables)
~initial:
{
callables_processed = 0;
callables_to_dump = Target.Set.empty;
expensive_callables = [];
}
~reduce
~inputs:callables_to_analyze
()
in
callables_to_dump := Target.Set.union !callables_to_dump iteration_callables_to_dump;
FixpointState.remove_old old_batch;
if not (List.is_empty iteration_expensive_callables) then
Log.log
~section:`Performance
"Expensive callables for iteration %d: %s"
iteration
(iteration_expensive_callables
|> List.sort ~compare:(fun left right ->
Int.compare right.time_to_analyze_in_ms left.time_to_analyze_in_ms)
|> List.map ~f:(fun { time_to_analyze_in_ms; callable } ->
Format.sprintf "`%s`: %d ms" (Target.show callable) time_to_analyze_in_ms)
|> String.concat ~sep:", ")
in
let callables_to_analyze =
compute_callables_to_reanalyze
step
callables_to_analyze
~dependencies
~filtered_callables
~all_callables
in
Memory.SharedMemory.collect `aggressive;
let () =
Log.info
"Iteration #%n, %d callables, heap size %.3fGB took %.2fs"
iteration
number_of_callables
(Int.to_float (SharedMemory.heap_size ()) /. 1000000000.0)
(Timer.stop_in_sec timer)
in
iterate ~iteration:(iteration + 1) callables_to_analyze
in
try
let iterations = iterate ~iteration:0 all_callables in
let dump_callable callable =
let global_resolution = Analysis.TypeEnvironment.ReadOnly.global_resolution environment in
let resolution =
Analysis.TypeCheck.resolution
global_resolution
(* TODO(T65923817): Eliminate the need of creating a dummy context here *)
(module Analysis.TypeCheck.DummyContext)
in
let resolution = Analysis.Resolution.global_resolution resolution in
let { Define.signature = { name; _ }; _ } =
match callable with
| #Target.callable_t as callable ->
Target.get_module_and_definition callable ~resolution
>>| (fun (_, { Node.value; _ }) -> value)
|> fun value -> Option.value_exn value
| _ -> failwith "No real target to dump"
in
let model =
FixpointState.get_model callable |> Option.value ~default:AnalysisResult.empty_model
in
Log.dump
"Model for `%s` after %d iterations:\n%a"
(Log.Color.yellow (Reference.show name))
iterations
AnalysisResult.pp_model_t
model
in
Target.Set.iter dump_callable !callables_to_dump;
iterations
with
| exn ->
Log.error
"Fixpoint iteration failed.\nException %s\nBacktrace: %s"
(Exn.to_string exn)
(Printexc.get_backtrace ());
raise exn
let report_results
~scheduler
~static_analysis_configuration
~environment
~filename_lookup
~analysis
~callables
~skipped_overrides
~fixpoint_timer
~fixpoint_iterations
=
let report_analysis (AnalysisResult.Analysis { analysis; _ }) =
let module Analysis = (val analysis) in
Analysis.report
~scheduler
~static_analysis_configuration
~environment
~filename_lookup
~callables
~skipped_overrides
~fixpoint_timer
~fixpoint_iterations
in
AnalysisResult.get_abstract_analysis analysis |> report_analysis