source/analysis/attributeResolution.ml (4,724 lines of code) (raw):

(* * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the * LICENSE file in the root directory of this source tree. *) open Core open Pyre open Ast open Statement open Assumptions open ClassSummary module Global = struct type t = { annotation: Annotation.t; undecorated_signature: Type.Callable.t option; problem: AnnotatedAttribute.problem option; } [@@deriving show, compare, sexp] end module UninstantiatedAnnotation = struct type property_annotation = { self: Type.t option; value: Type.t option; } [@@deriving compare] type kind = | Attribute of Type.t | Property of { getter: property_annotation; setter: property_annotation option; } [@@deriving compare] type t = { accessed_via_metaclass: bool; kind: kind; } [@@deriving compare] end type uninstantiated = UninstantiatedAnnotation.t type uninstantiated_attribute = uninstantiated AnnotatedAttribute.t type resolved_define = { undecorated_signature: Type.Callable.t; decorated: (Type.t, AnnotatedAttribute.problem) Result.t; } module Argument = struct type t = { expression: Expression.t option; kind: Ast.Expression.Call.Argument.kind; resolved: Type.t; } module WithPosition = struct type t = { position: int; expression: Expression.t option; kind: Ast.Expression.Call.Argument.kind; resolved: Type.t; } [@@deriving compare, show] end end type matched_argument = | MatchedArgument of { argument: Argument.WithPosition.t; index_into_starred_tuple: int option; } | Default [@@deriving compare, show] let make_matched_argument ?index_into_starred_tuple argument = MatchedArgument { argument; index_into_starred_tuple } type ranks = { arity: int; annotation: int; position: int; } [@@deriving compare, show] type reasons = { arity: SignatureSelectionTypes.reason list; annotation: SignatureSelectionTypes.reason list; } [@@deriving compare, show] type extracted_ordered_type = { ordered_type: Type.OrderedTypes.t; argument: Argument.WithPosition.t; item_type_for_error: Type.t; } let location_insensitive_compare_reasons { arity = left_arity; annotation = left_annotation } { arity = right_arity; annotation = right_annotation } = match List.compare SignatureSelectionTypes.location_insensitive_compare_reason left_arity right_arity with | x when not (Int.equal x 0) -> x | _ -> List.compare SignatureSelectionTypes.location_insensitive_compare_reason left_annotation right_annotation let empty_reasons = { arity = []; annotation = [] } module ParameterArgumentMapping = struct type t = { parameter_argument_mapping: matched_argument list Type.Callable.Parameter.Map.t; reasons: reasons; } [@@deriving compare] let pp format { parameter_argument_mapping; reasons } = Format.fprintf format "ParameterArgumentMapping { parameter_argument_mapping: %s; reasons: %a }" ([%show: (Type.Callable.Parameter.parameter * matched_argument list) list] (Map.to_alist parameter_argument_mapping)) pp_reasons reasons end type signature_match = { callable: Type.Callable.t; parameter_argument_mapping: matched_argument list Type.Callable.Parameter.Map.t; constraints_set: TypeConstraints.t list; ranks: ranks; reasons: reasons; } [@@deriving compare] let pp_signature_match format { callable; parameter_argument_mapping; constraints_set; ranks; reasons } = Format.fprintf format "{ callable = %a; parameter_argument_mapping = %s; constraints_set = %s; ranks = %a; reasons = \ %a }" Type.Callable.pp callable ([%show: (Type.Callable.Parameter.parameter * matched_argument list) list] (Map.to_alist parameter_argument_mapping)) ([%show: TypeConstraints.t list] constraints_set) pp_ranks ranks pp_reasons reasons let show_signature_match = Format.asprintf "%a" pp_signature_match let create_uninstantiated_method ?(accessed_via_metaclass = false) callable = { UninstantiatedAnnotation.accessed_via_metaclass; kind = Attribute (Callable callable) } module UninstantiatedAttributeTable = struct type element = UninstantiatedAnnotation.t AnnotatedAttribute.t [@@deriving compare] type table = (string, element) Caml.Hashtbl.t type t = { attributes: table; names: string list ref; } let create () = { attributes = Caml.Hashtbl.create 15; names = ref [] } let add { attributes; names } attribute = let name = AnnotatedAttribute.name attribute in if Caml.Hashtbl.mem attributes name then () else ( Caml.Hashtbl.add attributes name attribute; names := name :: !names) let mark_as_implicitly_initialized_if_uninitialized { attributes; _ } name = let is_uninitialized attribute = match AnnotatedAttribute.initialized attribute with | NotInitialized -> true | _ -> false in match Caml.Hashtbl.find_opt attributes name with | Some attribute when is_uninitialized attribute -> AnnotatedAttribute.with_initialized ~initialized:OnlyOnInstance attribute |> Caml.Hashtbl.replace attributes name | _ -> () let lookup_name { attributes; _ } = Caml.Hashtbl.find_opt attributes let to_list { attributes; names } = List.rev_map !names ~f:(Caml.Hashtbl.find attributes) let names { names; _ } = !names let compare ({ names = left_names; _ } as left) ({ names = right_names; _ } as right) = let left_names = !left_names in let right_names = !right_names in match List.compare String.compare left_names right_names with | 0 -> let rec compare_elements = function | [] -> 0 | name :: names -> ( match Option.compare compare_element (lookup_name left name) (lookup_name right name) with | 0 -> compare_elements names | nonzero -> nonzero) in compare_elements left_names | nonzero -> nonzero end (* These modules get included at the bottom of this file, they're just here for aesthetic purposes *) module TypeParameterValidationTypes = struct type generic_type_problems = | IncorrectNumberOfParameters of { actual: int; expected: int; can_accept_more_parameters: bool; } | ViolateConstraints of { actual: Type.t; expected: Type.Variable.Unary.t; } | UnexpectedKind of { actual: Type.Parameter.t; expected: Type.Variable.t; } [@@deriving compare, sexp, show, hash] type type_parameters_mismatch = { name: string; kind: generic_type_problems; } [@@deriving compare, sexp, show, hash] end let class_hierarchy_environment class_metadata_environment = ClassMetadataEnvironment.ReadOnly.class_hierarchy_environment class_metadata_environment let alias_environment class_metadata_environment = ClassHierarchyEnvironment.ReadOnly.alias_environment (class_hierarchy_environment class_metadata_environment) let empty_stub_environment class_metadata_environment = alias_environment class_metadata_environment |> AliasEnvironment.ReadOnly.empty_stub_environment let unannotated_global_environment class_metadata_environment = alias_environment class_metadata_environment |> AliasEnvironment.ReadOnly.unannotated_global_environment let class_definition class_metadata_environment annotation ~dependency = Type.split annotation |> fst |> Type.primitive_name >>= UnannotatedGlobalEnvironment.ReadOnly.get_class_definition (unannotated_global_environment class_metadata_environment) ?dependency let aliases class_metadata_environment ~dependency = AliasEnvironment.ReadOnly.get_alias ?dependency (alias_environment class_metadata_environment) let is_suppressed_module class_metadata_environment ~dependency reference = EmptyStubEnvironment.ReadOnly.from_empty_stub (empty_stub_environment class_metadata_environment) ?dependency reference let is_final_class class_metadata_environment ~dependency class_name = match ClassMetadataEnvironment.ReadOnly.get_class_metadata ?dependency class_metadata_environment class_name with | Some { ClassMetadataEnvironment.is_final; _ } -> is_final | _ -> false let class_name { Node.value = { ClassSummary.name; _ }; _ } = name module ClassDecorators = struct type options = { init: bool; repr: bool; eq: bool; order: bool; match_args: bool; field_descriptors: Ast.Expression.t list; } let find_decorator ~class_metadata_environment ~names ?dependency class_summary = UnannotatedGlobalEnvironment.ReadOnly.first_matching_class_decorator (unannotated_global_environment class_metadata_environment) ?dependency ~names class_summary let extract_options ~default ~init ~repr ~eq ~order decorator = let open Expression in let extract_options_from_arguments = let apply_arguments default argument = let recognize_value ~default = function | Expression.Constant Constant.False -> false | Expression.Constant Constant.True -> true | _ -> default in match argument with | { Call.Argument.name = Some { Node.value = argument_name; _ }; value = { Node.value; _ } } -> let argument_name = Identifier.sanitized argument_name in (* We need to check each keyword sequentially because different keywords may correspond to the same string. *) let default = if String.equal argument_name init then { default with init = recognize_value value ~default:default.init } else default in let default = if String.equal argument_name repr then { default with repr = recognize_value value ~default:default.repr } else default in let default = if String.equal argument_name eq then { default with eq = recognize_value value ~default:default.eq } else default in let default = if String.equal argument_name order then { default with order = recognize_value value ~default:default.order } else default in let default = if String.equal argument_name "match_args" then { default with match_args = recognize_value value ~default:default.match_args } else default in let default = if String.equal argument_name "field_descriptors" then match value with | Expression.Tuple field_descriptors -> { default with field_descriptors } | _ -> default else default in default | _ -> default in List.fold ~init:default ~f:apply_arguments in match decorator with | { Decorator.arguments = Some arguments; _ } -> extract_options_from_arguments arguments | _ -> default let dataclass_options ~class_metadata_environment ?dependency class_summary = let field_descriptors = [Reference.create "dataclasses.field" |> Ast.Expression.from_reference ~location:Location.any] in find_decorator ~names:["dataclasses.dataclass"; "dataclass"] ~class_metadata_environment ?dependency class_summary >>| extract_options ~default: { init = true; repr = true; eq = true; order = false; match_args = true; field_descriptors; } ~init:"init" ~repr:"repr" ~eq:"eq" ~order:"order" let attrs_attributes ~class_metadata_environment ?dependency class_summary = find_decorator ~names:["attr.s"; "attr.attrs"] ~class_metadata_environment ?dependency class_summary >>| extract_options ~default: { init = true; repr = true; eq = true; order = true; match_args = false; field_descriptors = []; } ~init:"init" ~repr:"repr" ~eq:"cmp" ~order:"cmp" let is_dataclass_transform decorator = let decorator_reference { Decorator.name = { Node.value; _ }; _ } = value in Decorator.from_expression decorator >>| decorator_reference >>| Reference.last >>| String.equal "__dataclass_transform__" |> Option.value ~default:false let dataclass_transform_default = { init = true; repr = false; eq = true; order = false; match_args = false; field_descriptors = []; } let find_dataclass_transform_decorator_with_default ~class_metadata_environment ?dependency { Node.value = { ClassSummary.decorators; _ }; _ } = let get_dataclass_transform_decorator_with_default decorator = let decorator_reference { Decorator.name = { Node.value; _ }; _ } = value in let lookup_function reference = UnannotatedGlobalEnvironment.ReadOnly.get_define (unannotated_global_environment class_metadata_environment) ?dependency reference >>= fun { FunctionDefinition.body; _ } -> body >>| Node.value in let function_decorators { Define.signature = { Define.Signature.decorators; _ }; _ } = decorators in decorator_reference decorator |> lookup_function >>| function_decorators >>= List.find ~f:is_dataclass_transform >>= Decorator.from_expression >>| extract_options ~default:dataclass_transform_default ~init:"" ~repr:"" ~eq:"eq_default" ~order:"order_default" >>| fun default -> decorator, default in decorators |> List.filter_map ~f:Decorator.from_expression |> List.find_map ~f:get_dataclass_transform_decorator_with_default let dataclass_transform_options ~class_metadata_environment ?dependency class_summary = find_dataclass_transform_decorator_with_default ~class_metadata_environment ?dependency class_summary >>| fun (decorator, default) -> extract_options ~default ~init:"init" ~repr:"repr" ~eq:"eq" ~order:"order" decorator let find_dataclass_transform_class_as_decorator_with_default ~class_metadata_environment ?dependency { Node.value = { ClassSummary.name; bases = { init_subclass_arguments; _ }; _ }; _ } = let get_dataclass_transform_default name = let class_decorators { ClassSummary.decorators; _ } = decorators in name |> UnannotatedGlobalEnvironment.ReadOnly.get_class_definition (ClassMetadataEnvironment.ReadOnly.unannotated_global_environment class_metadata_environment) ?dependency >>| Node.value >>| class_decorators >>= List.find ~f:is_dataclass_transform >>= Decorator.from_expression >>| extract_options ~default:dataclass_transform_default ~init:"" ~repr:"" ~eq:"eq_default" ~order:"order_default" in ClassMetadataEnvironment.ReadOnly.successors class_metadata_environment ?dependency (Reference.show name) |> List.find_map ~f:get_dataclass_transform_default >>| fun default -> ( { Decorator.name = Node.create_with_default_location name; arguments = Some init_subclass_arguments; }, default ) let dataclass_transform_class_options ~class_metadata_environment ?dependency class_summary = find_dataclass_transform_class_as_decorator_with_default ~class_metadata_environment ?dependency class_summary >>| fun (decorator, default) -> extract_options ~default ~init:"init" ~repr:"repr" ~eq:"eq" ~order:"order" decorator let apply ~definition ~class_metadata_environment ~create_attribute ~instantiate_attribute ?dependency table = let open Expression in let { Node.value = { ClassSummary.name; _ }; _ } = definition in let parent_dataclasses = let class_definition = UnannotatedGlobalEnvironment.ReadOnly.get_class_definition (ClassMetadataEnvironment.ReadOnly.unannotated_global_environment class_metadata_environment) ?dependency in ClassMetadataEnvironment.ReadOnly.successors class_metadata_environment ?dependency (Reference.show name) |> List.filter_map ~f:class_definition in let generate_attributes ~options = let already_in_table name = UninstantiatedAttributeTable.lookup_name table name |> Option.is_some in let make_attribute ~annotation ~attribute_name = AnnotatedAttribute.create_uninstantiated ~uninstantiated_annotation: { UninstantiatedAnnotation.accessed_via_metaclass = false; kind = Attribute annotation } ~abstract:false ~async_property:false ~class_variable:false ~defined:true ~initialized:OnClass ~name:attribute_name ~parent:(Reference.show name) ~visibility:ReadWrite ~property:false ~undecorated_signature:None ~problem:None in let make_method ~parameters ~annotation ~attribute_name = let parameters = { Type.Callable.Parameter.name = "$parameter$self"; annotation = Type.Primitive (Reference.show name); default = false; } :: parameters in let callable = { Type.Callable.kind = Named (Reference.combine name (Reference.create attribute_name)); overloads = []; implementation = { annotation; parameters = Defined (Type.Callable.Parameter.create parameters) }; } in AnnotatedAttribute.create_uninstantiated ~uninstantiated_annotation: { UninstantiatedAnnotation.accessed_via_metaclass = false; kind = Attribute (Callable callable); } ~abstract:false ~async_property:false ~class_variable:false ~defined:true ~initialized:OnClass ~name:attribute_name ~parent:(Reference.show name) ~visibility:ReadWrite ~property:false ~undecorated_signature:(Some callable) ~problem:None in match options definition with | None -> [] | Some { init; repr; eq; order; match_args; field_descriptors } -> let init_parameters ~implicitly_initialize = let extract_dataclass_field_arguments (_, value) = match value with | { Node.value = Expression.Call { callee; arguments; _ }; _ } -> Option.some_if (List.exists field_descriptors ~f:(fun field_descriptor -> Int.equal (Ast.Expression.location_insensitive_compare callee field_descriptor) 0)) arguments | _ -> None in let init_not_disabled attribute = let is_disable_init { Call.Argument.name; value = { Node.value; _ } } = match name, value with | Some { Node.value = parameter_name; _ }, Expression.Constant Constant.False when String.equal "init" (Identifier.sanitized parameter_name) -> true | _ -> false in match extract_dataclass_field_arguments attribute with | Some arguments -> not (List.exists arguments ~f:is_disable_init) | _ -> true in let extract_init_value (attribute, value) = let initialized = AnnotatedAttribute.initialized attribute in let get_default_value { Call.Argument.name; value } = name >>| Node.value >>| Identifier.sanitized >>= function | "default" -> Some value | "default_factory" | "factory" -> let { Node.location; _ } = value in Some { Node.value = Expression.Call { Call.callee = value; arguments = [] }; location; } | _ -> None in match initialized with | NotInitialized -> None | _ -> ( match extract_dataclass_field_arguments (attribute, value) with | Some arguments -> List.find_map arguments ~f:get_default_value | _ -> Some value) in let collect_parameters parameters (attribute, value) = (* Parameters must be annotated attributes *) let annotation = instantiate_attribute attribute |> AnnotatedAttribute.annotation |> Annotation.original |> function | Type.Parametric { name = "dataclasses.InitVar"; parameters = [Single single_parameter] } -> single_parameter | annotation -> annotation in match AnnotatedAttribute.name attribute with | name when not (Type.contains_unknown annotation) -> if implicitly_initialize then UninstantiatedAttributeTable.mark_as_implicitly_initialized_if_uninitialized table name; let name = "$parameter$" ^ name in let value = extract_init_value (attribute, value) in let rec override_existing_parameters unchecked_parameters = match unchecked_parameters with | [] -> [ { Type.Callable.Parameter.name; annotation; default = Option.is_some value; }; ] | { Type.Callable.Parameter.name = old_name; default = old_default; _ } :: tail when Identifier.equal old_name name -> { name; annotation; default = Option.is_some value || old_default } :: tail | head :: tail -> head :: override_existing_parameters tail in override_existing_parameters parameters | _ -> parameters in let get_table ({ Node.value = class_summary; _ } as parent) = let create attribute : uninstantiated_attribute * Expression.t = let value = match attribute with | { Node.value = { Attribute.kind = Simple { values = { value; _ } :: _; _ }; _ }; _; } -> value | { Node.location; _ } -> Node.create (Expression.Constant Constant.Ellipsis) ~location in ( create_attribute ~parent ?defined:None ~accessed_via_metaclass:false (Node.value attribute), value ) in let compare_by_location left right = Ast.Location.compare (Node.location left) (Node.location right) in ClassSummary.attributes ~include_generated_attributes:false ~in_test:false class_summary |> Identifier.SerializableMap.bindings |> List.unzip |> snd |> List.filter ~f:(fun attribute -> (* ClassVar should be excluded from considerations as field. *) match Node.value attribute with | { Attribute.kind = Attribute.Simple { annotation = Some { Node.value = Expression.Call { callee = { value = Name (Name.Attribute { attribute = "__getitem__"; base = { Node.value = Name (Name.Attribute { attribute = "ClassVar"; base = { Node.value = Name (Name.Identifier "typing"); _; }; _; }); _; }; _; }); _; }; arguments = [_]; }; _; }; _; }; _; } -> false | _ -> true) |> List.sort ~compare:compare_by_location |> List.map ~f:create in let parent_attribute_tables = parent_dataclasses |> List.filter ~f:(fun definition -> options definition |> Option.is_some) |> List.rev |> List.map ~f:get_table in parent_attribute_tables @ [get_table definition] |> List.map ~f:(List.filter ~f:init_not_disabled) |> List.fold ~init:[] ~f:(fun parameters -> List.fold ~init:parameters ~f:collect_parameters) in let methods = if init && not (already_in_table "__init__") then [ make_method ~parameters:(init_parameters ~implicitly_initialize:true) ~annotation:Type.none ~attribute_name:"__init__"; ] else [] in let methods = if repr && not (already_in_table "__repr__") then let new_method = make_method ~parameters:[] ~annotation:Type.string ~attribute_name:"__repr__" in new_method :: methods else methods in let add_order_method methods name = let annotation = Type.object_primitive in if not (already_in_table name) then make_method ~parameters:[{ name = "$parameter$o"; annotation; default = false }] ~annotation:Type.bool ~attribute_name:name :: methods else methods in let methods = if eq then add_order_method methods "__eq__" else methods in let methods = if order then ["__lt__"; "__le__"; "__gt__"; "__ge__"] |> List.fold ~init:methods ~f:add_order_method else methods in let methods = if match_args && not (already_in_table "__match_args__") then let parameter_name { Callable.RecordParameter.name; _ } = Identifier.sanitized name in let init_parameter_names = List.map ~f:parameter_name (init_parameters ~implicitly_initialize:false) in let literal_string_value_type name = Type.Literal (String (LiteralValue name)) in let annotation = Type.tuple (List.map ~f:literal_string_value_type init_parameter_names) in make_attribute ~annotation ~attribute_name:"__match_args__" :: methods else methods in methods in let dataclass_attributes () = (* TODO (T43210531): Warn about inconsistent annotations *) generate_attributes ~options:(dataclass_options ~class_metadata_environment ?dependency) in let attrs_attributes () = (* TODO (T41039225): Add support for other methods *) generate_attributes ~options:(attrs_attributes ~class_metadata_environment ?dependency) in let dataclass_transform_attributes () = generate_attributes ~options:(dataclass_transform_options ~class_metadata_environment ?dependency) in let dataclass_transform_class_attributes () = generate_attributes ~options:(dataclass_transform_class_options ~class_metadata_environment ?dependency) in dataclass_attributes () @ attrs_attributes () @ dataclass_transform_attributes () @ dataclass_transform_class_attributes () |> List.iter ~f:(UninstantiatedAttributeTable.add table) end let partial_apply_self { Type.Callable.implementation; overloads; _ } ~order ~self_type = let open Type.Callable in let implementation, overloads = match implementation, overloads with | { Type.Callable.parameters = Defined (Named { annotation; _ } :: _); _ }, _ -> ( let solution = try TypeOrder.OrderedConstraintsSet.add ConstraintsSet.empty ~new_constraint:(LessOrEqual { left = self_type; right = annotation }) ~order |> TypeOrder.OrderedConstraintsSet.solve ~order |> Option.value ~default:ConstraintsSet.Solution.empty with | ClassHierarchy.Untracked _ -> ConstraintsSet.Solution.empty in let instantiated = ConstraintsSet.Solution.instantiate solution (Type.Callable { kind = Anonymous; implementation; overloads }) in match instantiated with | Type.Callable { implementation; overloads; _ } -> implementation, overloads | _ -> implementation, overloads) | _ -> implementation, overloads in let drop_self { Type.Callable.annotation; parameters } = let parameters = match parameters with | Type.Callable.Defined (_ :: parameters) -> Type.Callable.Defined parameters | ParameterVariadicTypeVariable { head = _ :: head; variable } -> ParameterVariadicTypeVariable { head; variable } | _ -> parameters in { Type.Callable.annotation; parameters } in { Type.Callable.kind = Anonymous; implementation = drop_self implementation; overloads = List.map overloads ~f:drop_self; } let callable_call_special_cases ~instantiated ~class_name ~attribute_name ~order ~accessed_through_class = match instantiated, class_name, attribute_name, accessed_through_class with | Some (Type.Callable _), "typing.Callable", "__call__", false -> instantiated | ( Some (Parametric { name = "BoundMethod"; parameters = [Single (Callable callable); Single self_type] }), "typing.Callable", "__call__", false ) -> let order = order () in partial_apply_self callable ~order ~self_type |> fun callable -> Type.Callable callable |> Option.some | _ -> None module SignatureSelection = struct (** Return a mapping from each parameter to the arguments that may be assigned to it. Also include any error reasons when there are too many or too few arguments. Parameters such as `*args: int` and `**kwargs: str` may have any number of arguments assigned to them. Other parameters such as named parameters (`x: int`), positional-only, or keyword-only parameters will have zero or one argument mapped to them. *) let get_parameter_argument_mapping ~all_parameters ~parameters ~self_argument arguments = let open Type.Callable in let all_arguments = arguments in let rec consume ({ ParameterArgumentMapping.parameter_argument_mapping; reasons = { arity; _ } as reasons } as parameter_argument_mapping_with_reasons) ~arguments ~parameters = let update_mapping parameter argument = Map.add_multi parameter_argument_mapping ~key:parameter ~data:argument in let arity_mismatch ?(unreachable_parameters = []) ~arguments reasons = match all_parameters with | Defined all_parameters -> let matched_keyword_arguments = let is_keyword_argument = function | { Argument.WithPosition.kind = Named _; _ } -> true | _ -> false in List.filter ~f:is_keyword_argument all_arguments in let positional_parameter_count = List.length all_parameters - List.length unreachable_parameters - List.length matched_keyword_arguments in let self_argument_adjustment = if Option.is_some self_argument then 1 else 0 in let error = SignatureSelectionTypes.TooManyArguments { expected = positional_parameter_count - self_argument_adjustment; provided = positional_parameter_count + List.length arguments - self_argument_adjustment; } in { reasons with arity = error :: arity } | _ -> reasons in match arguments, parameters with | [], [] -> (* Both empty *) parameter_argument_mapping_with_reasons | { Argument.WithPosition.kind = SingleStar; _ } :: arguments_tail, [] | { kind = DoubleStar; _ } :: arguments_tail, [] -> (* Starred or double starred arguments; parameters empty *) consume ~arguments:arguments_tail ~parameters parameter_argument_mapping_with_reasons | { kind = Named name; _ } :: _, [] -> (* Named argument; parameters empty *) let reasons = { reasons with arity = UnexpectedKeyword name.value :: arity } in { parameter_argument_mapping_with_reasons with reasons } | _, [] -> (* Positional argument; parameters empty *) { parameter_argument_mapping_with_reasons with reasons = arity_mismatch ~arguments reasons; } | [], (Parameter.KeywordOnly { default = true; _ } as parameter) :: parameters_tail | [], (Parameter.PositionalOnly { default = true; _ } as parameter) :: parameters_tail | [], (Parameter.Named { default = true; _ } as parameter) :: parameters_tail -> (* Arguments empty, default parameter *) let parameter_argument_mapping = update_mapping parameter Default in consume ~arguments ~parameters:parameters_tail { parameter_argument_mapping_with_reasons with parameter_argument_mapping } | [], parameter :: parameters_tail -> (* Arguments empty, parameter *) let parameter_argument_mapping = match Map.find parameter_argument_mapping parameter with | Some _ -> parameter_argument_mapping | None -> Map.set ~key:parameter ~data:[] parameter_argument_mapping in consume ~arguments ~parameters:parameters_tail { parameter_argument_mapping_with_reasons with parameter_argument_mapping } | ( ({ kind = Named _; _ } as argument) :: arguments_tail, (Parameter.Keywords _ as parameter) :: _ ) -> (* Labeled argument, keywords parameter *) let parameter_argument_mapping = update_mapping parameter (make_matched_argument argument) in consume ~arguments:arguments_tail ~parameters { parameter_argument_mapping_with_reasons with parameter_argument_mapping } | ({ kind = Named name; _ } as argument) :: arguments_tail, parameters -> (* Labeled argument *) let rec extract_matching_name searched to_search = match to_search with | [] -> None, List.rev searched | (Parameter.KeywordOnly { name = parameter_name; _ } as head) :: tail | (Parameter.Named { name = parameter_name; _ } as head) :: tail when Identifier.equal_sanitized parameter_name name.value -> Some head, List.rev searched @ tail | (Parameter.Keywords _ as head) :: tail -> let matching, parameters = extract_matching_name (head :: searched) tail in let matching = Some (Option.value matching ~default:head) in matching, parameters | head :: tail -> extract_matching_name (head :: searched) tail in let matching_parameter, remaining_parameters = extract_matching_name [] parameters in let parameter_argument_mapping, reasons = match matching_parameter with | Some matching_parameter -> update_mapping matching_parameter (make_matched_argument argument), reasons | None -> ( parameter_argument_mapping, { reasons with arity = UnexpectedKeyword name.value :: arity } ) in consume ~arguments:arguments_tail ~parameters:remaining_parameters { parameter_argument_mapping_with_reasons with parameter_argument_mapping; reasons } | ( ({ kind = DoubleStar; _ } as argument) :: arguments_tail, (Parameter.Keywords _ as parameter) :: _ ) | ( ({ kind = SingleStar; _ } as argument) :: arguments_tail, (Parameter.Variable _ as parameter) :: _ ) -> (* (Double) starred argument, (double) starred parameter *) let parameter_argument_mapping = update_mapping parameter (make_matched_argument argument) in consume ~arguments:arguments_tail ~parameters { parameter_argument_mapping_with_reasons with parameter_argument_mapping } | { kind = SingleStar; _ } :: _, Parameter.Keywords _ :: parameters_tail -> (* Starred argument, double starred parameter *) consume ~arguments ~parameters:parameters_tail { parameter_argument_mapping_with_reasons with parameter_argument_mapping } | { kind = Positional; _ } :: _, Parameter.Keywords _ :: parameters_tail -> (* Unlabeled argument, double starred parameter *) consume ~arguments ~parameters:parameters_tail { parameter_argument_mapping_with_reasons with parameter_argument_mapping } | { kind = DoubleStar; _ } :: _, Parameter.Variable _ :: parameters_tail -> (* Double starred argument, starred parameter *) consume ~arguments ~parameters:parameters_tail { parameter_argument_mapping_with_reasons with parameter_argument_mapping } | ( ({ kind = Positional; _ } as argument) :: arguments_tail, (Parameter.Variable _ as parameter) :: _ ) -> (* Unlabeled argument, starred parameter *) let parameter_argument_mapping_with_reasons = let parameter_argument_mapping = update_mapping parameter (make_matched_argument argument) in { parameter_argument_mapping_with_reasons with parameter_argument_mapping } in consume ~arguments:arguments_tail ~parameters parameter_argument_mapping_with_reasons | { kind = SingleStar; _ } :: arguments_tail, Type.Callable.Parameter.KeywordOnly _ :: _ -> (* Starred argument, keyword only parameter *) consume ~arguments:arguments_tail ~parameters parameter_argument_mapping_with_reasons | ({ kind = DoubleStar; _ } as argument) :: _, parameter :: parameters_tail | ({ kind = SingleStar; _ } as argument) :: _, parameter :: parameters_tail -> (* Double starred or starred argument, parameter *) let parameter_argument_mapping = update_mapping parameter (make_matched_argument argument) in consume ~arguments ~parameters:parameters_tail { parameter_argument_mapping_with_reasons with parameter_argument_mapping } | { kind = Positional; _ } :: _, (Parameter.KeywordOnly _ as parameter) :: parameters_tail -> (* Unlabeled argument, keyword only parameter *) let reasons = arity_mismatch reasons ~unreachable_parameters:(parameter :: parameters_tail) ~arguments in { parameter_argument_mapping_with_reasons with reasons } | ({ kind = Positional; _ } as argument) :: arguments_tail, parameter :: parameters_tail -> (* Unlabeled argument, parameter *) let parameter_argument_mapping = update_mapping parameter (make_matched_argument argument) in consume ~arguments:arguments_tail ~parameters:parameters_tail { parameter_argument_mapping_with_reasons with parameter_argument_mapping } in { ParameterArgumentMapping.parameter_argument_mapping = Parameter.Map.empty; reasons = empty_reasons; } |> consume ~arguments ~parameters (** Check all arguments against the respective parameter types. Return a signature match containing constraints from the above compatibility checks and any mismatch errors. *) let check_arguments_against_parameters ~order ~resolve_mutable_literals ~resolve_with_locals ~callable { ParameterArgumentMapping.parameter_argument_mapping; reasons } = let open SignatureSelectionTypes in let open Type.Callable in (* Check whether the parameter annotation is `Callable[[ParamVar], ReturnVar]` * and the argument is `lambda parameter: body` *) let is_generic_lambda parameter arguments = match parameter, arguments with | ( Parameter.PositionalOnly { annotation = Type.Callable { kind = Anonymous; implementation = { annotation = Type.Variable return_variable; parameters = Defined [ Parameter.PositionalOnly { index = 0; annotation = Type.Variable parameter_variable; default = false; }; ]; }; overloads = []; } as annotation; _; }, [ MatchedArgument { argument = { expression = Some { value = Lambda { body = lambda_body; parameters = [ { value = { name = lambda_parameter; value = None; annotation = None }; _; }; ]; }; _; }; _; }; _; }; ] ) when Type.Variable.Unary.is_free parameter_variable && Type.Variable.Unary.is_free return_variable -> Some (annotation, parameter_variable, return_variable, lambda_parameter, lambda_body) | _ -> None in let check_arguments_and_update_signature_match ~parameter ~arguments ({ reasons = { arity; _ } as reasons; _ } as signature_match) = let check_argument_and_set_constraints_and_reasons ~position ~argument_location ~name ~argument_annotation ~parameter_annotation ({ constraints_set; reasons = { annotation; _ } as reasons; _ } as signature_match) = let reasons_with_mismatch = let mismatch = let location = name >>| Node.location |> Option.value ~default:argument_location in { actual = argument_annotation; expected = parameter_annotation; name = Option.map name ~f:Node.value; position; } |> Node.create ~location |> fun mismatch -> Mismatches [Mismatch mismatch] in { reasons with annotation = mismatch :: annotation } in let updated_constraints_set = TypeOrder.OrderedConstraintsSet.add constraints_set ~new_constraint: (LessOrEqual { left = argument_annotation; right = parameter_annotation }) ~order in if ConstraintsSet.potentially_satisfiable updated_constraints_set then { signature_match with constraints_set = updated_constraints_set } else { signature_match with constraints_set; reasons = reasons_with_mismatch } in let extract_iterable_item_type ~synthetic_variable ~generic_iterable_type resolved = let iterable_constraints = if Type.is_unbound resolved then ConstraintsSet.impossible else TypeOrder.OrderedConstraintsSet.add ConstraintsSet.empty ~new_constraint:(LessOrEqual { left = resolved; right = generic_iterable_type }) ~order in TypeOrder.OrderedConstraintsSet.solve iterable_constraints ~order >>| fun solution -> ConstraintsSet.Solution.instantiate_single_variable solution synthetic_variable |> Option.value ~default:Type.Any in let bind_arguments_to_variadic ~expected ~arguments = let extract_ordered_types arguments = let extracted, errors = let extract ( ({ Argument.WithPosition.kind; resolved; expression; _ } as argument), index_into_starred_tuple ) = match kind with | SingleStar -> ( match resolved, index_into_starred_tuple with | Type.Tuple ordered_type, Some index_into_starred_tuple -> Type.OrderedTypes.drop_prefix ~length:index_into_starred_tuple ordered_type >>| (fun ordered_type -> Either.First { ordered_type; argument; item_type_for_error = Type.OrderedTypes.union_upper_bound ordered_type; }) |> Option.value ~default:(Either.Second { expression; annotation = resolved }) | Type.Tuple ordered_type, None -> Either.First { ordered_type; argument; item_type_for_error = Type.OrderedTypes.union_upper_bound ordered_type; } | _, _ -> ( let synthetic_variable = Type.Variable.Unary.create "$_T" in let generic_iterable_type = Type.iterable (Type.Variable synthetic_variable) in match extract_iterable_item_type ~synthetic_variable ~generic_iterable_type resolved with | Some item_type -> Either.First { ordered_type = Type.OrderedTypes.create_unbounded_concatenation item_type; argument; item_type_for_error = item_type; } | _ -> Either.Second { expression; annotation = resolved })) | _ -> Either.First { ordered_type = Type.OrderedTypes.Concrete [resolved]; argument; item_type_for_error = resolved; } in List.rev arguments |> List.partition_map ~f:extract in match errors with | [] -> Ok extracted | not_bounded_tuple :: _ -> Error (Mismatches [ MismatchWithUnpackableType { variable = expected; mismatch = NotUnpackableType not_bounded_tuple }; ]) in let concatenate extracted = let ordered_types = List.map extracted ~f:(fun { ordered_type; _ } -> ordered_type) in match Type.OrderedTypes.coalesce_ordered_types ordered_types with | Some concatenated -> Ok (concatenated, extracted) | None -> Error (Mismatches [ MismatchWithUnpackableType { variable = expected; mismatch = CannotConcatenate ordered_types }; ]) in let solve (concatenated, extracted_ordered_types) = let updated_constraints_set = TypeOrder.OrderedConstraintsSet.add signature_match.constraints_set ~new_constraint:(OrderedTypesLessOrEqual { left = concatenated; right = expected }) ~order in if ConstraintsSet.potentially_satisfiable updated_constraints_set then Ok updated_constraints_set else let expected_concatenation_type = match expected with | Concatenation concatenation -> Some concatenation | _ -> None in match expected_concatenation_type >>= Type.OrderedTypes.Concatenation.extract_sole_unbounded_annotation with | Some expected_item_type -> (* The expected type is `*args: *Tuple[X, ...]`. Raise an individual error for each argument that was passed. *) let make_mismatch { argument = { Argument.WithPosition.position; expression; kind; _ }; item_type_for_error; _; } = let name = match kind with | Named name -> Some name | _ -> None in let location = Option.first_some (name >>| Node.location) (expression >>| Node.location) |> Option.value ~default:Location.any in let is_mismatch = TypeOrder.OrderedConstraintsSet.add signature_match.constraints_set ~new_constraint: (LessOrEqual { left = item_type_for_error; right = expected_item_type }) ~order |> ConstraintsSet.potentially_satisfiable |> not in { actual = item_type_for_error; expected = expected_item_type; name = name >>| Node.value; position; } |> Node.create ~location |> fun mismatch -> Mismatch mismatch |> Option.some_if is_mismatch in Error (Mismatches (List.filter_map extracted_ordered_types ~f:make_mismatch)) | None -> (* The expected type is different from `*args: *Tuple[X, ...]`, such as `*Ts` or more complicated unbounded tuples. It may require a prefix or suffix of arguments. Since we cannot express that clearly by raising individual errors, we raise a combined error about the arguments. *) Error (Mismatches [ MismatchWithUnpackableType { variable = expected; mismatch = ConstraintFailure concatenated }; ]) in let make_signature_match = function | Ok constraints_set -> { signature_match with constraints_set } | Error error -> { signature_match with reasons = { reasons with arity = error :: arity } } in let arguments = List.map arguments ~f:(function | MatchedArgument { argument; index_into_starred_tuple } -> argument, index_into_starred_tuple | Default -> failwith "Variable parameters do not have defaults") in let open Result in extract_ordered_types arguments >>= concatenate >>= solve |> make_signature_match in match parameter, arguments with | Parameter.Variable (Concatenation concatenation), arguments -> bind_arguments_to_variadic ~expected:(Type.OrderedTypes.Concatenation concatenation) ~arguments | Parameter.Variable (Concrete parameter_annotation), arguments -> bind_arguments_to_variadic ~expected:(Type.OrderedTypes.create_unbounded_concatenation parameter_annotation) ~arguments | Parameter.Keywords _, [] -> (* Parameter was not matched, but empty is acceptable for variable arguments and keyword arguments. *) signature_match | Parameter.KeywordOnly { name; _ }, [] | Parameter.Named { name; _ }, [] -> (* Parameter was not matched *) let reasons = { reasons with arity = MissingArgument (Named name) :: arity } in { signature_match with reasons } | Parameter.PositionalOnly { index; _ }, [] -> (* Parameter was not matched *) let reasons = { reasons with arity = MissingArgument (PositionalOnly index) :: arity } in { signature_match with reasons } | PositionalOnly { annotation = parameter_annotation; _ }, arguments | KeywordOnly { annotation = parameter_annotation; _ }, arguments | Named { annotation = parameter_annotation; _ }, arguments | Keywords parameter_annotation, arguments -> ( let rec check ~arguments signature_match = match arguments with | [] -> signature_match | Default :: tail -> (* Parameter default value was used. Assume it is correct. *) check signature_match ~arguments:tail | MatchedArgument { argument = { expression; position; kind; resolved }; index_into_starred_tuple } :: tail -> ( let argument_location = expression >>| Node.location |> Option.value ~default:Location.any in let name = match kind with | Named name -> Some name | _ -> None in let check_argument argument_annotation = check_argument_and_set_constraints_and_reasons ~position ~argument_location ~argument_annotation ~parameter_annotation ~name signature_match in let add_annotation_error ({ reasons = { annotation; _ }; _ } as signature_match) error = { signature_match with reasons = { reasons with annotation = error :: annotation }; } in let update_signature_match_for_iterable ~create_error ~resolved iterable_item_type = let argument_location = expression >>| Node.location |> Option.value ~default:Location.any in match iterable_item_type with | Some iterable_item_type -> check_argument_and_set_constraints_and_reasons ~position ~argument_location ~argument_annotation:iterable_item_type ~parameter_annotation ~name signature_match |> check ~arguments:tail | None -> let argument_location = expression >>| Node.location |> Option.value ~default:Location.any in { expression; annotation = resolved } |> Node.create ~location:argument_location |> create_error |> add_annotation_error signature_match in match kind with | DoubleStar -> let create_error error = InvalidKeywordArgument error in let synthetic_variable = Type.Variable.Unary.create "$_T" in let generic_iterable_type = Type.parametric "typing.Mapping" [Single Type.string; Single (Type.Variable synthetic_variable)] in extract_iterable_item_type ~synthetic_variable ~generic_iterable_type resolved |> update_signature_match_for_iterable ~create_error ~resolved | SingleStar -> ( let signature_match_for_single_element = match parameter, index_into_starred_tuple, resolved with | ( (PositionalOnly _ | Named _), Some index_into_starred_tuple, Type.Tuple ordered_type ) -> Type.OrderedTypes.index ~python_index:index_into_starred_tuple ordered_type >>| check_argument >>| check ~arguments:tail | _ -> None in match signature_match_for_single_element with | Some signature_match_for_single_element -> signature_match_for_single_element | None -> let create_error error = InvalidVariableArgument error in let synthetic_variable = Type.Variable.Unary.create "$_T" in let generic_iterable_type = Type.iterable (Type.Variable synthetic_variable) in extract_iterable_item_type ~synthetic_variable ~generic_iterable_type resolved |> update_signature_match_for_iterable ~create_error ~resolved) | Named _ | Positional -> ( let argument_annotation, weakening_error = if Type.Variable.all_variables_are_resolved parameter_annotation then let { WeakenMutableLiterals.resolved; typed_dictionary_errors } = resolve_mutable_literals ~resolve:(resolve_with_locals ~locals:[]) ~expression ~resolved ~expected:parameter_annotation in let weakening_error = if List.is_empty typed_dictionary_errors then None else Some (TypedDictionaryInitializationError typed_dictionary_errors) in resolved, weakening_error else resolved, None in match weakening_error with | Some weakening_error -> add_annotation_error signature_match weakening_error | None -> argument_annotation |> check_argument |> check ~arguments:tail)) in match is_generic_lambda parameter arguments with | Some _ -> signature_match (* Handle this later in `special_case_lambda_parameter` *) | None -> check ~arguments:(List.rev arguments) signature_match) in let check_if_solution_exists ({ constraints_set; reasons = { annotation; _ } as reasons; callable; _ } as signature_match) = let solution = TypeOrder.OrderedConstraintsSet.solve constraints_set ~order ~only_solve_for:(Type.Variable.all_free_variables (Type.Callable callable)) in if Option.is_some solution then signature_match else (* All other cases should have been able to been blamed on a specefic argument, this is the only global failure. *) { signature_match with reasons = { reasons with annotation = MutuallyRecursiveTypeVariables :: annotation }; } in let special_case_dictionary_constructor ({ parameter_argument_mapping; callable; constraints_set; _ } as signature_match) = let open Type.Record.Callable in let has_matched_keyword_parameter parameters = List.find parameters ~f:(function | RecordParameter.Keywords _ -> true | _ -> false) >>= Type.Callable.Parameter.Map.find parameter_argument_mapping >>| List.is_empty >>| not |> Option.value ~default:false in match callable with | { kind = Named name; implementation = { parameters = Defined parameters; annotation = Type.Parametric { parameters = [Single key_type; _]; _ }; _; }; _; } when String.equal (Reference.show name) "dict.__init__" && has_matched_keyword_parameter parameters -> let updated_constraints = TypeOrder.OrderedConstraintsSet.add constraints_set ~new_constraint:(LessOrEqual { left = Type.string; right = key_type }) ~order in if ConstraintsSet.potentially_satisfiable updated_constraints then { signature_match with constraints_set = updated_constraints } else (* TODO(T41074174): Error here *) signature_match | _ -> signature_match in let special_case_lambda_parameter ({ parameter_argument_mapping; _ } as signature_match) = (* Special case: `Callable[[ParamVar], ReturnVar]` with `lambda parameter: body` *) let check_lambda_argument_and_update_signature_match ~parameter ~arguments ({ constraints_set; _ } as signature_match) = match is_generic_lambda parameter arguments with | None -> signature_match | Some (annotation, parameter_variable, _, lambda_parameter, lambda_body) -> ( (* Infer the parameter type using existing constraints. *) let solution = TypeOrder.OrderedConstraintsSet.solve constraints_set ~order ~only_solve_for:[Type.Record.Variable.Unary parameter_variable] >>= fun solution -> ConstraintsSet.Solution.instantiate_single_variable solution parameter_variable in match solution with | None -> signature_match | Some parameter_type -> (* Infer the return type by resolving the lambda body with the parameter type *) let updated_constraints = let resolved = let return_type = resolve_with_locals ~locals: [ ( Reference.create lambda_parameter, Annotation.create_mutable parameter_type ); ] lambda_body |> Type.weaken_literals in let parameters = Type.Callable.Parameter.create [ { Type.Callable.Parameter.name = lambda_parameter; annotation = parameter_type; default = false; }; ] in Type.Callable.create ~parameters:(Defined parameters) ~annotation:return_type () in TypeOrder.OrderedConstraintsSet.add constraints_set ~new_constraint:(LessOrEqual { left = resolved; right = annotation }) ~order (* Once we've used this solution, we have to commit to it *) |> TypeOrder.OrderedConstraintsSet.add ~new_constraint: (VariableIsExactly (UnaryPair (parameter_variable, parameter_type))) ~order in { signature_match with constraints_set = updated_constraints }) in Map.fold ~init:signature_match ~f:(fun ~key ~data -> check_lambda_argument_and_update_signature_match ~parameter:key ~arguments:data) parameter_argument_mapping in let signature_match = { callable; parameter_argument_mapping; constraints_set = [TypeConstraints.empty]; ranks = { arity = 0; annotation = 0; position = 0 }; reasons; } in Map.fold ~init:signature_match ~f:(fun ~key ~data -> check_arguments_and_update_signature_match ~parameter:key ~arguments:data) parameter_argument_mapping |> special_case_dictionary_constructor |> special_case_lambda_parameter |> check_if_solution_exists (** Check arguments against the given callable signature and returning possible signature matches. *) let rec check_arguments_against_signature ~order ~resolve_mutable_literals ~resolve_with_locals ~callable ~self_argument ~(arguments : Argument.WithPosition.t list) implementation = let open SignatureSelectionTypes in let open Type.Callable in let callable = { callable with Type.Callable.implementation; overloads = [] } in let base_signature_match = { callable; parameter_argument_mapping = Parameter.Map.empty; constraints_set = [TypeConstraints.empty]; ranks = { arity = 0; annotation = 0; position = 0 }; reasons = empty_reasons; } in let { parameters = all_parameters; _ } = implementation in let check_arguments_against_parameters = check_arguments_against_parameters ~order ~resolve_mutable_literals ~resolve_with_locals in match all_parameters with | Defined parameters -> get_parameter_argument_mapping ~parameters ~all_parameters ~self_argument arguments |> check_arguments_against_parameters ~callable |> fun signature_match -> [signature_match] | Undefined -> [base_signature_match] | ParameterVariadicTypeVariable { head; variable } when Type.Variable.Variadic.Parameters.is_free variable -> ( (* Handle callables where an early parameter binds a ParamSpec and later parameters expect the corresponding arguments. For example, when a function like `def foo(f: Callable[P, R], *args: P.args, **kwargs: P.kwargs) -> None` is called as `foo(add, 1, 2)`, first solve for the free variable `P` using the callable argument `add` and then use the solution to get concrete types for `P.args` and `P.kwargs`. *) let front, back = let is_labeled = function | { Argument.WithPosition.kind = Named _; _ } -> true | _ -> false in let labeled, unlabeled = List.partition_tf arguments ~f:is_labeled in let first_unlabeled, remainder = List.split_n unlabeled (List.length head) in first_unlabeled, labeled @ remainder in let ({ constraints_set; reasons = { arity = head_arity; annotation = head_annotation }; _ } as head_signature) = get_parameter_argument_mapping ~all_parameters ~parameters:(Type.Callable.prepend_anonymous_parameters ~head ~tail:[]) ~self_argument front |> check_arguments_against_parameters ~callable in let solve_back parameters = let constraints_set = (* If we use this option, we have to commit to it as to not move away from it later *) TypeOrder.OrderedConstraintsSet.add constraints_set ~new_constraint:(VariableIsExactly (ParameterVariadicPair (variable, parameters))) ~order in check_arguments_against_signature ~order ~resolve_mutable_literals ~resolve_with_locals ~callable ~self_argument ~arguments:back { implementation with parameters } |> List.map ~f:(fun { reasons = { arity = tail_arity; annotation = tail_annotation }; _ } -> { base_signature_match with constraints_set; reasons = { arity = head_arity @ tail_arity; annotation = head_annotation @ tail_annotation; }; }) in TypeOrder.OrderedConstraintsSet.get_parameter_specification_possibilities constraints_set ~parameter_specification:variable ~order |> List.concat_map ~f:solve_back |> function | [] -> [head_signature] | nonempty -> nonempty) | ParameterVariadicTypeVariable { head; variable } -> ( (* The ParamSpec variable `P` is in scope, so the only valid arguments are `*args` and `**kwargs` that have "type" `P.args` and `P.kwargs` respectively. If the ParamSpec has a `head` prefix of parameters, check for any prefix arguments. *) let combines_into_variable ~positional_component ~keyword_component = Type.Variable.Variadic.Parameters.Components.combine { positional_component; keyword_component } >>| Type.Variable.Variadic.Parameters.equal variable |> Option.value ~default:false in match List.rev arguments with | { kind = DoubleStar; resolved = keyword_component; _ } :: { kind = SingleStar; resolved = positional_component; _ } :: reversed_arguments_head when combines_into_variable ~positional_component ~keyword_component -> let arguments = List.rev reversed_arguments_head in get_parameter_argument_mapping ~parameters:(Type.Callable.prepend_anonymous_parameters ~head ~tail:[]) ~all_parameters ~self_argument arguments |> check_arguments_against_parameters ~callable |> fun signature_match -> [signature_match] | _ -> [ { base_signature_match with reasons = { arity = [CallingParameterVariadicTypeVariable]; annotation = [] }; }; ]) (** Given a signature match for a callable, solve for any type variables and instantiate the return annotation. *) let instantiate_return_annotation ?(skip_marking_escapees = false) ~order { callable = { implementation = { annotation = uninstantiated_return_annotation; _ }; _ } as callable; constraints_set; reasons = { arity; annotation; _ }; _; } = let open SignatureSelectionTypes in let instantiated_return_annotation = let local_free_variables = Type.Variable.all_free_variables (Type.Callable callable) in let solution = TypeOrder.OrderedConstraintsSet.solve constraints_set ~only_solve_for:local_free_variables ~order |> Option.value ~default:ConstraintsSet.Solution.empty in let instantiated = ConstraintsSet.Solution.instantiate solution uninstantiated_return_annotation in if skip_marking_escapees then instantiated else Type.Variable.mark_all_free_variables_as_escaped ~specific:local_free_variables instantiated (* We need to do transformations of the form Union[T_escaped, int] => int in order to properly handle some typeshed stubs which only sometimes bind type variables and expect them to fall out in this way (see Mapping.get) *) |> Type.Variable.collapse_all_escaped_variable_unions in let rev_filter_out_self_argument_errors reasons = let filter_too_many_arguments = function (* These would come from methods lacking a self argument called on an instance *) | TooManyArguments { expected; _ } -> not (Int.equal expected (-1)) | _ -> true in let filter_mismatches reason = match reason with | Mismatches mismatches -> Mismatches (List.filter mismatches ~f:(function | Mismatch { Node.value = { position; _ }; _ } -> not (Int.equal position 0) | _ -> true)) | _ -> reason in List.map (List.rev_filter ~f:filter_too_many_arguments reasons) ~f:filter_mismatches in match rev_filter_out_self_argument_errors arity, rev_filter_out_self_argument_errors annotation with | [], [] -> Found { selected_return_annotation = instantiated_return_annotation } | reason :: reasons, _ | [], reason :: reasons -> let importance = function | AbstractClassInstantiation _ -> 1 | CallingParameterVariadicTypeVariable -> 1 | InvalidKeywordArgument _ -> 0 | InvalidVariableArgument _ -> 0 | Mismatches _ -> -1 | MissingArgument _ -> 1 | MutuallyRecursiveTypeVariables -> 1 | ProtocolInstantiation _ -> 1 | TooManyArguments _ -> 1 | TypedDictionaryInitializationError _ -> 1 | UnexpectedKeyword _ -> 1 in let get_most_important best_reason reason = if importance reason > importance best_reason then reason else match best_reason, reason with | Mismatches mismatches, Mismatches other_mismatches -> Mismatches (List.append mismatches other_mismatches) | _, _ -> best_reason in let sort_mismatches reason = match reason with | Mismatches mismatches -> let compare_mismatches mismatch other_mismatch = match mismatch, other_mismatch with | ( Mismatch { Node.value = { position; _ }; _ }, Mismatch { Node.value = { position = other_position; _ }; _ } ) -> position - other_position | _, _ -> 0 in Mismatches (List.sort mismatches ~compare:compare_mismatches) | _ -> reason in let reason = Some (List.fold ~init:reason ~f:get_most_important reasons |> sort_mismatches) in NotFound { closest_return_annotation = instantiated_return_annotation; reason } let default_signature { Type.Callable.implementation = { annotation = default_return_annotation; _ }; _ } = let open SignatureSelectionTypes in NotFound { closest_return_annotation = default_return_annotation; reason = None } let calculate_rank ({ reasons = { arity; annotation; _ }; _ } as signature_match) = let open SignatureSelectionTypes in let arity_rank = List.length arity in let positions, annotation_rank = let count_unique (positions, count) = function | Mismatches mismatches -> let count_unique_mismatches (positions, count) mismatch = match mismatch with | Mismatch { Node.value = { position; _ }; _ } when not (Set.mem positions position) -> Set.add positions position, count + 1 | Mismatch _ -> positions, count | _ -> positions, count + 1 in List.fold ~init:(positions, count) mismatches ~f:count_unique_mismatches | _ -> positions, count + 1 in List.fold ~init:(Int.Set.empty, 0) ~f:count_unique annotation in let position_rank = Int.Set.min_elt positions >>| Int.neg |> Option.value ~default:Int.min_value in { signature_match with ranks = { arity = arity_rank; annotation = annotation_rank; position = position_rank }; } (** Find the signature that is "closest" to what the user intended. Essentially, sort signatures on the number of arity mismatches, number of annotation mismatches, and the earliest mismatch position. TODO(T109092235): Clean up the rank calculation to more clearly reflect that we want to do `maximum_by (arity, annotation, position)`. *) let find_closest_signature signature_matches = let get_arity_rank { ranks = { arity; _ }; _ } = arity in let get_annotation_rank { ranks = { annotation; _ }; _ } = annotation in let get_position_rank { ranks = { position; _ }; _ } = position in let rec get_best_rank ~best_matches ~best_rank ~getter = function | [] -> best_matches | head :: tail -> let rank = getter head in if rank < best_rank then get_best_rank ~best_matches:[head] ~best_rank:rank ~getter tail else if rank = best_rank then get_best_rank ~best_matches:(head :: best_matches) ~best_rank ~getter tail else get_best_rank ~best_matches ~best_rank ~getter tail in signature_matches |> List.map ~f:calculate_rank |> get_best_rank ~best_matches:[] ~best_rank:Int.max_value ~getter:get_arity_rank |> get_best_rank ~best_matches:[] ~best_rank:Int.max_value ~getter:get_annotation_rank |> get_best_rank ~best_matches:[] ~best_rank:Int.max_value ~getter:get_position_rank (* Each get_best_rank reverses the list, because we have an odd number, we need an extra reverse in order to prefer the first defined overload *) |> List.rev |> List.hd let prepare_arguments_for_signature_selection ~self_argument arguments = let add_positions arguments = let add_index index { Argument.expression; kind; resolved } = { Argument.WithPosition.position = index + 1; expression; kind; resolved } in List.mapi ~f:add_index arguments in let unpack_starred_arguments arguments = let unpack sofar argument = match argument with | { Argument.resolved = Tuple (Concrete tuple_parameters); kind = SingleStar; expression } -> let unpacked_arguments = List.map tuple_parameters ~f:(fun resolved -> { Argument.expression; kind = Positional; resolved }) in List.concat [List.rev unpacked_arguments; sofar] | _ -> argument :: sofar in List.fold ~f:unpack ~init:[] arguments |> List.rev in let separate_labeled_unlabeled_arguments arguments = let is_labeled = function | { Argument.WithPosition.kind = Named _; _ } -> true | _ -> false in let labeled_arguments, unlabeled_arguments = arguments |> List.partition_tf ~f:is_labeled in let self_argument = self_argument >>| (fun resolved -> { Argument.WithPosition.position = 0; expression = None; kind = Positional; resolved }) |> Option.to_list in self_argument @ labeled_arguments @ unlabeled_arguments in arguments |> unpack_starred_arguments |> add_positions |> separate_labeled_unlabeled_arguments end class base class_metadata_environment dependency = object (self) method get_typed_dictionary ~assumptions annotation = match annotation with | Type.Primitive class_name when ClassMetadataEnvironment.ReadOnly.is_typed_dictionary class_metadata_environment ?dependency class_name -> let fields = self#attribute ~assumptions ~transitive:false ~accessed_through_class:true ~include_generated_attributes:true ~instantiated:(Type.meta annotation) ~special_method:false ~attribute_name:"__init__" class_name >>| AnnotatedAttribute.annotation >>| Annotation.annotation >>= function | Type.Callable callable -> Type.TypedDictionary.fields_from_constructor callable | _ -> None in fields >>| fun fields -> { Type.Record.TypedDictionary.fields; name = class_name } | _ -> None method full_order ~assumptions = let resolve class_type = match Type.resolve_class class_type with | None -> None | Some [] -> None | Some [resolved] -> Some resolved | Some (_ :: _) -> (* These come from calling attributes on Unions, which are handled by solve_less_or_equal indirectly by breaking apart the union before doing the instantiate_protocol_parameters. Therefore, there is no reason to deal with joining the attributes together here *) None in let attribute class_type ~assumptions ~name = resolve class_type >>= fun { instantiated; accessed_through_class; class_name } -> self#attribute ~assumptions ~transitive:true ~accessed_through_class ~include_generated_attributes:true ?special_method:None ~attribute_name:name ~instantiated class_name in let all_attributes class_type ~assumptions = resolve class_type >>= fun { instantiated; accessed_through_class; class_name } -> self#all_attributes ~assumptions ~transitive:true ~accessed_through_class ~include_generated_attributes:true ?special_method:None class_name >>| List.map ~f: (self#instantiate_attribute ~assumptions ~instantiated ~accessed_through_class ?apply_descriptors:None) in let is_protocol annotation ~protocol_assumptions:_ = UnannotatedGlobalEnvironment.ReadOnly.is_protocol (unannotated_global_environment class_metadata_environment) ?dependency annotation in let class_hierarchy_handler = ClassHierarchyEnvironment.ReadOnly.class_hierarchy ?dependency (class_hierarchy_environment class_metadata_environment) in let metaclass class_name ~assumptions = self#metaclass class_name ~assumptions in { ConstraintsSet.class_hierarchy = { instantiate_successors_parameters = ClassHierarchy.instantiate_successors_parameters class_hierarchy_handler; is_transitive_successor = ClassHierarchy.is_transitive_successor class_hierarchy_handler; variables = ClassHierarchy.variables class_hierarchy_handler; least_upper_bound = ClassHierarchy.least_upper_bound class_hierarchy_handler; }; attribute; all_attributes; is_protocol; assumptions; get_typed_dictionary = self#get_typed_dictionary ~assumptions; metaclass; } method check_invalid_type_parameters ?(replace_unbound_parameters_with_any = true) ~assumptions annotation = let open TypeParameterValidationTypes in let module InvalidTypeParametersTransform = Type.Transform.Make (struct type state = type_parameters_mismatch list let visit_children_before _ _ = false let visit_children_after = true let visit sofar annotation = let transformed_annotation, new_state = let generics_for_name name = match name with | "type" | "typing.Type" | "typing.ClassVar" | "typing.Iterator" | "Optional" | "typing.Final" | "typing_extensions.Final" | "typing.Optional" -> [Type.Variable.Unary (Type.Variable.Unary.create "T")] | "typing.Callable" -> [ Type.Variable.ParameterVariadic (Type.Variable.Variadic.Parameters.create "Ps"); Type.Variable.Unary (Type.Variable.Unary.create "R"); ] | _ -> ClassHierarchyEnvironment.ReadOnly.variables (class_hierarchy_environment class_metadata_environment) ?dependency name |> Option.value ~default:[] in let invalid_type_parameters ~name ~given = let generics = generics_for_name name in match Type.Variable.zip_variables_with_parameters_including_mismatches ~parameters:given generics with | Some [] -> Type.Primitive name, sofar | Some paired -> let check_parameter { Type.Variable.variable_pair; received_parameter } = match variable_pair, received_parameter with | Type.Variable.UnaryPair (unary, given), Type.Parameter.Single _ -> let invalid = let order = self#full_order ~assumptions in TypeOrder.OrderedConstraints.add_lower_bound TypeConstraints.empty ~order ~pair:variable_pair >>| TypeOrder.OrderedConstraints.add_upper_bound ~order ~pair:variable_pair |> Option.is_none in if invalid then ( [Type.Parameter.Single Type.Any], Some { name; kind = ViolateConstraints { expected = unary; actual = given }; } ) else [Type.Parameter.Single given], None | ParameterVariadicPair (_, given), CallableParameters _ -> (* TODO(T47346673): accept w/ new kind of validation *) [CallableParameters given], None | TupleVariadicPair (_, given), Single (Tuple _) -> Type.OrderedTypes.to_parameters given, None | Type.Variable.UnaryPair (unary, given), _ -> ( [Single given], Some { name; kind = UnexpectedKind { expected = Type.Variable.Unary unary; actual = received_parameter; }; } ) | ParameterVariadicPair (parameter_variadic, given), _ -> ( [CallableParameters given], Some { name; kind = UnexpectedKind { expected = ParameterVariadic parameter_variadic; actual = received_parameter; }; } ) | TupleVariadicPair (tuple_variadic, given), _ -> ( Type.OrderedTypes.to_parameters given, Some { name; kind = UnexpectedKind { expected = TupleVariadic tuple_variadic; actual = received_parameter; }; } ) in List.map paired ~f:check_parameter |> List.unzip |> fun (list_of_parameters, errors) -> ( Type.parametric name (List.concat list_of_parameters), List.filter_map errors ~f:Fn.id @ sofar ) | None when not replace_unbound_parameters_with_any -> Type.parametric name (List.map generics ~f:Type.Variable.to_parameter), sofar | None -> let annotation, expected_parameter_count, can_accept_more_parameters = match name with | "typing.Callable" -> Type.Callable.create ~annotation:Type.Any (), List.length generics, false | "tuple" -> ( Type.Tuple (Type.OrderedTypes.create_unbounded_concatenation Type.Any), List.length generics, true ) | _ -> let is_tuple_variadic = function | Type.Variable.TupleVariadic _ -> true | _ -> false in let annotation = Type.parametric name (List.concat_map generics ~f:(function | Type.Variable.Unary _ -> [Type.Parameter.Single Type.Any] | ParameterVariadic _ -> [CallableParameters Undefined] | TupleVariadic _ -> Type.OrderedTypes.to_parameters Type.Variable.Variadic.Tuple.any)) in ( annotation, List.filter generics ~f:(fun x -> not (is_tuple_variadic x)) |> List.length, List.exists generics ~f:is_tuple_variadic ) in let mismatch = { name; kind = IncorrectNumberOfParameters { actual = List.length given; expected = expected_parameter_count; can_accept_more_parameters; }; } in annotation, mismatch :: sofar in match annotation with | Type.Primitive ("typing.Final" | "typing_extensions.Final") -> annotation, sofar | Type.Primitive name -> invalid_type_parameters ~name ~given:[] (* natural variadics *) | Type.Parametric { name = "typing.Protocol"; _ } | Type.Parametric { name = "typing.Generic"; _ } -> annotation, sofar | Type.Parametric { name; parameters } -> invalid_type_parameters ~name ~given:parameters | Type.IntExpression _ -> let invalid_generic_int given = let generic_int = Type.Variable.Unary.create "T" ~constraints:(Type.Record.Variable.Bound (Primitive "int")) in let invalid = let order = self#full_order ~assumptions in let pair = Type.Variable.UnaryPair (generic_int, given) in TypeOrder.OrderedConstraints.add_lower_bound TypeConstraints.empty ~order ~pair >>= TypeOrder.OrderedConstraints.add_upper_bound ~order ~pair |> Option.is_none in if invalid then Some { name = "IntExpression"; kind = ViolateConstraints { actual = given; expected = generic_int }; } else None in let errors = Type.Variable.GlobalTransforms.Unary.collect_all annotation |> List.filter_map ~f:(fun variable -> invalid_generic_int (Type.Variable variable)) in if List.length errors > 0 then Any, errors @ sofar else annotation, sofar | _ -> annotation, sofar in { Type.Transform.transformed_annotation; new_state } end) in InvalidTypeParametersTransform.visit [] annotation method parse_annotation ~assumptions ?(validation = SharedMemoryKeys.ParseAnnotationKey.ValidatePrimitivesAndTypeParameters) expression = let modify_aliases ?replace_unbound_parameters_with_any = function | Type.TypeAlias alias -> self#check_invalid_type_parameters ?replace_unbound_parameters_with_any alias ~assumptions |> snd |> fun alias -> Type.TypeAlias alias | result -> result in let allow_untracked = match validation with | NoValidation -> true | ValidatePrimitives | ValidatePrimitivesAndTypeParameters -> false in let annotation = AliasEnvironment.ReadOnly.parse_annotation_without_validating_type_parameters (alias_environment class_metadata_environment) ~modify_aliases ?dependency ~allow_untracked expression in let result = match validation with | ValidatePrimitivesAndTypeParameters -> self#check_invalid_type_parameters annotation ~assumptions |> snd | NoValidation | ValidatePrimitives -> annotation in result method sqlalchemy_attribute_table ~assumptions ~include_generated_attributes ~in_test ~accessed_via_metaclass ({ Node.value = { ClassSummary.name = parent_name; _ }; _ } as parent) = let class_name = Reference.show parent_name in let unannotated_attributes ~include_generated_attributes ~in_test ({ Node.value = class_summary; _ } as parent) = let attributes = ClassSummary.attributes ~include_generated_attributes ~in_test class_summary |> Identifier.SerializableMap.bindings |> List.map ~f:(fun (_, attribute) -> attribute) in let unannotated_attribute { Node.value = attribute; _ } = self#create_attribute ~assumptions ~parent ?defined:(Some true) ~accessed_via_metaclass attribute in List.map attributes ~f:unannotated_attribute in let add_constructor table = let successor_definitions = let class_definition = UnannotatedGlobalEnvironment.ReadOnly.get_class_definition (ClassMetadataEnvironment.ReadOnly.unannotated_global_environment class_metadata_environment) ?dependency in ClassMetadataEnvironment.ReadOnly.successors class_metadata_environment ?dependency class_name |> List.filter_map ~f:class_definition in let name_annotation_pairs = let name_annotation_pair attribute = let name = AnnotatedAttribute.name attribute in if Expression.is_dunder_attribute name || AnnotatedAttribute.is_private attribute then None else let annotation = self#instantiate_attribute ~assumptions ~accessed_through_class:false ?instantiated:None ?apply_descriptors:None attribute |> AnnotatedAttribute.annotation |> Annotation.annotation in Some (name, annotation) in parent :: successor_definitions |> List.concat_map ~f:(unannotated_attributes ~include_generated_attributes:false ~in_test:false) |> List.filter_map ~f:name_annotation_pair (* Pick the overriding attribute. *) |> Identifier.Map.of_alist_reduce ~f:(fun first _ -> first) |> Identifier.Map.to_alist in let parameters = let keyword_only_parameter (name, annotation) = Type.Record.Callable.RecordParameter.KeywordOnly { name = Format.asprintf "$parameter$%s" name; annotation; default = true } in let self_parameter = Type.Callable.Parameter.Named { name = "$parameter$self"; annotation = Type.Primitive class_name; default = false } in List.map ~f:keyword_only_parameter name_annotation_pairs |> fun parameters -> Type.Record.Callable.Defined (self_parameter :: parameters) in let constructor = { Type.Callable.kind = Named (Reference.create ~prefix:parent_name "__init__"); implementation = { annotation = Type.none; parameters }; overloads = []; } in AnnotatedAttribute.create_uninstantiated ~abstract:false ~uninstantiated_annotation:(create_uninstantiated_method constructor) ~async_property:false ~class_variable:false ~defined:true ~initialized:OnClass ~name:"__init__" ~parent:class_name ~visibility:ReadWrite ~property:false ~undecorated_signature:(Some constructor) ~problem:None |> UninstantiatedAttributeTable.add table in let add_special_attribute ~name ~annotation table = AnnotatedAttribute.create_uninstantiated ~abstract:false ~uninstantiated_annotation: { UninstantiatedAnnotation.accessed_via_metaclass = false; kind = UninstantiatedAnnotation.Attribute annotation; } ~async_property:false ~class_variable:false ~defined:true ~initialized:OnClass ~name ~parent:class_name ~visibility:ReadWrite ~property:false ~undecorated_signature:None ~problem:None |> UninstantiatedAttributeTable.add table in let table = UninstantiatedAttributeTable.create () in unannotated_attributes ~include_generated_attributes ~in_test parent |> List.iter ~f:(UninstantiatedAttributeTable.add table); if include_generated_attributes then add_constructor table; add_special_attribute ~name:"metadata" ~annotation:(Type.Primitive "sqlalchemy.sql.schema.MetaData") table; add_special_attribute ~name:"__table__" ~annotation:(Type.Primitive "sqlalchemy.sql.schema.Table") table; table method typed_dictionary_special_methods_table ~assumptions ~include_generated_attributes ~in_test ~accessed_via_metaclass ?dependency ~class_metadata_environment ~class_name ({ Node.value = { ClassSummary.name; _ }; _ } as parent_definition) = let table = UninstantiatedAttributeTable.create () in let add_special_methods () = let class_definition = UnannotatedGlobalEnvironment.ReadOnly.get_class_definition (ClassMetadataEnvironment.ReadOnly.unannotated_global_environment class_metadata_environment) ?dependency in let successor_definitions = ClassMetadataEnvironment.ReadOnly.successors class_metadata_environment ?dependency (Reference.show name) |> List.filter_map ~f:class_definition in let total = ClassHierarchy.is_total_typed_dictionary ~class_hierarchy: (ClassHierarchyEnvironment.ReadOnly.class_hierarchy ?dependency (class_hierarchy_environment class_metadata_environment)) class_name in let base_typed_dictionary_definition = match class_definition (Type.TypedDictionary.class_name ~total) with | Some definition -> definition | None -> failwith "Expected to find TypedDictionary" in let typed_dictionary_definitions = List.filter (parent_definition :: successor_definitions) ~f:(fun { Node.value = { ClassSummary.name; _ }; _ } -> ClassMetadataEnvironment.ReadOnly.is_typed_dictionary class_metadata_environment ?dependency (Reference.show name)) in let get_field_attributes ~include_generated_attributes { Node.value = { bases = { ClassSummary.base_classes; _ }; _ } as class_summary; _ } = let required = not (List.exists base_classes ~f:(fun base_expression -> String.equal (Expression.show base_expression) (Type.TypedDictionary.class_name ~total:false))) in ClassSummary.attributes ~include_generated_attributes ~in_test class_summary |> Identifier.SerializableMap.bindings |> List.map ~f:(fun (_, field_attribute) -> ( self#create_attribute ~assumptions ~parent:parent_definition ?defined:(Some true) ~accessed_via_metaclass (Node.value field_attribute), required )) in let attribute_to_typed_dictionary_field (attribute, required) = match AnnotatedAttribute.uninstantiated_annotation attribute with | { UninstantiatedAnnotation.kind = Attribute annotation; _ } -> Some (Type.TypedDictionary.create_field ~name:(AnnotatedAttribute.name attribute) ~annotation ~required) | _ -> None in let keep_last_declarations fields = List.map fields ~f:(fun (field : Type.t Type.Record.TypedDictionary.typed_dictionary_field) -> field.name, field) |> Map.of_alist_multi (module String) |> Map.to_alist |> List.map ~f:(fun (_, fields) -> List.last_exn fields) in let fields = List.rev typed_dictionary_definitions |> List.concat_map ~f:(get_field_attributes ~include_generated_attributes:false) |> List.filter_map ~f:attribute_to_typed_dictionary_field |> keep_last_declarations in let overload_method (attribute, _) = match AnnotatedAttribute.uninstantiated_annotation attribute with | { UninstantiatedAnnotation.kind = Attribute (Callable callable); _ } as uninstantiated_annotation -> let overloaded_callable overloads = { callable with Type.Callable.implementation = { annotation = Type.Top; parameters = Undefined }; overloads; } in Type.TypedDictionary.special_overloads ~class_name ~fields ~method_name:(AnnotatedAttribute.name attribute) >>| overloaded_callable >>| fun callable -> AnnotatedAttribute.with_uninstantiated_annotation ~uninstantiated_annotation: { uninstantiated_annotation with UninstantiatedAnnotation.kind = Attribute (Callable callable); } attribute |> AnnotatedAttribute.with_undecorated_signature ~undecorated_signature:(Some callable) | _ -> None in let constructor = let constructor = Type.TypedDictionary.constructor ~name:class_name ~fields in constructor |> create_uninstantiated_method |> fun uninstantiated_annotation -> AnnotatedAttribute.create_uninstantiated ~uninstantiated_annotation ~abstract:false ~async_property:false ~class_variable:false ~defined:true ~initialized:OnClass ~name:"__init__" ~parent:class_name ~visibility:ReadWrite ~property:false ~undecorated_signature:(Some constructor) ~problem:None in let all_special_methods = constructor :: (get_field_attributes ~include_generated_attributes:true base_typed_dictionary_definition |> List.filter_map ~f:overload_method) in List.iter ~f:(UninstantiatedAttributeTable.add table) all_special_methods in if include_generated_attributes then add_special_methods (); table method single_uninstantiated_attribute_table ~assumptions ~include_generated_attributes ~accessed_via_metaclass class_name = let handle ({ Node.value = class_summary; _ } as parent) ~in_test = let table = UninstantiatedAttributeTable.create () in let add_actual () = let collect_attributes attribute = self#create_attribute (Node.value attribute) ~assumptions ~parent ~accessed_via_metaclass |> UninstantiatedAttributeTable.add table in ClassSummary.attributes ~include_generated_attributes ~in_test class_summary |> fun attribute_map -> Identifier.SerializableMap.iter (fun _ data -> collect_attributes data) attribute_map in let add_placeholder_stub_inheritances () = let add_if_missing ~attribute_name ~annotation = if Option.is_none (UninstantiatedAttributeTable.lookup_name table attribute_name) then let callable = { Type.Callable.kind = Anonymous; implementation = { annotation; parameters = Undefined }; overloads = []; } in UninstantiatedAttributeTable.add table (AnnotatedAttribute.create_uninstantiated ~uninstantiated_annotation: { UninstantiatedAnnotation.accessed_via_metaclass; kind = Attribute (Callable callable); } ~abstract:false ~async_property:false ~class_variable:false ~defined:true ~initialized:OnClass ~name:attribute_name ~parent:class_name ~visibility:ReadWrite ~property:false ~undecorated_signature:(Some callable) ~problem:None) else () in add_if_missing ~attribute_name:"__init__" ~annotation:Type.none; add_if_missing ~attribute_name:"__getattr__" ~annotation:Type.Any in add_actual (); if include_generated_attributes && AnnotatedBases.extends_placeholder_stub_class parent ~aliases:(aliases class_metadata_environment ~dependency) ~from_empty_stub:(is_suppressed_module class_metadata_environment ~dependency) then add_placeholder_stub_inheritances (); let () = if include_generated_attributes then ClassDecorators.apply ~definition:parent ~class_metadata_environment ~create_attribute:(self#create_attribute ~assumptions) ~instantiate_attribute: (self#instantiate_attribute ~assumptions ?instantiated:None ~accessed_through_class: false (* TODO(T65806273): Right now we're just ignoring `__set__`s on dataclass attributes. This avoids needing to explicitly break the loop that would otherwise result or to somehow separate these results from the main set of attributes *) ~apply_descriptors:false) ?dependency table in table in match ( UnannotatedGlobalEnvironment.ReadOnly.get_class_definition (ClassMetadataEnvironment.ReadOnly.unannotated_global_environment class_metadata_environment) ?dependency class_name, ClassMetadataEnvironment.ReadOnly.get_class_metadata class_metadata_environment ?dependency class_name ) with | Some definition, Some { is_typed_dictionary; is_test = in_test; _ } -> let is_declarative_sqlalchemy_class () = Option.equal Type.equal (self#metaclass ~assumptions class_name) (Some (Type.Primitive "sqlalchemy.ext.declarative.api.DeclarativeMeta")) in let table = if is_typed_dictionary then self#typed_dictionary_special_methods_table ~assumptions ~include_generated_attributes ~in_test ~accessed_via_metaclass ~class_metadata_environment ?dependency ~class_name definition else if is_declarative_sqlalchemy_class () then self#sqlalchemy_attribute_table ~assumptions ~include_generated_attributes ~in_test ~accessed_via_metaclass definition else handle definition ~in_test in Some table | _ -> None method uninstantiated_attribute_tables ~assumptions ~transitive ~accessed_through_class ~include_generated_attributes ~special_method class_name = let handle { ClassMetadataEnvironment.successors; _ } = let get_table ~accessed_via_metaclass = self#single_uninstantiated_attribute_table ~assumptions ~include_generated_attributes ~accessed_via_metaclass in let normal_tables = let normal_hierarchy = (* Pass over normal class hierarchy. *) if accessed_through_class && special_method then [] else if transitive then class_name :: successors else [class_name] in Sequence.of_list normal_hierarchy |> Sequence.filter_map ~f:(get_table ~accessed_via_metaclass:false) in let metaclass_tables = (* We don't want to have to find our metaclass/it's parents if we successfully find the attribute in one of our actual parents *) lazy begin let metaclass_hierarchy = (* Class over meta hierarchy if necessary. *) if accessed_through_class then let successors_of class_name = ClassMetadataEnvironment.ReadOnly.successors class_metadata_environment ?dependency class_name in self#metaclass ~assumptions class_name >>| Type.split >>| fst >>= Type.primitive_name >>| (fun metaclass -> metaclass :: successors_of metaclass) |> Option.value ~default:[] else [] in metaclass_hierarchy |> Sequence.of_list |> Sequence.filter_map ~f:(get_table ~accessed_via_metaclass:true) end in Sequence.append normal_tables (Sequence.of_lazy metaclass_tables) in ClassMetadataEnvironment.ReadOnly.get_class_metadata class_metadata_environment ?dependency class_name >>| handle method attribute ~assumptions ~transitive ~accessed_through_class ~include_generated_attributes ?(special_method = false) ?instantiated ?apply_descriptors ~attribute_name class_name = let order () = self#full_order ~assumptions in match callable_call_special_cases ~instantiated ~class_name ~attribute_name ~accessed_through_class ~order with | Some callable -> AnnotatedAttribute.create ~annotation:callable ~original_annotation:callable ~uninstantiated_annotation:None ~visibility:ReadWrite ~abstract:false ~async_property:false ~class_variable:false ~defined:true ~initialized:OnClass ~name:"__call__" ~parent:"typing.Callable" ~property:false ~undecorated_signature:None ~problem:None |> Option.some | None -> self#uninstantiated_attribute_tables ~assumptions ~transitive ~accessed_through_class ~include_generated_attributes ~special_method class_name >>= Sequence.find_map ~f:(fun table -> UninstantiatedAttributeTable.lookup_name table attribute_name) >>| self#instantiate_attribute ~assumptions ~accessed_through_class ?instantiated ?apply_descriptors method all_attributes ~assumptions ~transitive ~accessed_through_class ~include_generated_attributes ?(special_method = false) class_name = let collect sofar table = let add ((sofar_list, sofar_set) as sofar) attribute = let name = AnnotatedAttribute.name attribute in if Set.mem sofar_set name then sofar else attribute :: sofar_list, Set.add sofar_set name in UninstantiatedAttributeTable.to_list table |> List.fold ~f:add ~init:sofar in self#uninstantiated_attribute_tables ~assumptions ~transitive ~accessed_through_class ~include_generated_attributes ~special_method class_name >>| Sequence.fold ~f:collect ~init:([], Identifier.Set.empty) >>| fst >>| List.rev method attribute_names ~assumptions ~transitive ~accessed_through_class ~include_generated_attributes ?(special_method = false) class_name = let collect sofar table = let add ((sofar_list, sofar_set) as sofar) name = if Set.mem sofar_set name then sofar else name :: sofar_list, Set.add sofar_set name in UninstantiatedAttributeTable.names table |> List.fold ~f:add ~init:sofar in self#uninstantiated_attribute_tables ~assumptions ~transitive ~accessed_through_class ~include_generated_attributes ~special_method class_name >>| Sequence.fold ~f:collect ~init:([], Identifier.Set.empty) >>| fst >>| List.rev method instantiate_attribute ~assumptions ~accessed_through_class ?instantiated ?(apply_descriptors = true) attribute = let get_attribute = self#attribute in let class_name = AnnotatedAttribute.parent attribute in let attribute_name = AnnotatedAttribute.name attribute in let { UninstantiatedAnnotation.accessed_via_metaclass; kind = annotation } = AnnotatedAttribute.uninstantiated_annotation attribute in let accessed_through_class = accessed_through_class && not accessed_via_metaclass in let uninstantiated_annotation = match annotation with | Attribute annotation -> Some annotation | Property _ -> None in let annotation = match instantiated with | None -> annotation | Some instantiated -> ( let solution = self#constraints ~target:class_name ~instantiated ~assumptions () in let instantiate annotation = ConstraintsSet.Solution.instantiate solution annotation in match annotation with | Attribute annotation -> UninstantiatedAnnotation.Attribute (instantiate annotation) | Property { getter; setter } -> let instantiate_property_annotation { UninstantiatedAnnotation.self; value } = { UninstantiatedAnnotation.self = self >>| instantiate; value = value >>| instantiate; } in Property { getter = instantiate_property_annotation getter; setter = setter >>| instantiate_property_annotation; }) in let annotation, original = let instantiated = match instantiated with | Some instantiated -> instantiated | None -> Type.Primitive class_name in let instantiated = if accessed_via_metaclass then Type.meta instantiated else instantiated in let special_case_methods callable = (* Certain callables' types can't be expressed directly and need to be special cased *) let self_parameter = Type.Callable.Parameter.Named { name = "self"; annotation = Type.Top; default = false } in match instantiated, attribute_name, class_name with | Type.Tuple (Concrete members), "__getitem__", _ -> let { Type.Callable.overloads; _ } = callable in let overload index member = { Type.Callable.annotation = member; parameters = Defined [ self_parameter; Named { name = "x"; annotation = Type.literal_integer index; default = false }; ]; } in let overloads = List.mapi ~f:overload members @ List.map2_exn ~f:overload (List.init ~f:(fun x -> -x - 1) (List.length members)) (List.rev members) @ overloads in Type.Callable { callable with overloads } | ( Parametric { name = "type"; parameters = [Single (Type.Primitive name)] }, "__getitem__", "typing.GenericMeta" ) -> let implementation, overloads = let generics = ClassHierarchyEnvironment.ReadOnly.variables (ClassMetadataEnvironment.ReadOnly.class_hierarchy_environment class_metadata_environment) ?dependency name |> Option.value ~default:[] in let create_parameter annotation = Type.Callable.Parameter.PositionalOnly { index = 0; annotation; default = false } in let synthetic = Type.Variable (Type.Variable.Unary.create "$synthetic_attribute_resolution_variable") in match name with (* This can't be expressed without IntVars, StrVars, and corresponding TypeVarTuple variants of them *) | "typing_extensions.Literal" | "typing.Literal" (* TODO:(T60535947) We can't do the Map[Ts, type] -> X[Ts] trick here because we don't yet support Union[Ts] *) | "typing.Union" -> { Type.Callable.annotation = Type.meta Type.Any; parameters = Undefined }, [] | "typing.Callable" -> ( { Type.Callable.annotation = Type.meta (Type.Callable.create ~annotation:synthetic ()); parameters = Defined [ self_parameter; create_parameter (Type.Tuple (Concrete [Type.Any; Type.meta synthetic])); ]; }, [] ) | _ -> ( let overload parameter = let generics = List.map generics ~f:Type.Variable.to_parameter in match name, generics with | "typing.Optional", [Single generic] -> { Type.Callable.annotation = Type.meta (Type.optional generic); parameters = Defined [self_parameter; parameter]; } | _ -> { Type.Callable.annotation = Type.meta (Type.parametric name generics); parameters = Defined [self_parameter; parameter]; } in match generics with | [Unary generic] -> overload (create_parameter (Type.meta (Variable generic))), [] | _ -> (* To support the value `GenericFoo[int, str]`, we need `class GenericFoo[T1, T2]` to have: `def __getitem__(cls, __x: Tuple[Type[T1], Type[T2]] ) -> GenericFoo[T1, T2]`. *) let meta_type_and_return_type = function | Type.Variable.Unary single -> ( Type.meta (Variable single), Type.Parameter.Single (Type.Variable single) ) | ParameterVariadic _ -> (* TODO:(T60536033) We'd really like to take FiniteList[Ts], but without that we can't actually return the correct metatype, which is a bummer *) Type.Any, Type.Parameter.CallableParameters Undefined | TupleVariadic _ -> Type.Any, Single Any in let meta_types, return_parameters = List.map generics ~f:meta_type_and_return_type |> List.unzip in ( { Type.Callable.annotation = Type.meta (Type.parametric name return_parameters); parameters = Defined [self_parameter; create_parameter (Type.tuple meta_types)]; }, [] )) in Type.Callable { callable with implementation; overloads } | Parametric { name = "type"; parameters = [Single meta_parameter] }, "__call__", "type" when accessed_via_metaclass -> let get_constructor { Type.instantiated; accessed_through_class; class_name } = if accessed_through_class then (* Type[Type[X]] is invalid *) None else Some (self#constructor ~assumptions class_name ~instantiated) in Type.resolve_class meta_parameter >>| List.map ~f:get_constructor >>= Option.all >>| Type.union |> Option.value ~default:(Type.Callable callable) | _ -> Type.Callable callable in match annotation with | Property { getter = getter_annotation; setter = setter_annotation } -> ( (* Special case properties with type variables. *) let solve_property { UninstantiatedAnnotation.self = self_annotation; value = value_annotation } = match value_annotation with | None -> Type.Top | Some annotation -> ( let order = self#full_order ~assumptions in let constraints = match self_annotation with | Some annotation -> TypeOrder.OrderedConstraintsSet.add ConstraintsSet.empty ~new_constraint:(LessOrEqual { left = instantiated; right = annotation }) ~order | None -> ConstraintsSet.empty in match TypeOrder.OrderedConstraintsSet.solve ~order constraints with | Some solution -> ConstraintsSet.Solution.instantiate solution annotation | None -> Type.Top) in match setter_annotation with | Some setter_annotation -> solve_property getter_annotation, solve_property setter_annotation | None -> let annotation = solve_property getter_annotation in annotation, annotation) | Attribute annotation -> ( let annotation = match annotation with | Type.Callable callable -> special_case_methods callable | other -> other in let order () = self#full_order ~assumptions in let special = callable_call_special_cases ~instantiated:(Some instantiated) ~class_name ~attribute_name ~order ~accessed_through_class >>| fun callable -> callable, callable in match special with | Some special -> special | None when [%compare.equal: AnnotatedAttribute.initialized] (AnnotatedAttribute.initialized attribute) OnClass && apply_descriptors -> ( let call_dunder_get (descriptor, callable) = let selection_result = self#signature_select ~assumptions ~arguments: [ { Argument.kind = Positional; expression = None; resolved = descriptor }; { Argument.kind = Positional; expression = None; resolved = (if accessed_through_class then Type.none else instantiated); }; { Argument.kind = Positional; expression = None; resolved = Type.meta instantiated; }; ] ~resolve_with_locals:(fun ~locals:_ _ -> Type.object_primitive) ~callable ~self_argument:None ~skip_marking_escapees:true in match selection_result with | SignatureSelectionTypes.NotFound _ -> None | Found { selected_return_annotation = return } -> Some return in let invert_dunder_set (descriptor, callable) ~order = let synthetic = Type.Variable.Unary.create "$synthetic_dunder_set_variable" in let right = Type.Callable.create ~annotation:Type.none ~parameters: (Defined [ PositionalOnly { index = 0; annotation = descriptor; default = false }; PositionalOnly { index = 1; annotation = instantiated; default = false }; PositionalOnly { index = 2; annotation = Variable synthetic; default = false }; ]) () in TypeOrder.OrderedConstraintsSet.add ConstraintsSet.empty ~new_constraint:(LessOrEqual { left = Type.Callable callable; right }) ~order |> TypeOrder.OrderedConstraintsSet.solve ~order >>= fun solution -> ConstraintsSet.Solution.instantiate_single_variable solution synthetic in let function_dunder_get callable = if accessed_through_class then Type.Callable callable else Type.parametric "BoundMethod" [Single (Callable callable); Single instantiated] in let get_descriptor_method { Type.instantiated; accessed_through_class; class_name } ~kind = if accessed_through_class then (* descriptor methods are statically looked up on the class (in this case `type`), not on the instance. `type` is not a descriptor. *) `NotDescriptor (Type.meta instantiated) else match instantiated with | Callable callable -> ( match kind with | `DunderGet -> (* We unsoundly assume all callables are callables with the `function` `__get__` *) `HadDescriptor (function_dunder_get callable) | `DunderSet -> `NotDescriptor instantiated) | _ -> ( let attribute = let attribute_name = match kind with | `DunderGet -> "__get__" | `DunderSet -> "__set__" in (* descriptor methods are statically looked up on the class, and are not themselves subject to description *) get_attribute ~assumptions ~transitive:true ~accessed_through_class:true ~include_generated_attributes:true ?special_method:None ?instantiated:(Some instantiated) ?apply_descriptors:(Some false) ~attribute_name class_name >>| AnnotatedAttribute.annotation >>| Annotation.annotation in match attribute with | None -> `NotDescriptor instantiated | Some (Type.Callable callable) -> let extracted = match kind with | `DunderGet -> call_dunder_get (instantiated, callable) | `DunderSet -> invert_dunder_set ~order:(order ()) (instantiated, callable) in extracted >>| (fun extracted -> `HadDescriptor extracted) |> Option.value ~default:`FailedToExtract | Some _ -> (* In theory we could support `__get__`s or `__set__`s that are not just Callables, but for now lets just ignore that *) `DescriptorNotACallable) in match Type.resolve_class annotation with | None -> (* This means we have a type that can't be `Type.split`, (most of) which aren't descriptors, so we should be usually safe to just ignore. In general we should fix resolve_class to always return something. *) annotation, annotation | Some elements -> let collect x = let partitioner = function | `NotDescriptor element -> `Fst element | `HadDescriptor element -> `Snd element (* Every descriptor should accept all hosts (and all host types) as a matter of Liskov substitutibility with `object`. This means we need to error on these invalid definitions (T65807232), and not on usages *) | `FailedToExtract | `DescriptorNotACallable -> `Trd () in match List.partition3_map x ~f:partitioner with | _, _, _ :: _ -> (* If we have broken descriptor methods we should error on them, not their usages *) Type.Any | _, [], _ -> (* If none of the components are descriptors, we don't need to worry about re-unioning together the components we split apart, we can just give back the original type *) annotation | normal, had_descriptors, _ -> Type.union (normal @ had_descriptors) in let elements_and_get_results = List.map elements ~f:(fun element -> element, get_descriptor_method element ~kind:`DunderGet) in let get_type = List.unzip elements_and_get_results |> snd |> collect in let set_type = if accessed_through_class then annotation else let process (element, get_result) = match get_descriptor_method element ~kind:`DunderSet, get_result with | `NotDescriptor _, `HadDescriptor element -> (* non-data descriptors set type should be their get type *) `HadDescriptor element | other, _ -> other in List.map elements_and_get_results ~f:process |> collect in get_type, set_type) | None -> annotation, annotation) in AnnotatedAttribute.instantiate attribute ~annotation ~original_annotation:original ~uninstantiated_annotation method create_attribute ~assumptions ~parent ?(defined = true) ~accessed_via_metaclass { Attribute.name = attribute_name; kind } = let { Node.value = { ClassSummary.name = parent_name; _ }; _ } = parent in let parent_name = Reference.show parent_name in let class_annotation = Type.Primitive parent_name in let annotation, class_variable, visibility, undecorated_signature, problem = match kind with | Simple { annotation; values; frozen; toplevel; implicit; primitive; _ } -> let value = List.hd values >>| fun { value; _ } -> value in let parsed_annotation = annotation >>| self#parse_annotation ~assumptions in (* Account for class attributes. *) let annotation, final, class_variable = parsed_annotation >>| (fun annotation -> let process_class_variable annotation = match Type.class_variable_value annotation with | Some annotation -> true, annotation | None -> false, annotation in match Type.final_value annotation with | `NoParameter -> None, true, false | `NotFinal -> let is_class_variable, annotation = process_class_variable annotation in Some annotation, false, is_class_variable | `Ok annotation -> let is_class_variable, annotation = process_class_variable annotation in Some annotation, true, is_class_variable) |> Option.value ~default:(None, false, false) in (* Handle enumeration attributes. *) let annotation, visibility = let superclasses = ClassMetadataEnvironment.ReadOnly.successors class_metadata_environment ?dependency parent_name |> String.Set.of_list in if (not (Set.mem Recognized.enumeration_classes (Type.show class_annotation))) && (not (Set.is_empty (Set.inter Recognized.enumeration_classes superclasses))) && primitive && defined && not implicit then ( Some (Type.Literal (Type.EnumerationMember { enumeration_type = class_annotation; member_name = attribute_name })), AnnotatedAttribute.ReadOnly (Refinable { overridable = true }) ) else let visibility = if final then AnnotatedAttribute.ReadOnly (Refinable { overridable = false }) else if frozen then ReadOnly (Refinable { overridable = true }) else ReadWrite in annotation, visibility in (* Try resolve to tuple of string literal types for __match_args__ *) let annotation = let open Expression in match attribute_name, annotation, value with | "__match_args__", None, Some { Node.value = Expression.Tuple elements; _ } -> let string_literal_value_to_type = function | { Node.value = Expression.Constant (Constant.String { StringLiteral.kind = String; value }); _; } -> Some (Type.Literal (String (LiteralValue value))) | _ -> None in List.map elements ~f:string_literal_value_to_type |> Option.all >>| Type.tuple | _ -> annotation in let annotation = match annotation, value with | Some annotation, _ -> annotation | None, Some value -> let literal_value_annotation = self#resolve_literal ~assumptions value in let is_dataclass_attribute = UnannotatedGlobalEnvironment.ReadOnly.exists_matching_class_decorator (unannotated_global_environment class_metadata_environment) ?dependency ~names:["dataclasses.dataclass"; "dataclass"] parent in if (not (Type.is_partially_typed literal_value_annotation)) && (not is_dataclass_attribute) && toplevel then (* Treat literal attributes as having been explicitly annotated. *) literal_value_annotation else Type.Top | _ -> Type.Top in UninstantiatedAnnotation.Attribute annotation, class_variable, visibility, None, None | Method { signatures; final; _ } -> (* Handle Callables *) let visibility = if final then AnnotatedAttribute.ReadOnly (Refinable { overridable = false }) else ReadWrite in let callable, undecorated_signature, problem = let overloads = let create_overload define = Define.Signature.is_overloaded_function define, define in List.map signatures ~f:create_overload in let implementation, overloads = let to_signature (implementation, overloads) (is_overload, signature) = if is_overload then implementation, signature :: overloads else Some signature, overloads in List.fold ~init:(None, []) ~f:to_signature overloads in let { decorated; undecorated_signature } = self#resolve_define ~implementation ~overloads ~assumptions in let annotation = match decorated with | Ok resolved -> ( match attribute_name, resolved with (* these names are only magic-ed into being ClassMethods/StaticMethods if they're "plain functions". We can't capture that in the type system, so we approximate with Callable *) | "__new__", Callable _ -> Type.parametric "typing.StaticMethod" [Single resolved] | "__init_subclass__", Callable _ | "__class_getitem__", Callable _ -> Type.parametric "typing.ClassMethod" [Single resolved] | _ -> resolved) | Error _ -> Any in ( UninstantiatedAnnotation.Attribute annotation, undecorated_signature, Result.error decorated ) in callable, false, visibility, Some undecorated_signature, problem | Property { kind; _ } -> ( let parse_annotation_option annotation = annotation >>| self#parse_annotation ~assumptions in match kind with | ReadWrite { getter = { self = getter_self_annotation; return = getter_annotation; _ }; setter = { self = setter_self_annotation; value = setter_annotation; _ }; } -> let getter_annotation = parse_annotation_option getter_annotation in let setter_annotation = parse_annotation_option setter_annotation in ( UninstantiatedAnnotation.Property { getter = { self = parse_annotation_option getter_self_annotation; value = getter_annotation; }; setter = Some { self = parse_annotation_option setter_self_annotation; value = setter_annotation; }; }, false, ReadWrite, None, None ) | ReadOnly { getter = { self = self_annotation; return = getter_annotation; _ } } -> let annotation = parse_annotation_option getter_annotation in ( UninstantiatedAnnotation.Property { getter = { self = parse_annotation_option self_annotation; value = annotation }; setter = None; }, false, ReadOnly Unrefinable, None, None )) in let initialized = match kind with | Simple { nested_class = true; _ } -> AnnotatedAttribute.OnClass | Simple { values; _ } -> let is_not_ellipsis = function | { Attribute.value = { Node.value = Constant Expression.Constant.Ellipsis; _ }; _ } -> false | _ -> true in List.find values ~f:is_not_ellipsis >>| (function | { Attribute.origin = Explicit; _ } -> AnnotatedAttribute.OnClass | { origin = Implicit; _ } -> OnlyOnInstance) |> Option.value ~default:AnnotatedAttribute.NotInitialized | Method _ | Property { class_property = true; _ } -> OnClass | Property { class_property = false; _ } -> OnlyOnInstance in AnnotatedAttribute.create_uninstantiated ~uninstantiated_annotation: { UninstantiatedAnnotation.accessed_via_metaclass; kind = annotation } ~visibility ~abstract: (match kind with | Method { signatures; _ } -> List.exists signatures ~f:Define.Signature.is_abstract_method | _ -> false) ~async_property: (match kind with | Property { async; _ } -> async | _ -> false) ~class_variable ~defined ~initialized ~name:attribute_name ~parent:parent_name ~property: (match kind with | Property _ -> true | _ -> false) ~undecorated_signature ~problem method metaclass ~assumptions target = (* See https://docs.python.org/3/reference/datamodel.html#determining-the-appropriate-metaclass for why we need to consider all metaclasses. *) let rec handle ({ Node.value = { ClassSummary.bases = { base_classes; metaclass; _ }; _ }; _ } as original) = let open Expression in let parse_annotation = self#parse_annotation ~assumptions ?validation:None in let metaclass_candidates = let explicit_metaclass = metaclass >>| parse_annotation in let metaclass_of_bases = let explicit_bases = let base_to_class base_expression = delocalize base_expression |> parse_annotation |> Type.split |> fst in base_classes |> List.map ~f:base_to_class |> List.filter_map ~f:(class_definition class_metadata_environment ~dependency) |> List.filter ~f:(fun base_class -> not ([%compare.equal: ClassSummary.t Node.t] base_class original)) in let filter_generic_meta base_metaclasses = (* We only want a class directly inheriting from Generic to have a metaclass of GenericMeta. *) if List.exists ~f:(fun base -> Reference.equal (Reference.create "typing.Generic") (class_name base)) explicit_bases then base_metaclasses else List.filter ~f:(fun metaclass -> not (Type.equal (Type.Primitive "typing.GenericMeta") metaclass)) base_metaclasses in explicit_bases |> List.map ~f:handle |> filter_generic_meta in match explicit_metaclass with | Some metaclass -> metaclass :: metaclass_of_bases | None -> metaclass_of_bases in match metaclass_candidates with | [] -> Type.Primitive "type" | first :: candidates -> ( let order = self#full_order ~assumptions in let candidate = List.fold candidates ~init:first ~f:(TypeOrder.meet order) in match candidate with | Type.Bottom -> (* If we get Bottom here, we don't have a "most derived metaclass", so default to one. *) first | _ -> candidate) in UnannotatedGlobalEnvironment.ReadOnly.get_class_definition (ClassMetadataEnvironment.ReadOnly.unannotated_global_environment class_metadata_environment) ?dependency target >>| handle method constraints ~assumptions ~target ?parameters ~instantiated () = let parameters = match parameters with | None -> ClassHierarchyEnvironment.ReadOnly.variables (ClassMetadataEnvironment.ReadOnly.class_hierarchy_environment class_metadata_environment) ?dependency target >>| List.map ~f:Type.Variable.to_parameter |> Option.value ~default:[] | Some parameters -> parameters in if List.is_empty parameters then ConstraintsSet.Solution.empty else let right = Type.parametric target parameters in match instantiated, right with | Type.Primitive name, Parametric { name = right_name; _ } when String.equal name right_name -> (* TODO(T42259381) This special case is only necessary because constructor calls attributes with an "instantiated" type of a bare parametric, which will fill with Anys *) ConstraintsSet.Solution.empty | _ -> let order = self#full_order ~assumptions in TypeOrder.OrderedConstraintsSet.add ConstraintsSet.empty ~new_constraint:(LessOrEqual { left = instantiated; right }) ~order |> TypeOrder.OrderedConstraintsSet.solve ~order (* TODO(T39598018): error in this case somehow, something must be wrong *) |> Option.value ~default:ConstraintsSet.Solution.empty (* In general, python expressions can be self-referential. This resolution only checks literals and annotations found in the resolution map, without resolving expressions. *) method resolve_literal ~assumptions expression = let open Ast.Expression in let is_concrete_class class_type = class_type |> class_definition class_metadata_environment ~dependency >>| (fun { Node.value = { name; _ }; _ } -> Reference.show name) >>= ClassHierarchyEnvironment.ReadOnly.variables (class_hierarchy_environment class_metadata_environment) ?dependency ~default:(Some []) >>| List.is_empty in let fully_specified_type = function | { Node.value = Expression.Name name; _ } as annotation when is_simple_name name -> let class_type = self#parse_annotation ~assumptions annotation in if Type.is_none class_type || is_concrete_class class_type |> Option.value ~default:false then Some class_type else None | { Node.value = Call { callee = { value = Name (Name.Attribute { base = { Node.value = Name generic_name; _ }; attribute = "__getitem__"; special = true; }); _; }; _; }; _; } as annotation when is_simple_name generic_name -> let class_type = self#parse_annotation ~assumptions annotation in if is_concrete_class class_type >>| not |> Option.value ~default:false then Some class_type else None | _ -> None in let resolve_name expression = if has_identifier_base expression then match fully_specified_type expression with | Some annotation -> if Type.is_none annotation then Type.none else Type.meta annotation | None -> Type.Any else Type.Any in let order = self#full_order ~assumptions in match Node.value expression with | Expression.Await expression -> self#resolve_literal ~assumptions expression |> Type.awaitable_value |> Option.value ~default:Type.Any | BooleanOperator { BooleanOperator.left; right; _ } -> let annotation = TypeOrder.join order (self#resolve_literal ~assumptions left) (self#resolve_literal ~assumptions right) in if Type.is_concrete annotation then annotation else Type.Any | Call { callee; _ } -> ( match fully_specified_type expression with | Some annotation -> (* Literal generic type, e.g. global = List[int] *) Type.meta annotation | None -> (* Constructor on concrete class or fully specified generic, * e.g. global = GenericClass[int](x, y) or global = ConcreteClass(x) *) Option.value (fully_specified_type callee) ~default:Type.Any) | Constant Constant.NoneLiteral -> Type.Any | Constant (Constant.Complex _) -> Type.complex | Constant (Constant.False | Constant.True) -> Type.bool | Constant (Constant.Float _) -> Type.float | Constant (Constant.Integer _) -> Type.integer | Constant (Constant.String { StringLiteral.kind; _ }) -> ( match kind with | StringLiteral.Bytes -> Type.bytes | _ -> Type.string) | FormatString _ -> Type.string | Name name when is_simple_name name -> ( let reference = name_to_reference_exn name in let unannotated_global_environment = unannotated_global_environment class_metadata_environment in match UnannotatedGlobalEnvironment.ReadOnly.get_unannotated_global ?dependency unannotated_global_environment reference with | Some (UnannotatedGlobal.Define defines) -> let { decorated; _ } = List.map defines ~f:(fun { define; _ } -> define) |> List.partition_tf ~f:Define.Signature.is_overloaded_function |> fun (overloads, implementations) -> self#resolve_define ~assumptions ~implementation:(List.last implementations) ~overloads in Result.ok decorated |> Option.value ~default:Type.Any | _ -> resolve_name expression) | Name _ -> resolve_name expression | Dictionary { Dictionary.entries; keywords = [] } -> let key_annotation, value_annotation = let join_entry (key_annotation, value_annotation) { Dictionary.Entry.key; value } = ( TypeOrder.join order key_annotation (self#resolve_literal ~assumptions key), TypeOrder.join order value_annotation (self#resolve_literal ~assumptions value) ) in List.fold ~init:(Type.Bottom, Type.Bottom) ~f:join_entry entries in let key = if Type.is_concrete key_annotation then key_annotation else Type.Any in let value = if Type.is_concrete value_annotation then value_annotation else Type.Any in Type.dictionary ~key ~value | Dictionary _ -> Type.dictionary ~key:Type.Any ~value:Type.Any | List elements -> let parameter = let join sofar element = TypeOrder.join order sofar (self#resolve_literal ~assumptions element) in List.fold ~init:Type.Bottom ~f:join elements in if Type.is_concrete parameter then Type.list parameter else Type.list Type.Any | Set elements -> let parameter = let join sofar element = TypeOrder.join order sofar (self#resolve_literal ~assumptions element) in List.fold ~init:Type.Bottom ~f:join elements in if Type.is_concrete parameter then Type.set parameter else Type.set Type.Any | Ternary { Ternary.target; alternative; _ } -> let annotation = TypeOrder.join order (self#resolve_literal ~assumptions target) (self#resolve_literal ~assumptions alternative) in if Type.is_concrete annotation then annotation else Type.Any | Tuple elements -> Type.tuple (List.map elements ~f:(self#resolve_literal ~assumptions)) | Expression.Yield _ -> Type.yield Type.Any | _ -> Type.Any method resolve_define ~assumptions ~implementation ~overloads = let apply_decorator (index, decorator) argument = let make_error reason = Result.Error (AnnotatedAttribute.InvalidDecorator { index; reason }) in match Decorator.from_expression decorator with | None -> make_error CouldNotResolve | Some { Decorator.name; arguments } -> ( let name = Node.value name |> Reference.delocalize in let decorator = UnannotatedGlobalEnvironment.ReadOnly.resolve_exports (unannotated_global_environment class_metadata_environment) ?dependency name in let simple_decorator_name = match decorator with | Some (ModuleAttribute { from; name; remaining; _ }) -> Reference.create_from_list (Reference.as_list from @ name :: remaining) |> Reference.show | _ -> Reference.show name in match simple_decorator_name, argument with | ( ("click.decorators.pass_context" | "click.decorators.pass_obj"), Type.Callable callable ) -> (* Suppress caller/callee parameter matching by altering the click entry point to have a generic parameter list. *) let parameters = Type.Callable.Defined [ Type.Callable.Parameter.Variable (Concrete Type.Any); Type.Callable.Parameter.Keywords Type.Any; ] in Type.Callable (Type.Callable.map_parameters callable ~f:(fun _ -> parameters)) |> Result.return | name, Callable callable when String.equal name "contextlib.asynccontextmanager" || Set.mem Recognized.asyncio_contextmanager_decorators name -> let process_overload ({ Type.Callable.annotation; _ } as overload) = let joined = let order = self#full_order ~assumptions in try TypeOrder.join order annotation (Type.async_iterator Type.Bottom) with | ClassHierarchy.Untracked _ -> (* create_overload gets called when building the environment, which is unsound and can raise. *) Type.Any in if Type.is_async_iterator joined then { overload with Type.Callable.annotation = Type.parametric "typing.AsyncContextManager" [Single (Type.single_parameter joined)]; } else overload in let { Type.Callable.implementation = old_implementation; overloads = old_overloads; _; } = callable in Type.Callable { callable with implementation = process_overload old_implementation; overloads = List.map old_overloads ~f:process_overload; } |> Result.return | name, callable when String.is_suffix name ~suffix:".validator" -> (* TODO(T70606997): We should be type checking attr validators properly. *) Result.return callable | "contextlib.contextmanager", Callable callable -> let process_overload ({ Type.Callable.annotation; _ } as overload) = let joined = let order = self#full_order ~assumptions in try TypeOrder.join order annotation (Type.iterator Type.Bottom) with | ClassHierarchy.Untracked _ -> (* create_overload gets called when building the environment, which is unsound and can raise. *) Type.Any in if Type.is_iterator joined then { overload with Type.Callable.annotation = Type.parametric "contextlib._GeneratorContextManager" [Single (Type.single_parameter joined)]; } else overload in let { Type.Callable.implementation = old_implementation; overloads = old_overloads; _; } = callable in Type.Callable { callable with implementation = process_overload old_implementation; overloads = List.map old_overloads ~f:process_overload; } |> Result.return | name, argument when Set.mem Decorators.special_decorators name -> Decorators.apply ~argument ~name |> Result.return | name, _ when Set.mem Recognized.classmethod_decorators name -> (* TODO (T67024249): convert these to just normal stubs *) Type.parametric "typing.ClassMethod" [Single argument] |> Result.return | "staticmethod", _ -> Type.parametric "typing.StaticMethod" [Single argument] |> Result.return | _ -> ( let { decorator_assumptions; _ } = assumptions in if Assumptions.DecoratorAssumptions.not_a_decorator decorator_assumptions ~candidate:name then make_error CouldNotResolve else let assumptions = { assumptions with decorator_assumptions = Assumptions.DecoratorAssumptions.add decorator_assumptions ~assume_is_not_a_decorator:name; } in let resolve_attribute_access ?special_method base ~attribute_name = let access { Type.instantiated; accessed_through_class; class_name } = self#attribute ~assumptions ~transitive:true ~accessed_through_class ~include_generated_attributes:true ?special_method ~attribute_name ~instantiated class_name in let join_all = function | head :: tail -> let order = self#full_order ~assumptions in List.fold tail ~init:head ~f:(TypeOrder.join order) |> Option.some | [] -> None in Type.resolve_class base >>| List.map ~f:access >>= Option.all >>| List.map ~f:AnnotatedAttribute.annotation >>| List.map ~f:Annotation.annotation >>= join_all in let resolver = function | UnannotatedGlobalEnvironment.ResolvedReference.Module _ -> None | PlaceholderStub _ -> Some Type.Any | ModuleAttribute { from; name; remaining; _ } -> let rec resolve_remaining base ~remaining = match remaining with | [] -> Some base | attribute_name :: remaining -> resolve_attribute_access base ~attribute_name >>= resolve_remaining ~remaining in Reference.create_from_list [name] |> Reference.combine from |> self#global_annotation ~assumptions >>| (fun { Global.annotation = { annotation; _ }; _ } -> annotation) >>= resolve_remaining ~remaining in let extract_callable = function | Type.Callable callable -> Some callable | other -> ( match resolve_attribute_access other ~attribute_name:"__call__" ~special_method:true with | None -> None | Some (Type.Callable callable) -> Some callable | Some other -> ( (* We potentially need to go specifically two layers in order to support when name resolves to Type[X], which has a __call__ of its constructor that is itself a BoundMethod, which has a Callable __call__ *) match resolve_attribute_access other ~attribute_name:"__call__" ~special_method:true with | Some (Callable callable) -> Some callable | _ -> None)) in let apply_arguments_to_decorator_factory ~factory_callable ~arguments = let arguments = let resolve argument_index argument = let expression, kind = Ast.Expression.Call.Argument.unpack argument in let make_matched_argument resolved = { Argument.kind; expression = Some expression; resolved } in let error = AnnotatedAttribute.CouldNotResolveArgument { argument_index } in match expression with | { Node.value = Expression.Expression.Constant Expression.Constant.NoneLiteral; _; } -> Ok (make_matched_argument Type.NoneType) | { Node.value = Expression.Expression.Name name; _ } -> Expression.name_to_reference name >>| Reference.delocalize >>= UnannotatedGlobalEnvironment.ReadOnly.resolve_exports (unannotated_global_environment class_metadata_environment) ?dependency >>= resolver >>| make_matched_argument |> Result.of_option ~error: (AnnotatedAttribute.InvalidDecorator { index; reason = error }) | expression -> let resolved = self#resolve_literal ~assumptions expression in if Type.is_untyped resolved || Type.contains_unknown resolved then make_error error else Ok (make_matched_argument resolved) in List.mapi arguments ~f:resolve |> Result.all in let select arguments = self#signature_select ~assumptions ~resolve_with_locals:(fun ~locals:_ _ -> Type.Top) ~arguments ~callable:factory_callable ~self_argument:None ~skip_marking_escapees:false in let extract = function | SignatureSelectionTypes.Found { selected_return_annotation; _ } -> Result.Ok selected_return_annotation | NotFound { reason; _ } -> make_error (FactorySignatureSelectionFailed { reason; callable = factory_callable }) in Result.map arguments ~f:select |> Result.bind ~f:extract in let resolved_decorator = match decorator >>= resolver with | None -> make_error CouldNotResolve | Some Any -> Ok Type.Any | Some fetched -> ( match arguments with | None -> Ok fetched | Some arguments -> ( match extract_callable fetched with | None -> make_error (NonCallableDecoratorFactory fetched) | Some factory_callable -> apply_arguments_to_decorator_factory ~factory_callable ~arguments)) in match resolved_decorator with | Error error -> Result.Error error | Ok Any -> Ok Any | Ok resolved_decorator -> ( match extract_callable resolved_decorator with | None -> make_error (NonCallableDecorator resolved_decorator) | Some callable -> ( match self#signature_select ~assumptions ~resolve_with_locals:(fun ~locals:_ _ -> Type.object_primitive) ~arguments: [{ kind = Positional; expression = None; resolved = argument }] ~callable ~self_argument:None ~skip_marking_escapees:false with | SignatureSelectionTypes.Found { selected_return_annotation; _ } -> Ok selected_return_annotation | NotFound { reason; _ } -> make_error (ApplicationFailed { reason; callable }))))) in let parse = let parser = { AnnotatedCallable.parse_annotation = self#parse_annotation ~assumptions; parse_as_parameter_specification_instance_annotation = AliasEnvironment.ReadOnly.parse_as_parameter_specification_instance_annotation (alias_environment class_metadata_environment) ?dependency; } in let variables = ClassHierarchyEnvironment.ReadOnly.variables (ClassMetadataEnvironment.ReadOnly.class_hierarchy_environment class_metadata_environment) ?dependency in AnnotatedCallable.create_overload_without_applying_decorators ~parser ~variables in (*parsed, decorators |> List.rev |> List.fold ~init:parsed ~f:apply_decorator*) let kind = match implementation, overloads with | Some { Define.Signature.name; _ }, _ | _, { Define.Signature.name; _ } :: _ -> Type.Callable.Named name | None, [] -> (* Should never happen, but not worth crashing over *) Type.Callable.Anonymous in let undefined_overload = { Type.Callable.annotation = Type.Top; parameters = Type.Callable.Undefined } in let parsed_overloads, parsed_implementation, decorators = match overloads, implementation with | ({ decorators = head_decorators; _ } as overload) :: tail, _ -> let purify = let is_not_overload_decorator decorator = not (Ast.Statement.Define.Signature.is_overloaded_function { overload with decorators = [decorator] }) in List.filter ~f:is_not_overload_decorator in let enforce_equality ~parsed ~current sofar = let equal left right = Int.equal (Ast.Expression.location_insensitive_compare left right) 0 in if List.equal equal sofar (purify current) then Ok sofar else Error (AnnotatedAttribute.DifferingDecorators { offender = parsed }) in let reversed_parsed_overloads, decorators = let collect (reversed_parsed_overloads, decorators_sofar) ({ Define.Signature.decorators = current; _ } as overload) = let parsed = parse overload in ( parsed :: reversed_parsed_overloads, Result.bind decorators_sofar ~f:(enforce_equality ~parsed ~current) ) in List.fold tail ~f:collect ~init:([parse overload], Result.Ok (purify head_decorators)) in let parsed_implementation, decorators = match implementation with | Some ({ Define.Signature.decorators = current; _ } as implementation) -> let parsed = parse implementation in Some parsed, Result.bind decorators ~f:(enforce_equality ~parsed ~current) | None -> None, decorators in List.rev reversed_parsed_overloads, parsed_implementation, decorators | [], Some { decorators; _ } -> [], implementation >>| parse, Result.Ok decorators | [], None -> [], None, Ok [] in let undecorated_signature = { Type.Callable.implementation = parsed_implementation |> Option.value ~default:undefined_overload; overloads = parsed_overloads; kind; } in let decorated = let apply_decorators decorators = let applied = let apply_decorator sofar current = Result.bind sofar ~f:(apply_decorator current) in List.mapi decorators ~f:(fun index decorator -> index, decorator) |> List.rev |> List.fold ~init:(Ok (Type.Callable undecorated_signature)) ~f:apply_decorator in match applied with | Result.Ok (Type.Callable callable) when Type.Callable.equal { callable with kind } undecorated_signature -> (* Do some amateur taint analysis and assume that this is calling the original function under the hood *) Ok (Type.Callable { callable with kind }) | other -> other in Result.bind decorators ~f:apply_decorators in { undecorated_signature; decorated } method signature_select ~assumptions ~resolve_with_locals ~arguments ~callable:({ Type.Callable.implementation; overloads; _ } as callable) ~self_argument ~skip_marking_escapees = let order = self#full_order ~assumptions in let get_match signatures = let check_arguments_against_signature = SignatureSelection.check_arguments_against_signature ~order ~resolve_mutable_literals:(self#resolve_mutable_literals ~assumptions) ~resolve_with_locals ~callable ~self_argument ~arguments: (SignatureSelection.prepare_arguments_for_signature_selection ~self_argument arguments) in signatures |> List.concat_map ~f:check_arguments_against_signature |> SignatureSelection.find_closest_signature >>| SignatureSelection.instantiate_return_annotation ~skip_marking_escapees ~order |> Option.value ~default:(SignatureSelection.default_signature callable) in if List.is_empty overloads then get_match [implementation] else get_match overloads method resolve_mutable_literals ~assumptions ~resolve = WeakenMutableLiterals.weaken_mutable_literals resolve ~get_typed_dictionary:(self#get_typed_dictionary ~assumptions) ~comparator:(self#constraints_solution_exists ~assumptions) method constraints_solution_exists ~assumptions ~get_typed_dictionary_override ~left ~right = let ({ ConstraintsSet.get_typed_dictionary; _ } as order) = self#full_order ~assumptions in let order = { order with get_typed_dictionary = (fun annotation -> Option.first_some (get_typed_dictionary_override annotation) (get_typed_dictionary annotation)); } in TypeOrder.OrderedConstraintsSet.add ConstraintsSet.empty ~new_constraint:(LessOrEqual { left; right }) ~order |> TypeOrder.OrderedConstraintsSet.solve ~order |> Option.is_some method constructor ~assumptions class_name ~instantiated = let return_annotation = let generics = ClassHierarchyEnvironment.ReadOnly.variables (ClassMetadataEnvironment.ReadOnly.class_hierarchy_environment class_metadata_environment) ?dependency class_name >>| List.map ~f:Type.Variable.to_parameter |> Option.value ~default:[] in (* Tuples are special. *) if String.equal class_name "tuple" then match generics with | [Single tuple_variable] -> Type.Tuple (Type.OrderedTypes.create_unbounded_concatenation tuple_variable) | _ -> Type.Tuple (Type.OrderedTypes.create_unbounded_concatenation Type.Any) else let backup = Type.parametric class_name generics in match instantiated, generics with | _, [] -> instantiated | Type.Primitive instantiated_name, _ when String.equal instantiated_name class_name -> backup | Type.Parametric { parameters; name = instantiated_name }, generics when String.equal instantiated_name class_name && List.length parameters <> List.length generics -> backup | _ -> instantiated in let definitions = class_name :: ClassMetadataEnvironment.ReadOnly.successors class_metadata_environment ?dependency class_name in let definition_index parent = parent |> (fun class_annotation -> List.findi definitions ~f:(fun _ annotation -> Type.equal (Primitive annotation) class_annotation)) >>| fst |> Option.value ~default:Int.max_value in let signature_index_and_parent ~name = let signature, parent_name = match self#attribute ~assumptions ~transitive:true ~accessed_through_class:false ~include_generated_attributes:true ?special_method:None ?instantiated:(Some return_annotation) ~attribute_name:name class_name with | Some attribute -> ( AnnotatedAttribute.annotation attribute |> Annotation.annotation, AnnotatedAttribute.parent attribute ) | None -> Type.Top, class_name in signature, definition_index (Type.Primitive parent_name), parent_name in let constructor_signature, constructor_index, _ = signature_index_and_parent ~name:"__init__" in let new_signature, new_index, new_parent_name = let new_signature, new_index, new_parent_name = signature_index_and_parent ~name:"__new__" in ( Type.parametric "BoundMethod" [Single new_signature; Single (Type.meta instantiated)], new_index, new_parent_name ) in let signature, with_return = if new_index < constructor_index then (* If it is a Final class, we respect the return type of `__new__` and its overloads. *) if is_final_class class_metadata_environment ~dependency new_parent_name then new_signature, Fn.id else new_signature, Type.Callable.with_return_annotation ~annotation:return_annotation else constructor_signature, Type.Callable.with_return_annotation ~annotation:return_annotation in match signature with | Type.Callable callable -> Type.Callable (with_return callable) | Parametric { name = "BoundMethod"; parameters = [Single (Callable callable); Single self_type] } -> Parametric { name = "BoundMethod"; parameters = [Single (Callable (with_return callable)); Single self_type]; } | _ -> signature method global_annotation ~assumptions name = let process_unannotated_global global = let produce_assignment_global ~is_explicit ~is_final annotation = let original = if is_explicit then None else if (* Treat literal globals as having been explicitly annotated. *) Type.is_partially_typed annotation then Some Type.Top else None in Annotation.create_immutable ~final:is_final ~original annotation in match global with | UnannotatedGlobal.Define defines -> let { undecorated_signature; decorated } = List.map defines ~f:(fun { define; _ } -> define) |> List.partition_tf ~f:Define.Signature.is_overloaded_function |> fun (overloads, implementations) -> self#resolve_define ~implementation:(List.last implementations) ~overloads ~assumptions in let annotation = Result.ok decorated |> Option.value ~default:Type.Any |> Annotation.create_immutable in Some { Global.annotation; undecorated_signature = Some undecorated_signature; problem = Result.error decorated; } | SimpleAssign { explicit_annotation = None; value = { Node.value = Call { callee = { value = Name (Attribute { base = { Node.value = Name (Identifier "typing"); _ }; attribute = "TypeAlias"; _; }); _; }; _; }; _; }; target_location = location; } -> let location = Location.strip_module location in Ast.Expression.Expression.Name (Expression.create_name_from_reference ~location name) |> Node.create ~location |> self#parse_annotation ~validation:ValidatePrimitives ~assumptions |> Type.meta |> Annotation.create_immutable |> fun annotation -> Some { Global.annotation; undecorated_signature = None; problem = None } | SimpleAssign { explicit_annotation; value; _ } -> let explicit_annotation = explicit_annotation >>| self#parse_annotation ~assumptions >>= fun annotation -> Option.some_if (not (Type.is_type_alias annotation)) annotation in let annotation, is_explicit, is_final = match explicit_annotation with | None -> self#resolve_literal value ~assumptions, false, false | Some explicit -> ( match Type.final_value explicit with | `Ok final_value -> final_value, true, true | `NoParameter -> self#resolve_literal value ~assumptions, false, true | `NotFinal -> explicit, true, false) in produce_assignment_global ~is_explicit ~is_final annotation |> fun annotation -> Some { Global.annotation; undecorated_signature = None; problem = None } | TupleAssign { value; index; total_length; _ } -> let extracted = match self#resolve_literal ~assumptions value with | Type.Tuple (Concrete parameters) when List.length parameters = total_length -> List.nth parameters index (* This should always be Some, but I don't think its worth being fragile here *) |> Option.value ~default:Type.Top | Type.Tuple (Concatenation concatenation) -> Type.OrderedTypes.Concatenation.extract_sole_unbounded_annotation concatenation |> Option.value ~default:Type.Top | _ -> Type.Top in produce_assignment_global ~is_explicit:false ~is_final:false extracted |> fun annotation -> Some { Global.annotation; undecorated_signature = None; problem = None } | _ -> None in let class_lookup = Reference.show name |> UnannotatedGlobalEnvironment.ReadOnly.class_exists (unannotated_global_environment class_metadata_environment) ?dependency in if class_lookup then let primitive = Type.Primitive (Reference.show name) in Annotation.create_immutable (Type.meta primitive) |> fun annotation -> Some { Global.annotation; undecorated_signature = None; problem = None } else UnannotatedGlobalEnvironment.ReadOnly.get_unannotated_global (unannotated_global_environment class_metadata_environment) ?dependency name >>= fun global -> let timer = Timer.start () in let result = process_unannotated_global global in Statistics.performance ~flush:false ~randomly_log_every:500 ~section:`Check ~name:"SingleGlobalTypeCheck" ~timer ~normals:["name", Reference.show name; "request kind", "SingleGlobalTypeCheck"] (); result end let empty_assumptions = { protocol_assumptions = ProtocolAssumptions.empty; callable_assumptions = CallableAssumptions.empty; decorator_assumptions = DecoratorAssumptions.empty; } module ParseAnnotationCache = struct module Cache = ManagedCache.Make (struct module PreviousEnvironment = ClassMetadataEnvironment module Key = SharedMemoryKeys.ParseAnnotationKey module Value = struct type t = Type.t [@@deriving compare] let prefix = Prefix.make () let description = "parse annotation" let unmarshall value = Marshal.from_string value 0 end module KeySet = SharedMemoryKeys.ParseAnnotationKey.Set module HashableKey = SharedMemoryKeys.ParseAnnotationKey let lazy_incremental = false let produce_value class_metadata_environment { SharedMemoryKeys.ParseAnnotationKey.validation; expression } ~dependency = let implementation = new base class_metadata_environment dependency in implementation#parse_annotation ~assumptions:empty_assumptions ~validation expression let filter_upstream_dependency = function | SharedMemoryKeys.ParseAnnotation key -> Some key | _ -> None let trigger_to_dependency key = SharedMemoryKeys.ParseAnnotation key end) include Cache module ReadOnly = struct include Cache.ReadOnly class with_cached_parse_annotation dependency read_only = object inherit base (upstream_environment read_only) dependency method! parse_annotation ~assumptions:_ ?(validation = SharedMemoryKeys.ParseAnnotationKey.ValidatePrimitivesAndTypeParameters) expression = get read_only ?dependency { SharedMemoryKeys.ParseAnnotationKey.validation; expression } end end end module MetaclassCache = struct module Cache = ManagedCache.Make (struct module PreviousEnvironment = ParseAnnotationCache module Key = struct type t = string [@@deriving compare, show, sexp, hash] let to_string = show let compare = compare type out = t let from_string = Fn.id end module Value = struct type t = Type.t option [@@deriving compare] let prefix = Prefix.make () let description = "metaclasses" let unmarshall value = Marshal.from_string value 0 end module KeySet = String.Set module HashableKey = String let lazy_incremental = false let produce_value parse_annotation_cache key ~dependency = let implementation_with_cached_parse_annotation = new ParseAnnotationCache.ReadOnly.with_cached_parse_annotation dependency parse_annotation_cache in implementation_with_cached_parse_annotation#metaclass key ~assumptions:empty_assumptions let filter_upstream_dependency = function | SharedMemoryKeys.Metaclass key -> Some key | _ -> None let trigger_to_dependency key = SharedMemoryKeys.Metaclass key end) include Cache module ReadOnly = struct include Cache.ReadOnly class with_parse_annotation_and_metaclass_caches dependency read_only = object inherit ParseAnnotationCache.ReadOnly.with_cached_parse_annotation dependency (upstream_environment read_only) method! metaclass ~assumptions:_ key = get read_only ?dependency key end end end module AttributeCache = struct module Cache = ManagedCache.Make (struct module PreviousEnvironment = MetaclassCache module Key = SharedMemoryKeys.AttributeTableKey module Value = struct type t = UninstantiatedAttributeTable.t option [@@deriving compare] let prefix = Prefix.make () let description = "attributes" let unmarshall value = Marshal.from_string value 0 end module KeySet = SharedMemoryKeys.AttributeTableKey.Set module HashableKey = SharedMemoryKeys.AttributeTableKey let lazy_incremental = true let produce_value metaclass_cache { SharedMemoryKeys.AttributeTableKey.include_generated_attributes; accessed_via_metaclass; name; } ~dependency = let implementation_with_cached_parse_annotation = new MetaclassCache.ReadOnly.with_parse_annotation_and_metaclass_caches dependency metaclass_cache in implementation_with_cached_parse_annotation#single_uninstantiated_attribute_table ~include_generated_attributes ~accessed_via_metaclass ~assumptions:empty_assumptions name let filter_upstream_dependency = function | SharedMemoryKeys.AttributeTable key -> Some key | _ -> None let trigger_to_dependency key = SharedMemoryKeys.AttributeTable key end) include Cache module ReadOnly = struct include Cache.ReadOnly let metaclass_cache = upstream_environment let cached_single_uninstantiated_attribute_table read_only dependency ~include_generated_attributes ~accessed_via_metaclass name = get read_only ?dependency { SharedMemoryKeys.AttributeTableKey.include_generated_attributes; accessed_via_metaclass; name; } class with_parse_annotation_metaclass_and_attribute_caches dependency read_only = object inherit MetaclassCache.ReadOnly.with_parse_annotation_and_metaclass_caches dependency (metaclass_cache read_only) method! single_uninstantiated_attribute_table ~assumptions:_ = cached_single_uninstantiated_attribute_table read_only dependency end end end module GlobalAnnotationCache = struct module Cache = Environment.EnvironmentTable.WithCache (struct let legacy_invalidated_keys upstream = let previous_classes = UnannotatedGlobalEnvironment.UpdateResult.previous_classes upstream |> Type.Primitive.Set.to_list |> List.map ~f:Reference.create in let previous_unannotated_globals = UnannotatedGlobalEnvironment.UpdateResult.previous_unannotated_globals upstream in List.fold ~init:previous_unannotated_globals ~f:Set.add previous_classes let all_keys = UnannotatedGlobalEnvironment.ReadOnly.all_unannotated_globals let show_key = Reference.show module PreviousEnvironment = AttributeCache module Key = SharedMemoryKeys.ReferenceKey module Value = struct type t = Global.t option let prefix = Prefix.make () let description = "Global" let unmarshall value = Marshal.from_string value 0 let compare = Option.compare Global.compare end type trigger = Reference.t [@@deriving sexp, compare] let convert_trigger = Fn.id let key_to_trigger = Fn.id module TriggerSet = Reference.Set let lazy_incremental = false let produce_value attribute_cache key ~dependency = let implementation_with_preceding_caches = new AttributeCache.ReadOnly.with_parse_annotation_metaclass_and_attribute_caches dependency attribute_cache in implementation_with_preceding_caches#global_annotation ~assumptions:empty_assumptions key let filter_upstream_dependency = function | SharedMemoryKeys.AnnotateGlobal name -> Some name | _ -> None let trigger_to_dependency name = SharedMemoryKeys.AnnotateGlobal name let serialize_value = function | Some annotation -> Global.sexp_of_t annotation |> Sexp.to_string | None -> "None" let equal_value = Option.equal [%compare.equal: Global.t] end) include Cache module ReadOnly = struct include Cache.ReadOnly let attribute_cache = upstream_environment class with_all_caches dependency read_only = object inherit AttributeCache.ReadOnly.with_parse_annotation_metaclass_and_attribute_caches dependency (attribute_cache read_only) method! global_annotation ~assumptions:_ = get read_only ?dependency end end end module PreviousEnvironment = ClassMetadataEnvironment include GlobalAnnotationCache module ReadOnly = struct include GlobalAnnotationCache.ReadOnly let attribute_cache = upstream_environment let metaclass_cache read_only = attribute_cache read_only |> AttributeCache.ReadOnly.upstream_environment let parse_annotation_cache read_only = metaclass_cache read_only |> MetaclassCache.ReadOnly.upstream_environment let class_metadata_environment read_only = ParseAnnotationCache.ReadOnly.upstream_environment (parse_annotation_cache read_only) class with_uninstantiated_attributes_cache dependency read_only = object inherit base (class_metadata_environment read_only) dependency method! single_uninstantiated_attribute_table ~assumptions:_ = AttributeCache.ReadOnly.cached_single_uninstantiated_attribute_table (attribute_cache read_only) dependency end let add_all_caches_and_empty_assumptions f read_only ?dependency = new GlobalAnnotationCache.ReadOnly.with_all_caches dependency read_only |> f |> fun method_ -> method_ ~assumptions:empty_assumptions let instantiate_attribute = add_all_caches_and_empty_assumptions (fun o -> o#instantiate_attribute ?apply_descriptors:None) let attribute = add_all_caches_and_empty_assumptions (fun o -> o#attribute ?apply_descriptors:None) let all_attributes = add_all_caches_and_empty_assumptions (fun o -> o#all_attributes) let attribute_names = add_all_caches_and_empty_assumptions (fun o -> o#attribute_names) let check_invalid_type_parameters = add_all_caches_and_empty_assumptions (fun o -> o#check_invalid_type_parameters ~replace_unbound_parameters_with_any:true) let parse_annotation read_only ?dependency = let attributes_cached_but_not_annotations = new with_uninstantiated_attributes_cache dependency read_only in attributes_cached_but_not_annotations#parse_annotation ~assumptions:empty_assumptions let metaclass = add_all_caches_and_empty_assumptions (fun o -> o#metaclass) let constraints = add_all_caches_and_empty_assumptions (fun o -> o#constraints) let resolve_literal = add_all_caches_and_empty_assumptions (fun o -> o#resolve_literal) let resolve_define = add_all_caches_and_empty_assumptions (fun o -> o#resolve_define) let resolve_mutable_literals = add_all_caches_and_empty_assumptions (fun o -> o#resolve_mutable_literals) let constraints_solution_exists = add_all_caches_and_empty_assumptions (fun o -> o#constraints_solution_exists) let full_order ?dependency read_only = let implementation = new with_all_caches dependency read_only in implementation#full_order ~assumptions:empty_assumptions let get_typed_dictionary = add_all_caches_and_empty_assumptions (fun o -> o#get_typed_dictionary) let signature_select = add_all_caches_and_empty_assumptions (fun o -> o#signature_select ~skip_marking_escapees:false) let get_global = add_all_caches_and_empty_assumptions (fun o -> o#global_annotation) end module AttributeReadOnly = ReadOnly include TypeParameterValidationTypes