def extract_log_probability()

in src/fmeval/model_runners/extractors/jumpstart_extractor.py [0:0]

• 19 lines of code
• 4 McCabe index (conditional complexity)

    def extract_log_probability(self, data: Union[List, Dict], num_records: int = 1) -> float:
        """
        Extracts the log probability from the JumpStartModel response. This value is not provided by all JS text models.

        :param data: The model response from the JumpStart Model
        :param num_records: The number of records in the model response. Must be 1.
        """
        assert num_records == 1, "Jumpstart extractor does not support batch requests"
        util.require(
            self._log_prob_compiler, f"Model {self._model_id} does not include input log probabilities in it's response"
        )
        try:
            assert (
                self._log_prob_compiler
            ), f"Model {self._model_id} does not include input log probabilities in it's response"
            log_probs = self._log_prob_compiler.search(data)
            if log_probs is None and not isinstance(log_probs, list):
                raise EvalAlgorithmClientError(
                    f"Unable to extract log probability from Jumpstart model: {self._model_id}"
                )
            return sum(log_probs)
        except ValueError as e:
            raise EvalAlgorithmClientError(
                f"Unable to extract log probability from Jumpstart model: {self._model_id}", e
            )