in src/dfcx_scrapi/tools/evaluations.py [0:0]
def run_detect_intent_queries(self, df: pd.DataFrame) -> pd.DataFrame:
for index, row in tqdm(df.iterrows(), total=df.shape[0]):
data = {}
if row["action_id"] == 1:
self.session_id = self.sessions_client.build_session_id(
self.agent_id)
data["session_id"] = self.session_id
data["agent_id"] = self.agent_id
else:
data["session_id"] = self.session_id
data["agent_id"] = self.agent_id
# If the incoming dataset has an empty value in the row, skip it
# this is because we build the incoming dataset with multi-row
# actions to be able to evaluate `inner-loop` tasks
if row["action_type"] != "User Utterance":
continue
session_parameters = None
if "session_parameters" in row:
session_parameters = self.str_to_dict(row["session_parameters"])
res = self.sessions_client.detect_intent(
agent_id=self.agent_id,
session_id=self.session_id,
text=row["action_input"],
parameters=session_parameters
)
# Add data to the existing row
df.loc[index, ["session_id", "agent_id"]] = [
data["session_id"],
data["agent_id"],
]
text_res = self.ar._extract_text(res)
# Handle Agent Responses
if row["utterance_pair"] != "":
utterance_idx = int(row["utterance_pair"])
df.loc[utterance_idx, ["agent_response"]] = [text_res]
else:
# collect the data for inserting later
self.unexpected_rows.append(
{
"session_id": data["session_id"],
"agent_id": data["agent_id"],
"action_type": "UNEXPECTED Agent Response",
"index": index,
"column": "agent_response",
"data": text_res
}
)
# Handle Playbook Invocations
playbook_responses = (
self.sessions_client.collect_playbook_responses(res)
)
if len(playbook_responses) > 0:
df = self.process_playbook_invocations(
playbook_responses, index, row, df
)
# Handle Flow Invocations
flow_responses = self.sessions_client.collect_flow_responses(res)
if len(flow_responses) > 0:
df = self.process_flow_invocations(
flow_responses, index, row, df
)
# Handle Tool Invocations
if "tool_call_quality" in self.user_input_metrics:
tool_responses = (
self.sessions_client.collect_tool_responses(res)
)
if tool_responses: # Only call if not empty
df = self.process_tool_invocations(
tool_responses,
index,
row,
df
)
return df