def create_clarify_bias_job()

in code/workflow/implementations/autopilot/bp_bias_analysis_stage.py [0:0]

• 48 lines of code
• 1 McCabe index (conditional complexity)

def create_clarify_bias_job(event) :
    
    role = event["Input"]["Payload"]["security-config"]["iam_role"]
    ws_params = event["Input"]["Payload"]["workspace-config"]
    data_params = event["Input"]["Payload"]["data-config"]
    model_params = event["Input"]["Payload"]["model-config"]
    automl_params = event["Input"]["Payload"]["automl-config"]
    bias_analysis_params = event["Input"]["Payload"]["bias-analysis-config"]
    
    # This is a temporary workaround. The bias detection job behaves differently when
    # files are split. Remove when the bug is fixed.
    ##############################################################################################
    merged_files_dst = "s3://{}/{}/{}".format(  ws_params["s3_bucket"],
                                                ws_params["s3_prefix"],
                                                "data/merged")
    
    input_path = create_merged_dataset(automl_params["data_uri"], merged_files_dst, automl_params["target_name"])
    ################################## End Workaround ############################################

    session = Session()
    clarify_processor = clarify.SageMakerClarifyProcessor(  role=role,
                                                            instance_count=bias_analysis_params["instance_count"],
                                                            instance_type=bias_analysis_params["instance_type"],
                                                            sagemaker_session=session)
                                                          
    output_uri = "s3://{}/{}/{}".format(ws_params["s3_bucket"],
                                        ws_params["s3_prefix"],
                                        bias_analysis_params["output_prefix"])
    
    bias_data_config = clarify.DataConfig(  s3_data_input_path=input_path,
                                            s3_output_path=output_uri,
                                            label=automl_params["target_name"],
                                            headers=get_data_columns(automl_params["data_uri"]),
                                            dataset_type='text/csv')
        
    model_config = clarify.ModelConfig( model_name=model_params["model_name"],
                                        instance_type=model_params["instance_type"],
                                        instance_count=model_params["instance_count"],
                                        accept_type='text/csv',
                                        content_type = 'text/csv')
        
    pred_params = bias_analysis_params["prediction-config"]
    predictions_config = clarify.ModelPredictedLabelConfig( label= pred_params["label"],
                                                            probability= pred_params["probability"],
                                                            probability_threshold=pred_params["probability_threshold"],
                                                            label_headers=pred_params["label_headers"])
        
    bias_params = bias_analysis_params["bias-config"]
    bias_config = clarify.BiasConfig(   label_values_or_threshold=bias_params["label_values_or_threshold"],
                                        facet_name=bias_params["facet_name"],
                                        facet_values_or_threshold=bias_params["facet_values_or_threshold"],
                                        group_name=bias_params["group_name"])   
                                        
    clarify_processor.run_bias( job_name = bias_analysis_params["job_name"],
                                data_config=bias_data_config,
                                bias_config=bias_config,
                                model_config=model_config,
                                model_predicted_label_config=predictions_config,
                                pre_training_methods='all',
                                post_training_methods='all',
                                wait=False,
                                logs=False)