in use-cases/model-fine-tuning-pipeline/model-eval/src/validate_fine_tuned_model.py [0:0]
def __init__(self): # Constructor
self.api_endpoint = os.environ["ENDPOINT"]
self.model_name = os.environ["MODEL_PATH"]
self.output_file = os.environ["PREDICTIONS_FILE"]
self.gcs_bucket = os.environ["BUCKET"]
self.dataset_output_path = os.environ["DATASET_OUTPUT_PATH"]
training_dataset = load_from_disk(
f"gs://{self.gcs_bucket}/{self.dataset_output_path}/training"
)
validation_dataset = load_from_disk(
f"gs://{self.gcs_bucket}/{self.dataset_output_path}/validation"
)
test_dataset = load_from_disk(
f"gs://{self.gcs_bucket}/{self.dataset_output_path}/test"
)
# convert output to pandas dataframe
self.training_df = training_dataset.to_pandas()
self.validation_df = validation_dataset.to_pandas()
self.test_df = test_dataset.to_pandas()
# Concatenate vertically (stack rows)
self.df = pd.concat([self.validation_df, self.test_df], axis=0)
self.df.reset_index(drop=True, inplace=True)