cli/foundation-models/system/distillation/conversation/distillation_conversation.yaml

type: distillation

name: "Distillation-conversation-llama"
description: "Distill student model using a teacher model"
experiment_name: "Distillation-Conversation"

# Data Generation Properties
data_generation_type: label_generation
data_generation_task_type: conversation

# Input data
training_data:
  type: uri_file
  path: ./train_conversation.jsonl
validation_data:
  type: uri_file
  path: ./validation_conversation.jsonl

# Teacher model serverless endpoint information
# REPLACE WITH YOUR ENDPOINT INFORMATION
teacher_model_endpoint_connection: 
  type: serverless
  name: Meta-Llama-3-1-405B-Instruct-vkn
  endpoint: https://Meta-Llama-3-1-405B-Instruct-vkn.westus3.models.ai.azure.com/chat/completions
  api_key: EXAMPLE_API_KEY

# Model ID
student_model: azureml://registries/azureml-meta/models/Meta-Llama-3.1-8B-Instruct/versions/2

# Output distilled model
outputs:
  registered_model:
    type: mlflow_model
    name: llama-conversation-distilled


# Teacher model related properties (OPTIONAL)
teacher_model_settings:
  inference_parameters:
    temperature: 0.1
    max_tokens: 100
    top_p: 0.95
  endpoint_request_settings:
    request_batch_size: 10
    min_endpoint_success_ratio: 0.7

# For finetuning (OPTIONAL)
hyperparameters:
  learning_rate_multiplier: "0.2"
  n_epochs: "5"
  batch_size: "2"

# Resource for Data Generation Step (OPTIONAL)
resources:
  instance_type: Standard_D2_v2