Path	Lines of Code
src/alpaca_eval/__init__.py	4
src/alpaca_eval/analyze.py	310
src/alpaca_eval/annotators/__init__.py	2
src/alpaca_eval/annotators/base.py	454
src/alpaca_eval/annotators/pairwise_evaluator.py	199
src/alpaca_eval/completion_parsers.py	155
src/alpaca_eval/constants.py	211
src/alpaca_eval/decoders/__init__.py	82
src/alpaca_eval/decoders/anthropic.py	99
src/alpaca_eval/decoders/bedrock_anthropic.py	85
src/alpaca_eval/decoders/cohere.py	69
src/alpaca_eval/decoders/google.py	99
src/alpaca_eval/decoders/huggingface_api.py	71
src/alpaca_eval/decoders/huggingface_local.py	106
src/alpaca_eval/decoders/jinachat.py	57
src/alpaca_eval/decoders/openai.py	251
src/alpaca_eval/decoders/replicate.py	52
src/alpaca_eval/decoders/vllm_local.py	48
src/alpaca_eval/evaluators_configs/alpaca_eval_clf_cot_gpt4_turbo/configs.yaml	19
src/alpaca_eval/evaluators_configs/alpaca_eval_clf_gpt4_turbo/configs.yaml	16
src/alpaca_eval/evaluators_configs/alpaca_eval_cot_gpt4_turbo_fn/configs.yaml	41
src/alpaca_eval/evaluators_configs/alpaca_eval_gpt4/configs.yaml	9
src/alpaca_eval/evaluators_configs/alpaca_eval_gpt4_0314/configs.yaml	9
src/alpaca_eval/evaluators_configs/alpaca_eval_gpt4_0613/configs.yaml	9
src/alpaca_eval/evaluators_configs/alpaca_eval_gpt4_fn/configs.yaml	35
src/alpaca_eval/evaluators_configs/alpaca_eval_gpt4_turbo_fn/configs.yaml	36
src/alpaca_eval/evaluators_configs/alpaca_farm/configs.yaml	166
src/alpaca_eval/evaluators_configs/alpaca_farm_greedy_gpt4/configs.yaml	12
src/alpaca_eval/evaluators_configs/aviary_gpt4/configs.yaml	10
src/alpaca_eval/evaluators_configs/bedrock_claude/configs.yaml	12
src/alpaca_eval/evaluators_configs/bedrock_claude_2/configs.yaml	12
src/alpaca_eval/evaluators_configs/chatgpt/configs.yaml	12
src/alpaca_eval/evaluators_configs/chatgpt_fn/configs.yaml	24
src/alpaca_eval/evaluators_configs/claude/configs.yaml	12
src/alpaca_eval/evaluators_configs/claude_2/configs.yaml	12
src/alpaca_eval/evaluators_configs/claude_ranking/configs.yaml	9
src/alpaca_eval/evaluators_configs/cohere/configs.yaml	11
src/alpaca_eval/evaluators_configs/gpt35_turbo_instruct/configs.yaml	12
src/alpaca_eval/evaluators_configs/gpt4/configs.yaml	12
src/alpaca_eval/evaluators_configs/gpt4_turbo/configs.yaml	12
src/alpaca_eval/evaluators_configs/gpt4_turbo_clf/configs.yaml	16
src/alpaca_eval/evaluators_configs/gpt4_turbo_cot_clf/configs.yaml	17
src/alpaca_eval/evaluators_configs/gpt4_turbo_cot_logprob/configs.yaml	17
src/alpaca_eval/evaluators_configs/gpt4_turbo_logprob/configs.yaml	16
src/alpaca_eval/evaluators_configs/guanaco_33b/configs.yaml	11
src/alpaca_eval/evaluators_configs/improved_aviary_gpt4/configs.yaml	9
src/alpaca_eval/evaluators_configs/improved_lmsys_gpt4/configs.yaml	9
src/alpaca_eval/evaluators_configs/lmsys_gpt4/configs.yaml	10
src/alpaca_eval/evaluators_configs/oasst_pythia_12b/configs.yaml	11
src/alpaca_eval/evaluators_configs/text_davinci_003/configs.yaml	12
src/alpaca_eval/evaluators_configs/weighted_alpaca_eval_cot_gpt4_turbo/configs.yaml	17
src/alpaca_eval/evaluators_configs/weighted_alpaca_eval_gpt4_turbo/configs.yaml	16
src/alpaca_eval/main.py	327
src/alpaca_eval/metrics.py	113
src/alpaca_eval/models_configs/LMCocktail-10.7B-v1/configs.yaml	16
src/alpaca_eval/models_configs/Mistral-7B-Instruct-v0.2/configs.yaml	12
src/alpaca_eval/models_configs/Mistral-7B-ReMax-v0.1/configs.yaml	13
src/alpaca_eval/models_configs/Mixtral-8x7B-Instruct-v0.1/configs.yaml	12
src/alpaca_eval/models_configs/Mixtral-8x7B-Instruct-v0.1_concise/configs.yaml	12
src/alpaca_eval/models_configs/Mixtral-8x7B-Instruct-v0.1_verbose/configs.yaml	12
src/alpaca_eval/models_configs/OpenHermes-2.5-Mistral-7B/configs.yaml	12
src/alpaca_eval/models_configs/Qwen-14B-Chat/configs.yaml	16
src/alpaca_eval/models_configs/Qwen1.5-72B-Chat/configs.yaml	15
src/alpaca_eval/models_configs/Snorkel-Mistral-PairRM-DPO-best-of-16/configs.yaml	12
src/alpaca_eval/models_configs/Snorkel-Mistral-PairRM-DPO/configs.yaml	12
src/alpaca_eval/models_configs/Starling-LM-7B-alpha/configs.yaml	13
src/alpaca_eval/models_configs/Yi-34B-Chat-Verified/configs.yaml	14
src/alpaca_eval/models_configs/Yi-34B-Chat/configs.yaml	16
src/alpaca_eval/models_configs/airoboros-33b/configs.yaml	13
src/alpaca_eval/models_configs/airoboros-65b/configs.yaml	13
src/alpaca_eval/models_configs/alpaca-7b-neft/configs.yaml	6
src/alpaca_eval/models_configs/alpaca-7b/configs.yaml	13
src/alpaca_eval/models_configs/alpaca-7b_concise/configs.yaml	14
src/alpaca_eval/models_configs/alpaca-7b_verbose/configs.yaml	14
src/alpaca_eval/models_configs/alpaca-farm-ppo-human/configs.yaml	13
src/alpaca_eval/models_configs/alpaca-farm-ppo-sim-gpt4-20k/configs.yaml	13
src/alpaca_eval/models_configs/baichuan-13b-chat/configs.yaml	13
src/alpaca_eval/models_configs/baize-v2-13b/configs.yaml	13
src/alpaca_eval/models_configs/baize-v2-7b/configs.yaml	13
src/alpaca_eval/models_configs/bedrock_claude/configs.yaml	7
src/alpaca_eval/models_configs/bedrock_claude_2/configs.yaml	7
src/alpaca_eval/models_configs/causallm-14b/configs.yaml	14
src/alpaca_eval/models_configs/chatglm2-6b/configs.yaml	13
src/alpaca_eval/models_configs/claude-2.1/configs.yaml	7
src/alpaca_eval/models_configs/claude-2.1_concise/configs.yaml	7
src/alpaca_eval/models_configs/claude-2.1_verbose/configs.yaml	7
src/alpaca_eval/models_configs/claude-2/configs.yaml	7
src/alpaca_eval/models_configs/claude-instant-1.2/configs.yaml	7
src/alpaca_eval/models_configs/claude/configs.yaml	7
src/alpaca_eval/models_configs/claude2-alpaca-13b/configs.yaml	13
src/alpaca_eval/models_configs/cohere/configs.yaml	7
src/alpaca_eval/models_configs/cut-13b/configs.yaml	13
src/alpaca_eval/models_configs/deepseek-llm-67b-chat/configs.yaml	13
src/alpaca_eval/models_configs/deita-7b-v1.0/configs.yaml	13
src/alpaca_eval/models_configs/dolphin-2.2.1-mistral-7b/configs.yaml	13
src/alpaca_eval/models_configs/evo-7b/configs.yaml	12
src/alpaca_eval/models_configs/evo-v2-7b/configs.yaml	13
src/alpaca_eval/models_configs/falcon-40b-instruct/configs.yaml	14
src/alpaca_eval/models_configs/falcon-7b-instruct/configs.yaml	14
src/alpaca_eval/models_configs/gemini-pro/configs.yaml	8
src/alpaca_eval/models_configs/gemma-2b-it/configs.yaml	12
src/alpaca_eval/models_configs/gemma-7b-it/configs.yaml	12
src/alpaca_eval/models_configs/gpt-3.5-turbo-0301/configs.yaml	7
src/alpaca_eval/models_configs/gpt-3.5-turbo-0613/configs.yaml	7
src/alpaca_eval/models_configs/gpt-3.5-turbo-1106/configs.yaml	7
src/alpaca_eval/models_configs/gpt-3.5-turbo-1106_concise/configs.yaml	7
src/alpaca_eval/models_configs/gpt-3.5-turbo-1106_verbose/configs.yaml	7
src/alpaca_eval/models_configs/gpt-3.5-turbo-16k-0613/configs.yaml	7
src/alpaca_eval/models_configs/gpt35_turbo_instruct/configs.yaml	7
src/alpaca_eval/models_configs/gpt4/configs.yaml	7
src/alpaca_eval/models_configs/gpt4_0314/configs.yaml	7
src/alpaca_eval/models_configs/gpt4_0613/configs.yaml	7
src/alpaca_eval/models_configs/gpt4_0613_concise/configs.yaml	7
src/alpaca_eval/models_configs/gpt4_0613_verbose/configs.yaml	7
src/alpaca_eval/models_configs/gpt4_1106_preview/configs.yaml	7
src/alpaca_eval/models_configs/gpt4_1106_preview_concise/configs.yaml	7
src/alpaca_eval/models_configs/gpt4_1106_preview_verbose/configs.yaml	7
src/alpaca_eval/models_configs/gpt4_gamed/configs.yaml	7
src/alpaca_eval/models_configs/gpt4_turbo/configs.yaml	7
src/alpaca_eval/models_configs/guanaco-13b/configs.yaml	14
src/alpaca_eval/models_configs/guanaco-33b-api/configs.yaml	9
src/alpaca_eval/models_configs/guanaco-33b/configs.yaml	14
src/alpaca_eval/models_configs/guanaco-65b/configs.yaml	14
src/alpaca_eval/models_configs/guanaco-7b/configs.yaml	14
src/alpaca_eval/models_configs/humpback-llama-65b/configs.yaml	10
src/alpaca_eval/models_configs/humpback-llama2-70b/configs.yaml	10
src/alpaca_eval/models_configs/internlm2-chat-20b-ppo/configs.yaml	15
src/alpaca_eval/models_configs/jina-chat/configs.yaml	5
src/alpaca_eval/models_configs/llama-2-13b-chat-hf/configs.yaml	13
src/alpaca_eval/models_configs/llama-2-70b-chat-hf/configs.yaml	10
src/alpaca_eval/models_configs/llama-2-7b-chat-hf/configs.yaml	13
src/alpaca_eval/models_configs/llama-2-chat-7b-evol70k-neft/configs.yaml	6
src/alpaca_eval/models_configs/minichat-1.5-3b/configs.yaml	13
src/alpaca_eval/models_configs/minichat-3b/configs.yaml	13
src/alpaca_eval/models_configs/minotaur-13b/configs.yaml	13
src/alpaca_eval/models_configs/mistral-medium/configs.yaml	12
src/alpaca_eval/models_configs/nous-hermes-13b/configs.yaml	13
src/alpaca_eval/models_configs/oasst-rlhf-llama-33b/configs.yaml	13
src/alpaca_eval/models_configs/oasst-sft-llama-33b/configs.yaml	13
src/alpaca_eval/models_configs/oasst-sft-pythia-12b/configs.yaml	13
src/alpaca_eval/models_configs/openbuddy-falcon-40b-v9/configs.yaml	13
src/alpaca_eval/models_configs/openbuddy-falcon-7b-v6/configs.yaml	13
src/alpaca_eval/models_configs/openbuddy-llama-30b-v7.1/configs.yaml	13
src/alpaca_eval/models_configs/openbuddy-llama-65b-v8/configs.yaml	13
src/alpaca_eval/models_configs/openbuddy-llama2-13b-v11.1/configs.yaml	15
src/alpaca_eval/models_configs/openbuddy-llama2-70b-v10.1/configs.yaml	15
src/alpaca_eval/models_configs/openchat-13b/configs.yaml	13
src/alpaca_eval/models_configs/openchat-v2-13b/configs.yaml	14
src/alpaca_eval/models_configs/openchat-v2-w-13b/configs.yaml	14
src/alpaca_eval/models_configs/openchat-v3.1-13b/configs.yaml	14
src/alpaca_eval/models_configs/openchat8192-13b/configs.yaml	13
src/alpaca_eval/models_configs/opencoderplus-15b/configs.yaml	13
src/alpaca_eval/models_configs/pairrm-Yi-34B-Chat/configs.yaml	17
src/alpaca_eval/models_configs/pairrm-tulu-2-13b/configs.yaml	16
src/alpaca_eval/models_configs/pairrm-tulu-2-70b/configs.yaml	16
src/alpaca_eval/models_configs/pairrm-zephyr-7b-beta/configs.yaml	14
src/alpaca_eval/models_configs/phi-2-dpo/configs.yaml	14
src/alpaca_eval/models_configs/phi-2-sft/configs.yaml	14
src/alpaca_eval/models_configs/phi-2/configs.yaml	16
src/alpaca_eval/models_configs/platolm-7b/configs.yaml	13
src/alpaca_eval/models_configs/pythia-12b-mix-sft/configs.yaml	13
src/alpaca_eval/models_configs/recycled-wizardlm-7b-v1.0/configs.yaml	13
src/alpaca_eval/models_configs/recycled-wizardlm-7b-v2.0/configs.yaml	13
src/alpaca_eval/models_configs/text_davinci_001/configs.yaml	7
src/alpaca_eval/models_configs/text_davinci_003/configs.yaml	7
src/alpaca_eval/models_configs/tulu-2-dpo-13b/configs.yaml	15
src/alpaca_eval/models_configs/tulu-2-dpo-70b/configs.yaml	15
src/alpaca_eval/models_configs/tulu-2-dpo-7b/configs.yaml	15
src/alpaca_eval/models_configs/ultralm-13b-best-of-16/configs.yaml	4
src/alpaca_eval/models_configs/ultralm-13b-v2.0-best-of-16/configs.yaml	4
src/alpaca_eval/models_configs/ultralm-13b-v2.0/configs.yaml	14
src/alpaca_eval/models_configs/ultralm-13b/configs.yaml	14
src/alpaca_eval/models_configs/vicuna-13b-v1.3/configs.yaml	13
src/alpaca_eval/models_configs/vicuna-13b-v1.5/configs.yaml	13
src/alpaca_eval/models_configs/vicuna-13b/configs.yaml	13
src/alpaca_eval/models_configs/vicuna-33b-v1.3/configs.yaml	13
src/alpaca_eval/models_configs/vicuna-7b-v1.3/configs.yaml	13
src/alpaca_eval/models_configs/vicuna-7b-v1.5/configs.yaml	13
src/alpaca_eval/models_configs/vicuna-7b/configs.yaml	13
src/alpaca_eval/models_configs/wizardlm-13b-v1.1/configs.yaml	13
src/alpaca_eval/models_configs/wizardlm-13b-v1.2/configs.yaml	13
src/alpaca_eval/models_configs/wizardlm-13b/configs.yaml	13
src/alpaca_eval/models_configs/wizardlm-70b/configs.yaml	13
src/alpaca_eval/models_configs/xwinlm-13b-v0.1/configs.yaml	13
src/alpaca_eval/models_configs/xwinlm-70b-v0.1/configs.yaml	13
src/alpaca_eval/models_configs/xwinlm-70b-v0.3/configs.yaml	13
src/alpaca_eval/models_configs/xwinlm-7b-v0.1/configs.yaml	13
src/alpaca_eval/models_configs/zephyr-7b-alpha/configs.yaml	13
src/alpaca_eval/models_configs/zephyr-7b-beta/configs.yaml	13
src/alpaca_eval/plotting.py	463
src/alpaca_eval/processors.py	167
src/alpaca_eval/types.py	8
src/alpaca_eval/utils.py	338