plotstables/repetition.ipynb (1,376 lines of code) (raw):
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "JuBZROnbMNkQ",
"outputId": "17fe8964-0315-4938-d4da-c4ccbcbdc772"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"fatal: destination path 'lm1-2b8-55b-oscar-repetitions' already exists and is not an empty directory.\n",
"fatal: destination path 'lm1-2b8-55b-c4-repetitions' already exists and is not an empty directory.\n",
"fatal: destination path 'lm1-2b8-55b-oscarseeds' already exists and is not an empty directory.\n",
"fatal: destination path 'lm1-2b8-55b-c4seeds' already exists and is not an empty directory.\n",
"mv: cannot move 'lm1-2b8-55b-oscarseeds/tensorboard' to 'lm1-2b8-55b-oscar-repetitions/tensorboard': Directory not empty\n",
"mv: cannot move 'lm1-2b8-55b-c4seeds/tensorboard' to 'lm1-2b8-55b-c4-repetitions/tensorboard': Directory not empty\n",
"fatal: destination path 'lm1-4b2-84b-c4-repetitions' already exists and is not an empty directory.\n",
"fatal: destination path 'lm1-4b2-84b-c4seeds' already exists and is not an empty directory.\n",
"fatal: destination path 'lm1-4b2-84b-oscar-repetitions' already exists and is not an empty directory.\n",
"fatal: destination path 'lm1-4b2-84b-oscarseeds' already exists and is not an empty directory.\n",
"mv: cannot move 'lm1-4b2-84b-oscarseeds/tensorboard' to 'lm1-4b2-84b-oscar-repetitions/tensorboard': Directory not empty\n",
"mv: cannot move 'lm1-4b2-84b-c4seeds/tensorboard' to 'lm1-4b2-84b-c4-repetitions/tensorboard': Directory not empty\n",
"fatal: destination path 'lm1-8b7-178b-c4-repetitions' already exists and is not an empty directory.\n",
"fatal: destination path 'lm1-8b7-178b-oscar-repetitions' already exists and is not an empty directory.\n",
"mkdir: cannot create directory ‘lm1-4b2-84b-c4-opt’: File exists\n",
"mkdir: cannot create directory ‘lm1-8b7-178b-c4-opt’: File exists\n",
"mv: cannot move 'lm1-2b8-55b-c4seeds/tensorboard' to 'lm1-2b8-55b-c4-repetitions/tensorboard': Directory not empty\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b1b25c4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b28bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b11bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b14bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b4bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b9bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-2b8-55b-c4-repetitions/2b855b18bc4; Setting to 0 (which is OK for C4)\n",
"mv: cannot move 'lm1-2b8-55b-oscarseeds/tensorboard' to 'lm1-2b8-55b-oscar-repetitions/tensorboard': Directory not empty\n",
"mv: cannot move 'lm1-4b2-84b-c4seeds/tensorboard' to 'lm1-4b2-84b-c4-repetitions/tensorboard': Directory not empty\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b1b9c4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b84bc4v2seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b12bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b42bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b21bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b28bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b6bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-repetitions/4b284b17bc4seed4; Setting to 0 (which is OK for C4)\n",
"mv: 'lm1-4b2-84b-c4-opt/4b284b12bc4' and 'lm1-4b2-84b-c4-opt/4b284b12bc4' are the same file\n",
"mv: 'lm1-4b2-84b-c4-opt/4b284b12bc4opt1' and 'lm1-4b2-84b-c4-opt/4b284b12bc4opt1' are the same file\n",
"mv: 'lm1-4b2-84b-c4-opt/4b284b12bc4opt2' and 'lm1-4b2-84b-c4-opt/4b284b12bc4opt2' are the same file\n",
"mv: 'lm1-4b2-84b-c4-opt/4b284b12bc4seed1' and 'lm1-4b2-84b-c4-opt/4b284b12bc4seed1' are the same file\n",
"mv: 'lm1-4b2-84b-c4-opt/4b284b12bc4seed2' and 'lm1-4b2-84b-c4-opt/4b284b12bc4seed2' are the same file\n",
"mv: 'lm1-4b2-84b-c4-opt/4b284b12bc4seed3' and 'lm1-4b2-84b-c4-opt/4b284b12bc4seed3' are the same file\n",
"mv: 'lm1-4b2-84b-c4-opt/4b284b12bc4seed4' and 'lm1-4b2-84b-c4-opt/4b284b12bc4seed4' are the same file\n",
"mv: 'lm1-4b2-84b-c4-opt/4b284b12bc4subopt' and 'lm1-4b2-84b-c4-opt/4b284b12bc4subopt' are the same file\n",
"mv: 'lm1-4b2-84b-c4-opt/sbatch_4b284b12bc4opt3.sh' and 'lm1-4b2-84b-c4-opt/sbatch_4b284b12bc4opt3.sh' are the same file\n",
"mv: 'lm1-4b2-84b-c4-opt/sbatch_4b284b12bc4seed1.sh' and 'lm1-4b2-84b-c4-opt/sbatch_4b284b12bc4seed1.sh' are the same file\n",
"mv: 'lm1-4b2-84b-c4-opt/sbatch_4b284b12bc4seed2.sh' and 'lm1-4b2-84b-c4-opt/sbatch_4b284b12bc4seed2.sh' are the same file\n",
"mv: 'lm1-4b2-84b-c4-opt/sbatch_4b284b12bc4seed3.sh' and 'lm1-4b2-84b-c4-opt/sbatch_4b284b12bc4seed3.sh' are the same file\n",
"mv: 'lm1-4b2-84b-c4-opt/sbatch_4b284b12bc4seed4.sh' and 'lm1-4b2-84b-c4-opt/sbatch_4b284b12bc4seed4.sh' are the same file\n",
"mv: 'lm1-4b2-84b-c4-opt/sbatch_4b284b12bc4.sh' and 'lm1-4b2-84b-c4-opt/sbatch_4b284b12bc4.sh' are the same file\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed3; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed4; Setting to 0 (which is OK for C4)\n",
"Skipping: lm1-4b2-84b-c4-opt/4b284b12bc4opt4seed2\n",
"Skipping: lm1-4b2-84b-c4-opt/4b284b12bc4opt4seed3\n",
"Skipping: lm1-4b2-84b-c4-opt/4b284b12bc4opt4seed4\n",
"Skipping: lm1-4b2-84b-c4-opt/4b284b12bc4opt4\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed1; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed2; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4seed2; Setting to 0 (which is OK for C4)\n",
"Skipping: lm1-4b2-84b-c4-opt/4b284b12bc4opt4seed1\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-4b2-84b-c4-opt/4b284b12bc4; Setting to 0 (which is OK for C4)\n",
"mv: cannot move 'lm1-4b2-84b-oscarseeds/tensorboard' to 'lm1-4b2-84b-oscar-repetitions/tensorboard': Directory not empty\n",
"mv: 'lm1-8b7-178b-c4-opt/8b7178b25b' and 'lm1-8b7-178b-c4-opt/8b7178b25b' are the same file\n",
"mv: 'lm1-8b7-178b-c4-opt/8b7178b25bopt' and 'lm1-8b7-178b-c4-opt/8b7178b25bopt' are the same file\n",
"mv: 'lm1-8b7-178b-c4-opt/sbatch_8b7178b25b.sh' and 'lm1-8b7-178b-c4-opt/sbatch_8b7178b25b.sh' are the same file\n",
"Skipping: lm1-8b7-178b-c4-opt/8b7178b25boptseed2\n",
"Skipping: lm1-8b7-178b-c4-opt/8b7178b25boptseed4\n",
"Skipping: lm1-8b7-178b-c4-opt/8b7178b25boptseed1\n",
"Skipping: lm1-8b7-178b-c4-opt/8b7178b25bseed1\n",
"Missing bAbI: lm1-8b7-178b-c4-opt/8b7178b25bopt; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-opt/8b7178b25bopt; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-opt/8b7178b25bopt; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-opt/8b7178b25bopt; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-opt/8b7178b25bopt; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-opt/8b7178b25bopt; Setting to 0 (which is OK for C4)\n",
"Skipping: lm1-8b7-178b-c4-opt/8b7178b25bseed4\n",
"Skipping: lm1-8b7-178b-c4-opt/8b7178b25bseed2\n",
"Skipping: lm1-8b7-178b-c4-opt/8b7178b25boptseed3\n",
"Skipping: lm1-8b7-178b-c4-opt/8b7178b25bseed3\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b35b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b35b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b35b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b35b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b35b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b35b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b4b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b4b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b4b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b4b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b4b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b4b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b58b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b58b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b58b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b58b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b58b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b58b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b88b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b88b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b88b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b88b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b88b; Setting to 0 (which is OK for C4)\n",
"Missing bAbI: lm1-8b7-178b-c4-repetitions/8b7178b88b; Setting to 0 (which is OK for C4)\n",
"Skipping: lm1-8b7-178b-oscar-repetitions/8b7178b25bopt\n"
]
}
],
"source": [
"import glob\n",
"import os\n",
"import pandas as pd\n",
"\n",
"!GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/datablations/lm1-2b8-55b-oscar-repetitions\n",
"!GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/datablations/lm1-2b8-55b-c4-repetitions\n",
"!GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/datablations/lm1-2b8-55b-oscarseeds\n",
"!GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/datablations/lm1-2b8-55b-c4seeds\n",
"!mv lm1-2b8-55b-oscarseeds/* lm1-2b8-55b-oscar-repetitions/\n",
"!mv lm1-2b8-55b-c4seeds/* lm1-2b8-55b-c4-repetitions/\n",
"\n",
"!GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/datablations/lm1-4b2-84b-c4-repetitions\n",
"!GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/datablations/lm1-4b2-84b-c4seeds\n",
"!GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/datablations/lm1-4b2-84b-oscar-repetitions\n",
"!GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/datablations/lm1-4b2-84b-oscarseeds\n",
"!mv lm1-4b2-84b-oscarseeds/* lm1-4b2-84b-oscar-repetitions/\n",
"!mv lm1-4b2-84b-c4seeds/* lm1-4b2-84b-c4-repetitions/\n",
"\n",
"!GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/datablations/lm1-8b7-178b-c4-repetitions\n",
"!GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/datablations/lm1-8b7-178b-oscar-repetitions\n",
"\n",
"!mkdir lm1-4b2-84b-c4-opt\n",
"!cp -r lm1-4b2-84b-c4-repetitions/*4b284b12bc4* lm1-4b2-84b-c4-opt/\n",
"\n",
"!mkdir lm1-8b7-178b-c4-opt\n",
"!cp -r lm1-8b7-178b-c4-repetitions/*8b7178b25b* lm1-8b7-178b-c4-opt/\n",
"\n",
"\n",
"TASK_TO_BASELINE = {\n",
" \"anli_r1\": 1/3,\n",
" \"anli_r2\": 1/3,\n",
" \"anli_r3\": 1/3,\n",
" \"arc_challenge\": 1/4,\n",
" \"arc_easy\": 1/4,\n",
" \"boolq\": 1/2,\n",
" \"cb\": 1/3,\n",
" \"copa\": 1/2,\n",
" \"hellaswag\": 1/4,\n",
" \"piqa\": 1/2,\n",
" \"rte\": 1/2,\n",
" \"sciq\": 1/4,\n",
" \"storycloze_2016\": 1/4,\n",
" \"winogrande\": 1/2,\n",
" \"babi\": 0,\n",
"}\n",
"\n",
"\n",
"MODELS_MAP = {\n",
" \"lm1-2b8-55b-c4-repetitions\": {\n",
" 44: \"2b855b1b25c4\",\n",
" 14: \"2b855b4bc4\",\n",
" 7: \"2b855b9bc4\",\n",
"\t 5: \"2b855b11bc4\",\n",
" 4: \"2b855b14bc4\",\n",
" 3: \"2b855b18bc4\",\n",
" 2: \"2b855b28bc4\",\n",
" 1: \"2b855b55bc4\",\n",
" },\n",
" \"lm1-2b8-55b-oscar-repetitions\": {\n",
" 44: \"2b855b1b25oscar\",\n",
" 14: \"2b855b4boscar\",\n",
" 7: \"2b855b9boscar\",\n",
"\t 5: \"2b855b11boscar\",\n",
" 4: \"2b855b14boscar\",\n",
" 3: \"2b855b18boscar\",\n",
" 2: \"2b855b28boscar\",\n",
" 1: \"2b855b55boscar\",\n",
" },\n",
" \"lm1-4b2-84b-c4-repetitions\": {\n",
" 44: \"4b284b1b9c4\",\n",
"\t 14: \"4b284b6bc4\",\n",
" 7: \"4b284b12bc4\",\n",
"\t 5: \"4b284b17bc4\",\n",
" 4: \"4b284b21bc4\",\n",
" 3: \"4b284b28bc4\",\n",
" 2: \"4b284b42bc4\",\n",
" 1: \"4b284b84bc4v2\",\n",
" },\n",
" \"lm1-4b2-84b-c4-opt\": {\n",
" 10: \"4b284b12bc4opt4\",\n",
" 7: \"4b284b12bc4\",\n",
" }, \n",
"\n",
" \"lm1-4b2-84b-oscar-repetitions\": {\n",
" 44: \"4b284b1b9oscar\",\n",
"\t 14: \"4b284b6boscar\",\n",
" 7: \"4b284b12boscar\",\n",
"\t 5: \"4b284b17boscar\",\n",
" 4: \"4b284b21boscar\",\n",
" 3: \"4b284b28boscar\",\n",
" 2: \"4b284b42boscar\",\n",
" 1: \"4b284b84boscarv2\",\n",
" },\n",
" \"lm1-8b7-178b-c4-opt\": {\n",
" 10: \"8b7178b25bopt\",\n",
" 7: \"8b7178b25b\",\n",
" }, \n",
" \"lm1-8b7-178b-c4-repetitions\": {\n",
" 44: \"8b7178b4b\",\n",
" 14: \"8b7178b13b\",\n",
" 7: \"8b7178b25b\",\n",
"\t 5: \"8b7178b35b\",\n",
" 4: \"8b7178b44b\",\n",
" 3: \"8b7178b58b\",\n",
" 2: \"8b7178b88b\",\n",
" 1: \"8b7178b178b\",\n",
" },\n",
" \"lm1-8b7-178b-oscar-repetitions\": {\n",
" 10: \"8b7178b25bopt\",\n",
" 44: \"8b7178b4b\",\n",
" 14: \"8b7178b13b\", \n",
" 7: \"8b7178b25b\",\n",
" 5: \"8b7178b35b\",\n",
" 4: \"8b7178b44b\",\n",
" 3: \"8b7178b58b\",\n",
" 2: \"8b7178b88b\",\n",
" 1: \"8b7178b178b\",\n",
" }\n",
"}\n",
"\n",
"MODEL_TO_FEWSHOT_SCORES = {}\n",
"MODEL_TO_FEWSHOT_SCORES_ACC = {}\n",
"MODEL_TO_FEWSHOT_SCORES_GEN = {}\n",
"\n",
"SHOTS = list(range(6))\n",
"\n",
"for (BASE_MODEL, MODELS) in MODELS_MAP.items():\n",
" MODELS = set(MODELS.values())\n",
"\n",
" MODEL_TO_FEWSHOT_SCORES.setdefault(BASE_MODEL, {})\n",
" MODEL_TO_FEWSHOT_SCORES_ACC.setdefault(BASE_MODEL, {})\n",
" MODEL_TO_FEWSHOT_SCORES_GEN.setdefault(BASE_MODEL, {})\n",
"\n",
" if os.path.isdir(BASE_MODEL.replace(\"-repetitions\", \"seeds\")):\n",
" !mv {BASE_MODEL.replace(\"-repetitions\", \"seeds\")}/* {BASE_MODEL}/\n",
" MODELS |= {f\"{MODEL}seed{seed}\" for MODEL in MODELS for seed in [1, 2, 3, 4]}\n",
"\n",
" for MODEL in MODELS:\n",
"\n",
" MODEL_TO_FEWSHOT_SCORES[BASE_MODEL].setdefault(MODEL, {})\n",
" MODEL_TO_FEWSHOT_SCORES_ACC[BASE_MODEL].setdefault(MODEL, {})\n",
" MODEL_TO_FEWSHOT_SCORES_GEN[BASE_MODEL].setdefault(MODEL, {})\n",
"\n",
" path = f\"{BASE_MODEL}/{MODEL}/evaluation/generation/merged.csv\"\n",
" if not os.path.exists(path):\n",
" print(\"Skipping: \", f\"{BASE_MODEL}/{MODEL}\")\n",
" continue\n",
" generation = pd.read_csv(path)\n",
" for SHOT in SHOTS: \n",
" \n",
" MODEL_TO_FEWSHOT_SCORES[BASE_MODEL][MODEL].setdefault(SHOT, {})\n",
" MODEL_TO_FEWSHOT_SCORES_ACC[BASE_MODEL][MODEL].setdefault(SHOT, {})\n",
" MODEL_TO_FEWSHOT_SCORES_GEN[BASE_MODEL][MODEL].setdefault(SHOT, {})\n",
"\n",
" rankeval_files = glob.glob(f\"{BASE_MODEL}/{MODEL}/evaluation/rankeval/*_{SHOT}.csv\")\n",
" assert len(rankeval_files) == 1, f\"{rankeval_files}\"\n",
" rankeval = pd.read_csv(rankeval_files[0])\n",
"\n",
" # Rescale to 0 - 1, where 0 is random performance\n",
" rankeval[\"normalized\"] = rankeval.apply(lambda x: (x[\"value\"] - TASK_TO_BASELINE[x[\"task\"]]) / (1 - TASK_TO_BASELINE[x[\"task\"]]), axis=1)\n",
" rankeval = rankeval[rankeval[\"metric\"] == \"acc\"]\n",
" rankeval_scores = rankeval.normalized.values.tolist()\n",
"\n",
" gen_sub = generation[generation[\"fewshots\"] == SHOT]\n",
" gen_sub = gen_sub[gen_sub[\"prompt\"] != \"median\"]\n",
" gen_sub = gen_sub[gen_sub[\"prompt\"] != \"average\"]\n",
"\n",
" # 0 is already random performance, i.e. no rescaling necessary\n",
" generation_scores = gen_sub.value.values.tolist()\n",
" scores = rankeval_scores + generation_scores\n",
" average_score = sum(scores) / len(scores)\n",
"\n",
" for i, row in rankeval.iterrows():\n",
" MODEL_TO_FEWSHOT_SCORES[BASE_MODEL][MODEL][SHOT][row[\"task\"]] = row[\"normalized\"]\n",
" for i, row in gen_sub.iterrows():\n",
" MODEL_TO_FEWSHOT_SCORES[BASE_MODEL][MODEL][SHOT][row[\"dataset\"]] = row[\"value\"]\n",
"\n",
" babi_files = glob.glob(f\"{BASE_MODEL}/{MODEL}/evaluation/*{SHOT}_babi.json\")\n",
" if len(babi_files) == 1:\n",
" import json\n",
" with open(babi_files[0], \"r\") as f:\n",
" MODEL_TO_FEWSHOT_SCORES[BASE_MODEL][MODEL][SHOT][\"babi\"] = json.load(f)[\"results\"][\"babi\"][\"em\"]\n",
" else:\n",
" print(f\"Missing bAbI: {BASE_MODEL}/{MODEL}; Setting to 0 (which is OK for C4)\")\n",
" MODEL_TO_FEWSHOT_SCORES[BASE_MODEL][MODEL][SHOT][\"babi\"] = 0\n",
"\n",
" #MODEL_TO_FEWSHOT_SCORES[BASE_MODEL][MODEL][SHOT]\n",
" #MODEL_TO_FEWSHOT_SCORES_ACC[BASE_MODEL][MODEL].append(sum(rankeval_scores) / len(rankeval_scores))\n",
" #MODEL_TO_FEWSHOT_SCORES_GEN[BASE_MODEL][MODEL].append(sum(generation_scores) / len(generation_scores))\n"
]
},
{
"cell_type": "code",
"source": [
"### Latex Table ###\n",
"\n",
"import numpy as np\n",
"\n",
"TASK_TO_NAME = {\n",
" \"anli_r1\": \"ANLI R1\",\n",
" \"anli_r2\": \"ANLI R2\",\n",
" \"anli_r3\": \"ANLI R3\",\n",
" \"arc_challenge\": \"ARC-Challenge\",\n",
" \"arc_easy\": \"ARC-Easy\",\n",
" \"boolq\": \"BoolQ\",\n",
" \"cb\": \"CB\",\n",
" \"copa\": \"COPA\",\n",
" \"hellaswag\": \"HellaSwag\",\n",
" \"piqa\": \"PiQA\",\n",
" \"rte\": \"RTE\",\n",
" \"sciq\": \"SciQ\",\n",
" \"storycloze_2016\": \"StoryCloze 2016\",\n",
" \"winogrande\": \"WinoGrande XL\",\n",
"\n",
" \"e2e_nlg_cleaned\": \"E2E NLG\",\n",
" \"gem_xsum\": \"XSUM\",\n",
" \"web_nlg_en\": \"WebNLG EN\",\n",
" \"wiki_lingua_en\": \"WikiLingua EN\",\n",
" \n",
" \"babi\": \"bAbI\",\n",
"}\n",
"\n",
"\n",
"TABLE = \"\"\"\n",
"\\\\begin{table}\n",
" \\\\centering\n",
" \\\\caption{2.8B perf.}\n",
" \\\\resizebox{\\\\textwidth}{!}{\n",
" \\\\begin{tabular}{l|llllllll}\n",
"\"\"\"\n",
"\n",
"MODEL_KEY = 'lm1-2b8-55b-oscar-repetitions' # Change to desired model\n",
"MODEL_KEY = 'lm1-4b2-84b-c4-repetitions' # Change to desired model\n",
"\n",
"\n",
"MODEL_FAM = list(MODELS_MAP[MODEL_KEY].values())[::-1]\n",
"MODEL_EPS = list(MODELS_MAP[MODEL_KEY].keys())[::-1]\n",
"\n",
"SUBHEADER = \"Unique Tokens & \" + \" & \".join([str(x) for x in MODEL_EPS]) + \" \\\\\\\\\" + \"\\n\"\n",
"SUBSUBHEADER = \"Epochs & \" + \" & \".join([str(x) for x in MODEL_EPS]) + \" \\\\\\\\\" + \"\\n\"\n",
"\n",
"TABLE += SUBHEADER + SUBSUBHEADER\n",
"\n",
"for task in TASK_TO_NAME:\n",
" scores_c4 = []\n",
" std_c4 = []\n",
" for model in MODEL_FAM:\n",
" if \"seed\" in model: continue\n",
" model_scores = []\n",
" for model_name in [model+\"\", model+\"seed1\", model+\"seed2\", model+\"seed3\", model+\"seed4\"]:\n",
" model_scores.append(sum([MODEL_TO_FEWSHOT_SCORES[MODEL_KEY][model_name][SHOT][task] for SHOT in [0, 1, 2, 3, 4, 5]]) / 6)\n",
" scores_c4.append(np.mean(model_scores))\n",
" std_c4.append(np.std(model_scores))\n",
"\n",
" scores_c4 = [f\"{str(round(score * 100, 1))} ± {str(round(std * 100, 1))}\" for score, std in zip(scores_c4, std_c4)]\n",
" TABLE += \"\\n\" + f\"{TASK_TO_NAME[task]} & \" + \" & \".join(scores_c4) + \" \\\\\\\\\"\n",
"\n",
" if task in (\"winogrande\", \"wiki_lingua_en\", \"babi\"):\n",
" TABLE += \"\\n\" + \"\\\\midrule\"\n",
"\n",
"# Add average\n",
"scores_avg_str_c4 = []\n",
"for model in MODEL_FAM:\n",
" if \"seed\" in model: continue\n",
" all_scores = []\n",
" for model_name in [model+\"\", model+\"seed1\", model+\"seed2\", model+\"seed3\", model+\"seed4\"]:\n",
" scores = [list(MODEL_TO_FEWSHOT_SCORES[MODEL_KEY][model_name][SHOT].values()) for SHOT in [0, 1, 2, 3, 4, 5]]\n",
" for task_scores in scores:\n",
" assert len(task_scores) == len(TASK_TO_NAME)\n",
" score_avg = [sub_score for shot_scores in scores for sub_score in shot_scores]\n",
" all_scores.append(np.mean(score_avg)*100)\n",
" scores_avg_str_c4.append(f\"{round(np.mean(all_scores), 1)} ± {round(np.std(all_scores), 1)}\")\n",
"\n",
"\"\"\"\n",
"scores_avg_str_c4_nobabi = []\n",
"for model in MODEL_FAM:\n",
" all_scores = []\n",
" for model_name in [model+\"\", model+\"seed1\", model+\"seed2\", model+\"seed3\", model+\"seed4\"]:\n",
" scores = [MODEL_TO_FEWSHOT_SCORES[MODEL_KEY][model_name][SHOT][task] for task in TASK_TO_NAME for SHOT in [0, 1, 2, 3, 4, 5] if task != \"babi\"]\n",
" all_scores.append(np.mean(scores)*100)\n",
" scores_avg_str_c4_nobabi.append(f\"{round(np.mean(all_scores), 1)} ± {round(np.std(all_scores), 1)}\")\n",
"\"\"\"\n",
"\n",
"TABLE += \"\\nAverage & \" + \" & \".join(scores_avg_str_c4) + \" \\\\\\\\\"\n",
"#TABLE += \"\\nAverage (without bAbI) & \" + \" & \".join(scores_avg_str_c4_nobabi) + \" \\\\\\\\\"\n",
"\n",
"TABLE += \"\\n\\\\bottomrule\"\n",
"\n",
"\n",
"print(\"\\n\\n\\n\")\n",
"print(TABLE)\n",
"\n",
"\n",
"\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "GiS_F_KOQoGc",
"outputId": "1dfa0fe2-9e84-4ea1-b565-db458cb3302d"
},
"execution_count": 2,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\n",
"\n",
"\n",
"\n",
"\n",
"\\begin{table}\n",
" \\centering\n",
" \\caption{2.8B perf.}\n",
" \\resizebox{\\textwidth}{!}{\n",
" \\begin{tabular}{l|llllllll}\n",
"Unique Tokens & 1 & 2 & 3 & 4 & 5 & 7 & 14 & 44 \\\\\n",
"Epochs & 1 & 2 & 3 & 4 & 5 & 7 & 14 & 44 \\\\\n",
"\n",
"ANLI R1 & -1.0 ± 0.3 & -0.7 ± 1.1 & -0.7 ± 1.0 & -0.4 ± 1.1 & 0.4 ± 0.8 & 0.5 ± 1.1 & 0.1 ± 0.9 & 0.2 ± 0.9 \\\\\n",
"ANLI R2 & 0.8 ± 0.5 & 0.8 ± 0.8 & 0.0 ± 1.4 & 0.5 ± 0.7 & 0.5 ± 0.9 & 0.3 ± 1.0 & 0.7 ± 0.7 & 2.5 ± 1.0 \\\\\n",
"ANLI R3 & 1.1 ± 0.7 & 0.8 ± 0.9 & 0.3 ± 0.8 & 1.4 ± 1.1 & 1.3 ± 0.9 & 2.3 ± 0.2 & 1.3 ± 0.2 & 1.6 ± 1.2 \\\\\n",
"ARC-Challenge & 5.3 ± 0.6 & 5.1 ± 0.9 & 5.2 ± 2.0 & 6.0 ± 0.8 & 4.7 ± 0.8 & 3.1 ± 0.4 & 2.9 ± 1.0 & -1.3 ± 1.0 \\\\\n",
"ARC-Easy & 49.2 ± 0.9 & 50.4 ± 1.2 & 47.4 ± 4.5 & 49.4 ± 0.7 & 48.7 ± 1.5 & 44.9 ± 0.7 & 45.0 ± 1.2 & 31.9 ± 0.9 \\\\\n",
"BoolQ & 18.2 ± 4.0 & 19.6 ± 5.1 & 22.1 ± 1.0 & 20.4 ± 3.6 & 18.4 ± 6.0 & 18.4 ± 3.9 & 18.9 ± 2.6 & -3.3 ± 7.1 \\\\\n",
"CB & 12.0 ± 7.2 & 8.5 ± 9.2 & 7.9 ± 10.4 & 19.6 ± 7.3 & 17.8 ± 7.3 & 15.1 ± 5.8 & 17.5 ± 3.5 & 19.5 ± 6.6 \\\\\n",
"COPA & 59.1 ± 5.4 & 57.7 ± 3.5 & 56.7 ± 2.0 & 55.5 ± 2.4 & 56.8 ± 1.8 & 58.9 ± 1.7 & 48.7 ± 3.3 & 34.9 ± 3.4 \\\\\n",
"HellaSwag & 27.8 ± 4.8 & 30.2 ± 0.5 & 29.8 ± 0.9 & 29.9 ± 0.7 & 28.5 ± 1.1 & 29.0 ± 0.5 & 27.0 ± 1.2 & 19.7 ± 0.5 \\\\\n",
"PiQA & 50.6 ± 0.5 & 50.8 ± 0.5 & 48.6 ± 3.4 & 50.9 ± 0.7 & 50.3 ± 1.3 & 49.5 ± 0.4 & 47.6 ± 1.2 & 39.5 ± 1.3 \\\\\n",
"RTE & 5.6 ± 3.1 & 2.6 ± 3.9 & 7.2 ± 2.7 & 7.0 ± 3.2 & 8.8 ± 5.3 & 9.3 ± 3.6 & 3.0 ± 4.3 & 2.6 ± 4.2 \\\\\n",
"SciQ & 84.6 ± 3.9 & 86.1 ± 1.3 & 84.4 ± 3.7 & 85.9 ± 0.7 & 86.2 ± 0.8 & 79.0 ± 0.7 & 81.1 ± 1.4 & 65.3 ± 1.1 \\\\\n",
"StoryCloze 2016 & 61.1 ± 3.7 & 62.6 ± 0.2 & 61.9 ± 2.2 & 62.6 ± 0.4 & 61.8 ± 0.8 & 61.5 ± 0.8 & 60.1 ± 0.7 & 53.9 ± 0.5 \\\\\n",
"WinoGrande XL & 17.0 ± 2.6 & 17.8 ± 1.4 & 16.5 ± 1.8 & 17.1 ± 1.8 & 14.9 ± 1.5 & 15.9 ± 1.2 & 11.8 ± 1.5 & 3.9 ± 0.8 \\\\\n",
"\\midrule\n",
"E2E NLG & 18.2 ± 1.2 & 18.8 ± 0.8 & 17.8 ± 1.5 & 16.0 ± 2.2 & 15.9 ± 2.5 & 13.8 ± 1.3 & 15.7 ± 0.9 & 11.2 ± 1.4 \\\\\n",
"XSUM & 2.9 ± 0.2 & 3.0 ± 0.2 & 2.8 ± 0.3 & 2.9 ± 0.2 & 2.9 ± 0.2 & 1.0 ± 0.4 & 2.4 ± 0.1 & 1.8 ± 0.1 \\\\\n",
"WebNLG EN & 4.8 ± 2.0 & 5.7 ± 0.2 & 5.4 ± 0.3 & 5.6 ± 0.2 & 5.4 ± 0.5 & 5.5 ± 0.1 & 5.4 ± 0.2 & 3.4 ± 0.3 \\\\\n",
"WikiLingua EN & 3.3 ± 0.5 & 3.6 ± 0.1 & 3.4 ± 0.1 & 3.4 ± 0.1 & 3.3 ± 0.1 & 1.4 ± 0.6 & 3.0 ± 0.1 & 2.2 ± 0.1 \\\\\n",
"\\midrule\n",
"bAbI & 0.0 ± 0.0 & 0.0 ± 0.0 & 0.0 ± 0.0 & 0.0 ± 0.0 & 0.0 ± 0.0 & 0.0 ± 0.0 & 0.0 ± 0.0 & 0.0 ± 0.0 \\\\\n",
"\\midrule\n",
"Average & 22.1 ± 1.7 & 22.3 ± 0.9 & 21.9 ± 1.2 & 22.8 ± 0.5 & 22.5 ± 0.6 & 21.6 ± 0.5 & 20.6 ± 0.6 & 15.2 ± 1.0 \\\\\n",
"\\bottomrule\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"### Latex Table ###\n",
"\n",
"import numpy as np\n",
"\n",
"TASK_TO_NAME = {\n",
" \"anli_r1\": \"ANLI R1\",\n",
" \"anli_r2\": \"ANLI R2\",\n",
" \"anli_r3\": \"ANLI R3\",\n",
" \"arc_challenge\": \"ARC-Challenge\",\n",
" \"arc_easy\": \"ARC-Easy\",\n",
" \"boolq\": \"BoolQ\",\n",
" \"cb\": \"CB\",\n",
" \"copa\": \"COPA\",\n",
" \"hellaswag\": \"HellaSwag\",\n",
" \"piqa\": \"PiQA\",\n",
" \"rte\": \"RTE\",\n",
" \"sciq\": \"SciQ\",\n",
" \"storycloze_2016\": \"StoryCloze 2016\",\n",
" \"winogrande\": \"WinoGrande XL\",\n",
"\n",
" \"e2e_nlg_cleaned\": \"E2E NLG\",\n",
" \"gem_xsum\": \"XSUM\",\n",
" \"web_nlg_en\": \"WebNLG EN\",\n",
" \"wiki_lingua_en\": \"WikiLingua EN\",\n",
" \n",
" \"babi\": \"bAbI\",\n",
"}\n",
"\n",
"\n",
"TABLE = \"\"\"\n",
"\\\\begin{table}\n",
" \\\\centering\n",
" \\\\caption{2.8B perf.}\n",
" \\\\resizebox{\\\\textwidth}{!}{\n",
" \\\\begin{tabular}{l|llllllll}\n",
"\"\"\"\n",
"\n",
"MODEL_KEY = 'lm1-2b8-55b-oscar-repetitions' # Change to desired model\n",
"MODEL_KEY = 'lm1-4b2-84b-c4-repetitions' # Change to desired model\n",
"\n",
"# MODEL_KEY = 'lm1-4b2-84b-c4-opt' # Not enough eval datasets to make a good comparison (too close to rndm); Loss is better to compare\n",
"#MODEL_KEY = 'lm1-8b7-178b-oscar-repetitions' # Change to desired model\n",
"MODEL_KEY = 'lm1-8b7-178b-c4-opt' # Not enough eval datasets to make a good comparison (too close to rndm); Loss is better to compare\n",
"\n",
"#MODEL_KEY2 = 'lm1-8b7-178b-oscar-repetitions' # Change to desired model\n",
"#MODEL_KEY = 'lm1-8b7-178b-c4-repetitions' # Change to desired model\n",
"\n",
"MODEL_FAM = list(MODELS_MAP[MODEL_KEY].values())[::-1]# + list(MODELS_MAP[MODEL_KEY2].values())[::-1]\n",
"MODEL_EPS = list(MODELS_MAP[MODEL_KEY].keys())[::-1]# + list(MODELS_MAP[MODEL_KEY2].keys())[::-1]\n",
"\n",
"SUBHEADER = \"Unique Tokens & \" + \" & \".join([str(x) for x in MODEL_EPS]) + \" \\\\\\\\\" + \"\\n\"\n",
"SUBSUBHEADER = \"Epochs & \" + \" & \".join([str(x) for x in MODEL_EPS]) + \" \\\\\\\\\" + \"\\n\"\n",
"\n",
"TABLE += SUBHEADER + SUBSUBHEADER\n",
"\n",
"for task in TASK_TO_NAME:\n",
" scores_c4 = []\n",
" std_c4 = []\n",
" for model in MODEL_FAM:\n",
" if \"seed\" in model: continue\n",
" model_scores = []\n",
" for model_name in [model]:#+\"\", model+\"seed1\", model+\"seed2\", model+\"seed3\", model+\"seed4\"]:\n",
" model_scores.append(sum([MODEL_TO_FEWSHOT_SCORES[MODEL_KEY][model_name][SHOT][task] for SHOT in [0, 1, 2, 3, 4, 5]]) / 6)\n",
" scores_c4.append(np.mean(model_scores))\n",
" std_c4.append(np.std(model_scores))\n",
"\n",
" #scores_c4 = [f\"{str(round(score * 100, 1))} ± {str(round(std * 100, 1))}\" for score, std in zip(scores_c4, std_c4)]\n",
" scores_c4 = [f\"{str(round(score * 100, 1))}\" for score, std in zip(scores_c4, std_c4)]\n",
" TABLE += \"\\n\" + f\"{TASK_TO_NAME[task]} & \" + \" & \".join(scores_c4) + \" \\\\\\\\\"\n",
"\n",
" if task in (\"winogrande\", \"wiki_lingua_en\", \"babi\"):\n",
" TABLE += \"\\n\" + \"\\\\midrule\"\n",
"\n",
"# Add average\n",
"scores_avg_str_c4 = []\n",
"for model in MODEL_FAM:\n",
" if \"seed\" in model: continue\n",
" all_scores = []\n",
" for model_name in [model]:#+\"\", model+\"seed1\", model+\"seed2\", model+\"seed3\", model+\"seed4\"]:\n",
" scores = [list(MODEL_TO_FEWSHOT_SCORES[MODEL_KEY][model_name][SHOT].values()) for SHOT in [0, 1, 2, 3, 4, 5]]\n",
" for task_scores in scores:\n",
" assert len(task_scores) == len(TASK_TO_NAME)\n",
" score_avg = [sub_score for shot_scores in scores for sub_score in shot_scores]\n",
" all_scores.append(np.mean(score_avg)*100)\n",
" scores_avg_str_c4.append(f\"{round(np.mean(all_scores), 1)}\")# ± {round(np.std(all_scores), 1)}\")\n",
"\n",
"\"\"\"\n",
"scores_avg_str_c4_nobabi = []\n",
"for model in MODEL_FAM:\n",
" all_scores = []\n",
" for model_name in [model+\"\", model+\"seed1\", model+\"seed2\", model+\"seed3\", model+\"seed4\"]:\n",
" scores = [MODEL_TO_FEWSHOT_SCORES[MODEL_KEY][model_name][SHOT][task] for task in TASK_TO_NAME for SHOT in [0, 1, 2, 3, 4, 5] if task != \"babi\"]\n",
" all_scores.append(np.mean(scores)*100)\n",
" scores_avg_str_c4_nobabi.append(f\"{round(np.mean(all_scores), 1)} ± {round(np.std(all_scores), 1)}\")\n",
"\"\"\"\n",
"\n",
"TABLE += \"\\nAverage & \" + \" & \".join(scores_avg_str_c4) + \" \\\\\\\\\"\n",
"#TABLE += \"\\nAverage (without bAbI) & \" + \" & \".join(scores_avg_str_c4_nobabi) + \" \\\\\\\\\"\n",
"\n",
"TABLE += \"\\n\\\\bottomrule\"\n",
"\n",
"\n",
"print(\"\\n\\n\\n\")\n",
"print(TABLE)\n",
"\n",
"\n",
"\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "YKkEBxOHTzb2",
"outputId": "787c90e5-b483-47f9-bd34-578199805643"
},
"execution_count": 13,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\n",
"\n",
"\n",
"\n",
"\n",
"\\begin{table}\n",
" \\centering\n",
" \\caption{2.8B perf.}\n",
" \\resizebox{\\textwidth}{!}{\n",
" \\begin{tabular}{l|llllllll}\n",
"Unique Tokens & 7 & 10 \\\\\n",
"Epochs & 7 & 10 \\\\\n",
"\n",
"ANLI R1 & 0.1 & -0.9 \\\\\n",
"ANLI R2 & -0.1 & 1.0 \\\\\n",
"ANLI R3 & 1.6 & 2.6 \\\\\n",
"ARC-Challenge & 11.8 & 12.7 \\\\\n",
"ARC-Easy & 58.5 & 57.2 \\\\\n",
"BoolQ & 28.5 & 30.6 \\\\\n",
"CB & -22.8 & 6.2 \\\\\n",
"COPA & 70.0 & 66.0 \\\\\n",
"HellaSwag & 37.5 & 38.1 \\\\\n",
"PiQA & 53.9 & 54.3 \\\\\n",
"RTE & -2.6 & 7.7 \\\\\n",
"SciQ & 89.8 & 90.3 \\\\\n",
"StoryCloze 2016 & 66.8 & 68.4 \\\\\n",
"WinoGrande XL & 23.5 & 27.0 \\\\\n",
"\\midrule\n",
"E2E NLG & 17.7 & 16.9 \\\\\n",
"XSUM & 3.0 & 3.8 \\\\\n",
"WebNLG EN & 5.2 & 5.3 \\\\\n",
"WikiLingua EN & 4.0 & 4.0 \\\\\n",
"\\midrule\n",
"bAbI & 0.0 & 0.0 \\\\\n",
"\\midrule\n",
"Average & 23.5 & 25.9 \\\\\n",
"\\bottomrule\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"MODEL_TO_FEWSHOT_SCORES[MODEL_KEY]"
],
"metadata": {
"id": "AVjcM-oPUwV1",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "0f1a663f-dc28-4c77-fa81-bb12a47571c1"
},
"execution_count": 8,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"{'8b7178b25boptseed2': {},\n",
" '8b7178b25boptseed4': {},\n",
" '8b7178b25boptseed1': {},\n",
" '8b7178b25bseed1': {},\n",
" '8b7178b25bopt': {0: {'anli_r1': -0.004999999999999948,\n",
" 'anli_r2': 0.015999999999999983,\n",
" 'anli_r3': 0.04250000000000006,\n",
" 'arc_challenge': 0.1001137656427759,\n",
" 'arc_easy': 0.553872053872054,\n",
" 'boolq': 0.2703363914373089,\n",
" 'cb': -0.12499999999999996,\n",
" 'copa': 0.6599999999999999,\n",
" 'hellaswag': 0.37914094137953924,\n",
" 'piqa': 0.5353645266594125,\n",
" 'rte': 0.1335740072202165,\n",
" 'sciq': 0.8586666666666667,\n",
" 'storycloze_2016': 0.6764653482985925,\n",
" 'winogrande': 0.250197316495659,\n",
" 'e2e_nlg_cleaned': 0.0278991679592521,\n",
" 'gem_xsum': 0.0609067533158508,\n",
" 'web_nlg_en': 0.0543324401219052,\n",
" 'wiki_lingua_en': 0.0339801555675621,\n",
" 'babi': 0},\n",
" 1: {'anli_r1': -0.02149999999999996,\n",
" 'anli_r2': -0.02299999999999996,\n",
" 'anli_r3': 0.018750000000000013,\n",
" 'arc_challenge': 0.12286689419795212,\n",
" 'arc_easy': 0.5606060606060606,\n",
" 'boolq': 0.29235474006116213,\n",
" 'cb': 0.19642857142857145,\n",
" 'copa': 0.6200000000000001,\n",
" 'hellaswag': 0.3780787359755693,\n",
" 'piqa': 0.5473340587595212,\n",
" 'rte': 0.10469314079422376,\n",
" 'sciq': 0.904,\n",
" 'storycloze_2016': 0.6607874576875111,\n",
" 'winogrande': 0.23599052880820826,\n",
" 'e2e_nlg_cleaned': 0.1731365105367101,\n",
" 'gem_xsum': 0.0435091241930484,\n",
" 'web_nlg_en': 0.0555604021729821,\n",
" 'wiki_lingua_en': 0.0607570369852723,\n",
" 'babi': 0},\n",
" 2: {'anli_r1': -0.025999999999999964,\n",
" 'anli_r2': -0.001999999999999946,\n",
" 'anli_r3': 0.010000000000000063,\n",
" 'arc_challenge': 0.13196814562002265,\n",
" 'arc_easy': 0.5723905723905723,\n",
" 'boolq': 0.30336391437308863,\n",
" 'cb': -0.04464285714285723,\n",
" 'copa': 0.6799999999999999,\n",
" 'hellaswag': 0.37847706300205797,\n",
" 'piqa': 0.5375408052230686,\n",
" 'rte': 0.08303249097472931,\n",
" 'sciq': 0.912,\n",
" 'storycloze_2016': 0.6800285052556565,\n",
" 'winogrande': 0.27387529597474347,\n",
" 'e2e_nlg_cleaned': 0.202509848315452,\n",
" 'gem_xsum': 0.0542797442140433,\n",
" 'web_nlg_en': 0.0532534898798488,\n",
" 'wiki_lingua_en': 0.0672929256474048,\n",
" 'babi': 0},\n",
" 3: {'anli_r1': -0.024499999999999963,\n",
" 'anli_r2': 0.010000000000000063,\n",
" 'anli_r3': 0.010000000000000063,\n",
" 'arc_challenge': 0.12741751990898745,\n",
" 'arc_easy': 0.5819304152637486,\n",
" 'boolq': 0.31437308868501534,\n",
" 'cb': 0.0892857142857142,\n",
" 'copa': 0.6599999999999999,\n",
" 'hellaswag': 0.3833897629954192,\n",
" 'piqa': 0.5484221980413493,\n",
" 'rte': 0.06859205776173294,\n",
" 'sciq': 0.9106666666666667,\n",
" 'storycloze_2016': 0.6892927133440229,\n",
" 'winogrande': 0.28966061562746637,\n",
" 'e2e_nlg_cleaned': 0.2047491374803112,\n",
" 'gem_xsum': 0.0557825558290412,\n",
" 'web_nlg_en': 0.0521159190240214,\n",
" 'wiki_lingua_en': 0.0555020768402019,\n",
" 'babi': 0},\n",
" 4: {'anli_r1': 0.013000000000000065,\n",
" 'anli_r2': 0.032499999999999994,\n",
" 'anli_r3': 0.06749999999999996,\n",
" 'arc_challenge': 0.14448236632536973,\n",
" 'arc_easy': 0.5791245791245792,\n",
" 'boolq': 0.32538226299694184,\n",
" 'cb': 0.1696428571428571,\n",
" 'copa': 0.7,\n",
" 'hellaswag': 0.3833897629954192,\n",
" 'piqa': 0.5473340587595212,\n",
" 'rte': -0.010830324909747446,\n",
" 'sciq': 0.916,\n",
" 'storycloze_2016': 0.6985569214323891,\n",
" 'winogrande': 0.28808208366219423,\n",
" 'e2e_nlg_cleaned': 0.204719273743759,\n",
" 'gem_xsum': 0.0140225446870279,\n",
" 'web_nlg_en': 0.0515586819008743,\n",
" 'wiki_lingua_en': 0.0174032547088738,\n",
" 'babi': 0},\n",
" 5: {'anli_r1': 0.007000000000000061,\n",
" 'anli_r2': 0.027999999999999994,\n",
" 'anli_r3': 0.010000000000000063,\n",
" 'arc_challenge': 0.1353811149032992,\n",
" 'arc_easy': 0.5864197530864197,\n",
" 'boolq': 0.3278287461773701,\n",
" 'cb': 0.0892857142857142,\n",
" 'copa': 0.6399999999999999,\n",
" 'hellaswag': 0.38511584677687044,\n",
" 'piqa': 0.5440696409140371,\n",
" 'rte': 0.08303249097472931,\n",
" 'sciq': 0.916,\n",
" 'storycloze_2016': 0.6999821842152146,\n",
" 'winogrande': 0.28334648776637716,\n",
" 'e2e_nlg_cleaned': 0.2026341260855064,\n",
" 'gem_xsum': 0.0004619320135697,\n",
" 'web_nlg_en': 0.0520970349245116,\n",
" 'wiki_lingua_en': 0.0030819597283648,\n",
" 'babi': 0}},\n",
" '8b7178b25bseed4': {},\n",
" '8b7178b25bseed2': {},\n",
" '8b7178b25boptseed3': {},\n",
" '8b7178b25bseed3': {},\n",
" '8b7178b25b': {0: {'anli_r1': 0.0010000000000000562,\n",
" 'anli_r2': 0.011500000000000064,\n",
" 'anli_r3': 0.043750000000000004,\n",
" 'arc_challenge': 0.0853242320819112,\n",
" 'arc_easy': 0.5488215488215489,\n",
" 'boolq': 0.2666666666666666,\n",
" 'cb': 0.008928571428571478,\n",
" 'copa': 0.74,\n",
" 'hellaswag': 0.37675097922060674,\n",
" 'piqa': 0.5408052230685527,\n",
" 'rte': -0.03971119133574019,\n",
" 'sciq': 0.836,\n",
" 'storycloze_2016': 0.6643506146445751,\n",
" 'winogrande': 0.2375690607734806,\n",
" 'e2e_nlg_cleaned': 0.0015194719382464,\n",
" 'gem_xsum': 0.0564423062308188,\n",
" 'web_nlg_en': 0.0474669590178212,\n",
" 'wiki_lingua_en': 0.0404779665262007,\n",
" 'babi': 0.0},\n",
" 1: {'anli_r1': 0.0010000000000000562,\n",
" 'anli_r2': 0.004000000000000058,\n",
" 'anli_r3': 0.00874999999999995,\n",
" 'arc_challenge': 0.111490329920364,\n",
" 'arc_easy': 0.5808080808080808,\n",
" 'boolq': 0.2703363914373089,\n",
" 'cb': -0.20535714285714285,\n",
" 'copa': 0.7,\n",
" 'hellaswag': 0.3718382792272455,\n",
" 'piqa': 0.5255712731229598,\n",
" 'rte': -0.054151624548736454,\n",
" 'sciq': 0.896,\n",
" 'storycloze_2016': 0.6536611437733831,\n",
" 'winogrande': 0.23441199684293612,\n",
" 'e2e_nlg_cleaned': 0.1863711533213715,\n",
" 'gem_xsum': 0.0359889510962096,\n",
" 'web_nlg_en': 0.052665704185835,\n",
" 'wiki_lingua_en': 0.0622366147541512,\n",
" 'babi': 0.0},\n",
" 2: {'anli_r1': -0.003499999999999947,\n",
" 'anli_r2': -0.013999999999999955,\n",
" 'anli_r3': 0.012500000000000037,\n",
" 'arc_challenge': 0.11604095563139934,\n",
" 'arc_easy': 0.5813692480359146,\n",
" 'boolq': 0.29357798165137616,\n",
" 'cb': -0.28571428571428575,\n",
" 'copa': 0.6599999999999999,\n",
" 'hellaswag': 0.37276770895571926,\n",
" 'piqa': 0.5484221980413493,\n",
" 'rte': -0.010830324909747446,\n",
" 'sciq': 0.9173333333333332,\n",
" 'storycloze_2016': 0.6686264029930519,\n",
" 'winogrande': 0.22336227308603007,\n",
" 'e2e_nlg_cleaned': 0.213092798354988,\n",
" 'gem_xsum': 0.0372895118311674,\n",
" 'web_nlg_en': 0.0515794441635892,\n",
" 'wiki_lingua_en': 0.06251395339429,\n",
" 'babi': 0.0},\n",
" 3: {'anli_r1': 0.0025000000000000573,\n",
" 'anli_r2': -0.00799999999999995,\n",
" 'anli_r3': 0.015000000000000012,\n",
" 'arc_challenge': 0.1171786120591581,\n",
" 'arc_easy': 0.5942760942760943,\n",
" 'boolq': 0.2880733944954128,\n",
" 'cb': -0.28571428571428575,\n",
" 'copa': 0.6799999999999999,\n",
" 'hellaswag': 0.37608710084312547,\n",
" 'piqa': 0.5473340587595212,\n",
" 'rte': -0.018050541516245633,\n",
" 'sciq': 0.908,\n",
" 'storycloze_2016': 0.671476928558703,\n",
" 'winogrande': 0.22967640094711927,\n",
" 'e2e_nlg_cleaned': 0.2203233304219452,\n",
" 'gem_xsum': 0.0384820710887235,\n",
" 'web_nlg_en': 0.0522707618898852,\n",
" 'wiki_lingua_en': 0.0523326312861241,\n",
" 'babi': 0.0},\n",
" 4: {'anli_r1': 0.0055000000000000595,\n",
" 'anli_r2': 0.004000000000000058,\n",
" 'anli_r3': 0.00625000000000006,\n",
" 'arc_challenge': 0.1353811149032992,\n",
" 'arc_easy': 0.5998877665544332,\n",
" 'boolq': 0.2948012232415902,\n",
" 'cb': -0.31249999999999994,\n",
" 'copa': 0.7,\n",
" 'hellaswag': 0.3747593440881631,\n",
" 'piqa': 0.5397170837867247,\n",
" 'rte': -0.032490974729242006,\n",
" 'sciq': 0.912,\n",
" 'storycloze_2016': 0.678603242472831,\n",
" 'winogrande': 0.24546172059984217,\n",
" 'e2e_nlg_cleaned': 0.2199359497893921,\n",
" 'gem_xsum': 0.0112762080786988,\n",
" 'web_nlg_en': 0.0529565286021585,\n",
" 'wiki_lingua_en': 0.0168375868924872,\n",
" 'babi': 0.0},\n",
" 5: {'anli_r1': -0.0004999999999999448,\n",
" 'anli_r2': -0.001999999999999946,\n",
" 'anli_r3': 0.00874999999999995,\n",
" 'arc_challenge': 0.142207053469852,\n",
" 'arc_easy': 0.6038159371492705,\n",
" 'boolq': 0.29724770642201825,\n",
" 'cb': -0.28571428571428575,\n",
" 'copa': 0.72,\n",
" 'hellaswag': 0.3766182035451105,\n",
" 'piqa': 0.529923830250272,\n",
" 'rte': -0.0036101083032491488,\n",
" 'sciq': 0.9173333333333332,\n",
" 'storycloze_2016': 0.6729021913415286,\n",
" 'winogrande': 0.23914759273875297,\n",
" 'e2e_nlg_cleaned': 0.2179343802265635,\n",
" 'gem_xsum': 0.0005538280315731,\n",
" 'web_nlg_en': 0.0542664019748807,\n",
" 'wiki_lingua_en': 0.002813828065466,\n",
" 'babi': 0.0}}}"
]
},
"metadata": {},
"execution_count": 8
}
]
},
{
"cell_type": "code",
"source": [],
"metadata": {
"id": "nhlW1ZiHtWvU"
},
"execution_count": null,
"outputs": []
}
]
}