notebooks/plot_mean_score.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import cm, colors, rcParams\n",
    "\n",
    "import bayesmark.constants as cc\n",
    "import bayesmark.xr_util as xru\n",
    "from bayesmark.serialize import XRSerializer\n",
    "from bayesmark.constants import ITER, METHOD, ARG_DELIM, OBJECTIVE, VISIBLE_TO_OPT\n",
    "from bayesmark.path_util import abspath\n",
    "from bayesmark.util import preimage_func"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# User settings, must specify location of the data to make plots here for this to run\n",
    "DB_ROOT = abspath(\".\")\n",
    "DBID = \"bo_example_folder\"\n",
    "metric_for_scoring = VISIBLE_TO_OPT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Matplotlib setup\n",
    "# Note this will put type-3 font BS in the pdfs, if it matters\n",
    "rcParams[\"mathtext.fontset\"] = \"stix\"\n",
    "rcParams[\"font.family\"] = \"STIXGeneral\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_color_dict(names):\n",
    "    \"\"\"Make a color dictionary to give each name a mpl color.\n",
    "    \"\"\"\n",
    "    norm = colors.Normalize(vmin=0, vmax=1)\n",
    "    m = cm.ScalarMappable(norm, cm.tab20)\n",
    "    color_dict = m.to_rgba(np.linspace(0, 1, len(names)))\n",
    "    color_dict = dict(zip(names, color_dict))\n",
    "    return color_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the data\n",
    "summary_ds, meta = XRSerializer.load_derived(DB_ROOT, db=DBID, key=cc.MEAN_SCORE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "method_to_rgba = build_color_dict(summary_ds.coords[METHOD].values.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Group methods by the package behind them\n",
    "method_only = lambda method_rev: method_rev.split(ARG_DELIM, 1)[0]\n",
    "groups = preimage_func(method_only, summary_ds.coords[METHOD].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make a plot for each package\n",
    "for method_name in groups:\n",
    "    plt.figure(figsize=(5, 5), dpi=300)\n",
    "    for method_ver_name in groups[method_name]:\n",
    "        curr_ds = summary_ds.sel({METHOD: method_ver_name, OBJECTIVE: metric_for_scoring})\n",
    "        curr_ds.coords[ITER].values\n",
    "\n",
    "        plt.fill_between(\n",
    "            curr_ds.coords[ITER].values,\n",
    "            curr_ds[cc.LB_MED].values,\n",
    "            curr_ds[cc.UB_MED].values,\n",
    "            color=method_to_rgba[method_ver_name],\n",
    "            alpha=0.5,\n",
    "        )\n",
    "        plt.plot(\n",
    "            curr_ds.coords[ITER].values,\n",
    "            curr_ds[cc.PERF_MED].values,\n",
    "            color=method_to_rgba[method_ver_name],\n",
    "            label=method_name,\n",
    "            marker=\".\",\n",
    "        )\n",
    "    plt.xlabel(\"evaluation\", fontsize=10)\n",
    "    plt.ylabel(\"normalized median score\", fontsize=10)\n",
    "    plt.title(method_name)\n",
    "    plt.legend(fontsize=8, bbox_to_anchor=(1.05, 1), loc=\"upper left\", borderaxespad=0.0)\n",
    "    plt.grid()\n",
    "\n",
    "    plt.figure(figsize=(5, 5), dpi=300)\n",
    "    for method_ver_name in groups[method_name]:\n",
    "        curr_ds = summary_ds.sel({METHOD: method_ver_name, OBJECTIVE: metric_for_scoring})\n",
    "        curr_ds.coords[ITER].values\n",
    "\n",
    "        plt.fill_between(\n",
    "            curr_ds.coords[ITER].values,\n",
    "            curr_ds[cc.LB_MEAN].values,\n",
    "            curr_ds[cc.UB_MEAN].values,\n",
    "            color=method_to_rgba[method_ver_name],\n",
    "            alpha=0.5,\n",
    "        )\n",
    "        plt.plot(\n",
    "            curr_ds.coords[ITER].values,\n",
    "            curr_ds[cc.PERF_MEAN].values,\n",
    "            color=method_to_rgba[method_ver_name],\n",
    "            label=method_name,\n",
    "            marker=\".\",\n",
    "        )\n",
    "    plt.xlabel(\"evaluation\", fontsize=10)\n",
    "    plt.ylabel(\"mean score\", fontsize=10)\n",
    "    plt.title(method_name)\n",
    "    plt.legend(fontsize=8, bbox_to_anchor=(1.05, 1), loc=\"upper left\", borderaxespad=0.0)\n",
    "    plt.grid()\n",
    "\n",
    "    plt.figure(figsize=(5, 5), dpi=300)\n",
    "    for method_ver_name in groups[method_name]:\n",
    "        curr_ds = summary_ds.sel({METHOD: method_ver_name, OBJECTIVE: metric_for_scoring})\n",
    "        curr_ds.coords[ITER].values\n",
    "\n",
    "        plt.fill_between(\n",
    "            curr_ds.coords[ITER].values,\n",
    "            curr_ds[cc.LB_NORMED_MEAN].values,\n",
    "            curr_ds[cc.UB_NORMED_MEAN].values,\n",
    "            color=method_to_rgba[method_ver_name],\n",
    "            alpha=0.5,\n",
    "        )\n",
    "        plt.plot(\n",
    "            curr_ds.coords[ITER].values,\n",
    "            curr_ds[cc.NORMED_MEAN].values,\n",
    "            color=method_to_rgba[method_ver_name],\n",
    "            label=method_name,\n",
    "            marker=\".\",\n",
    "        )\n",
    "    plt.xlabel(\"evaluation\", fontsize=10)\n",
    "    plt.ylabel(\"normalized mean score\", fontsize=10)\n",
    "    plt.title(method_name)\n",
    "    plt.legend(fontsize=8, bbox_to_anchor=(1.05, 1), loc=\"upper left\", borderaxespad=0.0)\n",
    "    plt.grid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make the summary plot\n",
    "plt.figure(figsize=(5, 5), dpi=300)\n",
    "for method_ver_name in summary_ds.coords[METHOD].values:\n",
    "    curr_ds = summary_ds.sel({METHOD: method_ver_name, OBJECTIVE: metric_for_scoring})\n",
    "    curr_ds.coords[ITER].values\n",
    "\n",
    "    plt.fill_between(\n",
    "        curr_ds.coords[ITER].values,\n",
    "        curr_ds[cc.LB_MED].values,\n",
    "        curr_ds[cc.UB_MED].values,\n",
    "        color=method_to_rgba[method_ver_name],\n",
    "        alpha=0.5,\n",
    "    )\n",
    "    plt.plot(\n",
    "        curr_ds.coords[ITER].values,\n",
    "        curr_ds[cc.PERF_MED].values,\n",
    "        color=method_to_rgba[method_ver_name],\n",
    "        label=method_ver_name,\n",
    "        marker=\".\",\n",
    "    )\n",
    "plt.xlabel(\"evaluation\", fontsize=10)\n",
    "plt.ylabel(\"normalized median score\", fontsize=10)\n",
    "plt.legend(fontsize=8, bbox_to_anchor=(1.05, 1), loc=\"upper left\", borderaxespad=0.0)\n",
    "plt.grid()\n",
    "\n",
    "plt.figure(figsize=(5, 5), dpi=300)\n",
    "for method_ver_name in summary_ds.coords[METHOD].values:\n",
    "    curr_ds = summary_ds.sel({METHOD: method_ver_name, OBJECTIVE: metric_for_scoring})\n",
    "    curr_ds.coords[ITER].values\n",
    "\n",
    "    plt.fill_between(\n",
    "        curr_ds.coords[ITER].values,\n",
    "        curr_ds[cc.LB_MEAN].values,\n",
    "        curr_ds[cc.UB_MEAN].values,\n",
    "        color=method_to_rgba[method_ver_name],\n",
    "        alpha=0.5,\n",
    "    )\n",
    "    plt.plot(\n",
    "        curr_ds.coords[ITER].values,\n",
    "        curr_ds[cc.PERF_MEAN].values,\n",
    "        color=method_to_rgba[method_ver_name],\n",
    "        label=method_ver_name,\n",
    "        marker=\".\",\n",
    "    )\n",
    "plt.xlabel(\"evaluation\", fontsize=10)\n",
    "plt.ylabel(\"mean score\", fontsize=10)\n",
    "plt.legend(fontsize=8, bbox_to_anchor=(1.05, 1), loc=\"upper left\", borderaxespad=0.0)\n",
    "plt.grid()\n",
    "\n",
    "plt.figure(figsize=(5, 5), dpi=300)\n",
    "for method_ver_name in summary_ds.coords[METHOD].values:\n",
    "    curr_ds = summary_ds.sel({METHOD: method_ver_name, OBJECTIVE: metric_for_scoring})\n",
    "    curr_ds.coords[ITER].values\n",
    "\n",
    "    plt.fill_between(\n",
    "        curr_ds.coords[ITER].values,\n",
    "        curr_ds[cc.LB_NORMED_MEAN].values,\n",
    "        curr_ds[cc.UB_NORMED_MEAN].values,\n",
    "        color=method_to_rgba[method_ver_name],\n",
    "        alpha=0.5,\n",
    "    )\n",
    "    plt.plot(\n",
    "        curr_ds.coords[ITER].values,\n",
    "        curr_ds[cc.NORMED_MEAN].values,\n",
    "        color=method_to_rgba[method_ver_name],\n",
    "        label=method_ver_name,\n",
    "        marker=\".\",\n",
    "    )\n",
    "plt.xlabel(\"evaluation\", fontsize=10)\n",
    "plt.ylabel(\"normalized mean score\", fontsize=10)\n",
    "plt.legend(fontsize=8, bbox_to_anchor=(1.05, 1), loc=\"upper left\", borderaxespad=0.0)\n",
    "plt.grid()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "bobm_ipynb",
   "language": "python",
   "name": "bobm_ipynb"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}