{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "(Deprecated)\n", "This notebook performs performance analysis for the fde experiment" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import os, sys\n", "import re\n", "from six.moves.cPickle import load\n", "import argparse\n", "from time import time, strftime\n", "from inspect import getmembers, isfunction\n", "from imp import load_source\n", "import numpy as np\n", "from pandas import DataFrame, Series\n", "import json\n", "from subprocess import check_output\n", "import pickle\n", "import pandas as pd\n", "# Note: We import this *before* any import of TF to avoid weird issues\n", "# see- https://github.com/tensorflow/models/issues/523\n", "# Is a bit of a hack but seems harmless as long as SpaCy is installed\n", "import spacy" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "$SNORKELDB = postgres://matthieu:postgres@localhost:4554/snorkel_fda_new\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/home/matthieu/Documents/Snorkel/snorkel/.snorkelenv.gpu/local/lib/python2.7/site-packages/psycopg2/__init__.py:144: UserWarning: The psycopg2 wheel package will be renamed from release 2.8; in order to keep installing from binary please use \"pip install psycopg2-binary\" instead. For details see: .\n", " \"\"\")\n", "/home/matthieu/Documents/Snorkel/snorkel/.snorkelenv.gpu/local/lib/python2.7/site-packages/tensorflow/contrib/tensorboard/plugins/trace/trace.py:22: ImportWarning: Not importing directory '/home/matthieu/Documents/Snorkel/snorkel/parser': missing __init__.py\n", " import parser\n" ] } ], "source": [ "DB_address = 'postgres://matthieu:postgres@localhost:4554/snorkel_fda_new'\n", "# DB_address = 'sqlite:///snorkel_fda_beta.db'\n", "# DB_address = 'sqlite:///snorkel_fda_base.db'\n", "os.environ['SNORKELDB'] = DB_address\n", "print(\"$SNORKELDB = {0}\".format(os.environ['SNORKELDB']))\n", "\n", "# All Snorkel imports here, after SNORKELDB has been set\n", "from snorkel.annotations import (\n", " LabelAnnotator, load_label_matrix, load_gold_labels, load_marginals\n", ")\n", "from snorkel.learning.structure import DependencySelector\n", "from snorkel.learning import GenerativeModel, RandomSearch\n", "from snorkel.models.meta import SnorkelBase, snorkel_engine\n", "from snorkel.models import Document, Sentence\n", "from snorkel import SnorkelSession\n", "\n", "from utils import *\n", "\n", "\n", "# Start Snorkel sess\n", "sess = SnorkelSession()\n", "\n", "# Only use parallelism > 1 with UDFs if using Postgres\n", "UDF_THREADS = 2\n", "\n", "# Import from parsers.py\n", "preprocess = load_source('preprocess',\n", " os.path.join('experiments', \"fde\", 'preprocess.py'))\n", "\n", "# Get candidate subclass from loaded module\n", "C = preprocess.C" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "u'/home/matthieu/Documents/Snorkel/scuba'" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%pwd" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "-1\n", "-1\n", "-1\n", "-1\n" ] } ], "source": [ "for i in range(1,5):\n", " gl = np.fromfile(\"../ExtraData/Unipolar LFs/labels/labels_gold_{}.np\".format(i),dtype=int)\n", " print(gl[2])" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "labeled_cands = sess.query(C).filter(C.split.in_({1,2,3,4})).all()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Get predictions" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "predictions_df = pd.read_csv(\"../Notes/Predictions/fde_final/results_fde.csv\",sep = \",\",index_col=1)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Unnamed: 0labelmargpredsplit
cid
154628840-10.017843-11
154628851-10.160404-11
154628872-10.213403-11
154628903-10.218732-11
154628914-10.018498-11
154628965-10.250272-11
154628976-10.010525-11
154628987-10.038530-11
154628998-10.007943-11
154629009-10.136802-11
1546290110-10.284858-11
1547235611-10.028936-11
1547235712-10.272529-11
1552330513-10.75924311
1552330614-10.010990-11
1552330715-10.007623-11
1552330816-10.005565-11
1552330917-10.005975-11
1552331118-10.049002-11
1552331219-10.339343-11
1552331320-10.369974-11
1552331521-10.068182-11
1552331622-10.056450-11
1552331723-10.005567-11
1552331824-10.105247-11
1558658325-10.051365-11
1558658526-10.162297-11
1583528627-10.158337-11
1583528728-10.013231-11
1583528829-10.037000-11
..................
264914386401-10.048927-14
264914396402-10.417940-14
264914406403-10.161154-14
264914416404-10.041714-14
264914426405-10.054722-14
264914436406-10.021913-14
264914446407-10.026895-14
264914476408-10.058602-14
264914486409-10.008037-14
264914496410-10.003291-14
264914526411-10.157633-14
264914536412-10.073751-14
264914546413-10.017260-14
264914556414-10.009815-14
264914566415-10.146647-14
264914576416-10.016393-14
264914596417-10.008133-14
264914606418-10.072313-14
264914616419-10.172497-14
264914626420-10.140382-14
264914636421-10.185595-14
264914656422-10.012733-14
264914666423-10.186255-14
264914676424-10.032360-14
264914726425-10.015105-14
264914736426-10.162567-14
264914746427-10.015464-14
264914756428-10.430372-14
264914766429-10.490200-14
264914776430-10.010446-14
\n", "

6431 rows × 5 columns

\n", "
" ], "text/plain": [ " Unnamed: 0 label marg pred split\n", "cid \n", "15462884 0 -1 0.017843 -1 1\n", "15462885 1 -1 0.160404 -1 1\n", "15462887 2 -1 0.213403 -1 1\n", "15462890 3 -1 0.218732 -1 1\n", "15462891 4 -1 0.018498 -1 1\n", "15462896 5 -1 0.250272 -1 1\n", "15462897 6 -1 0.010525 -1 1\n", "15462898 7 -1 0.038530 -1 1\n", "15462899 8 -1 0.007943 -1 1\n", "15462900 9 -1 0.136802 -1 1\n", "15462901 10 -1 0.284858 -1 1\n", "15472356 11 -1 0.028936 -1 1\n", "15472357 12 -1 0.272529 -1 1\n", "15523305 13 -1 0.759243 1 1\n", "15523306 14 -1 0.010990 -1 1\n", "15523307 15 -1 0.007623 -1 1\n", "15523308 16 -1 0.005565 -1 1\n", "15523309 17 -1 0.005975 -1 1\n", "15523311 18 -1 0.049002 -1 1\n", "15523312 19 -1 0.339343 -1 1\n", "15523313 20 -1 0.369974 -1 1\n", "15523315 21 -1 0.068182 -1 1\n", "15523316 22 -1 0.056450 -1 1\n", "15523317 23 -1 0.005567 -1 1\n", "15523318 24 -1 0.105247 -1 1\n", "15586583 25 -1 0.051365 -1 1\n", "15586585 26 -1 0.162297 -1 1\n", "15835286 27 -1 0.158337 -1 1\n", "15835287 28 -1 0.013231 -1 1\n", "15835288 29 -1 0.037000 -1 1\n", "... ... ... ... ... ...\n", "26491438 6401 -1 0.048927 -1 4\n", "26491439 6402 -1 0.417940 -1 4\n", "26491440 6403 -1 0.161154 -1 4\n", "26491441 6404 -1 0.041714 -1 4\n", "26491442 6405 -1 0.054722 -1 4\n", "26491443 6406 -1 0.021913 -1 4\n", "26491444 6407 -1 0.026895 -1 4\n", "26491447 6408 -1 0.058602 -1 4\n", "26491448 6409 -1 0.008037 -1 4\n", "26491449 6410 -1 0.003291 -1 4\n", "26491452 6411 -1 0.157633 -1 4\n", "26491453 6412 -1 0.073751 -1 4\n", "26491454 6413 -1 0.017260 -1 4\n", "26491455 6414 -1 0.009815 -1 4\n", "26491456 6415 -1 0.146647 -1 4\n", "26491457 6416 -1 0.016393 -1 4\n", "26491459 6417 -1 0.008133 -1 4\n", "26491460 6418 -1 0.072313 -1 4\n", "26491461 6419 -1 0.172497 -1 4\n", "26491462 6420 -1 0.140382 -1 4\n", "26491463 6421 -1 0.185595 -1 4\n", "26491465 6422 -1 0.012733 -1 4\n", "26491466 6423 -1 0.186255 -1 4\n", "26491467 6424 -1 0.032360 -1 4\n", "26491472 6425 -1 0.015105 -1 4\n", "26491473 6426 -1 0.162567 -1 4\n", "26491474 6427 -1 0.015464 -1 4\n", "26491475 6428 -1 0.430372 -1 4\n", "26491476 6429 -1 0.490200 -1 4\n", "26491477 6430 -1 0.010446 -1 4\n", "\n", "[6431 rows x 5 columns]" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "predictions_df" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "-1.0" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "predictions_df.loc[19628035][\"label\"]" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "split_to_analyze = 4" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Prediction missing for 26490496\n", "Prediction missing for 26491220\n" ] } ], "source": [ "fp = list()\n", "fn = list()\n", "tp = list()\n", "tn = list()\n", "\n", "for c in labeled_cands:\n", " if c.split==split_to_analyze:\n", " try:\n", " d = predictions_df.loc[c.id]\n", " true,pred = d[\"label\"],d[\"pred\"]\n", " if true==1:\n", " if pred==1:\n", " tp.append(c)\n", " else:\n", " fn.append(c)\n", " else:\n", " if pred==-1:\n", " tn.append(c)\n", " else:\n", " fp.append(c)\n", " \n", " except:\n", " print(\"Prediction missing for {}\".format(c.id))\n", " " ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "from snorkel.viewer import SentenceNgramViewer" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "data": { "application/javascript": [ "require.undef('viewer');\n", "\n", "// NOTE: all elements should be selected using this.$el.find to avoid collisions with other Viewers\n", "\n", "define('viewer', [\"@jupyter-widgets/base\"], function(widgets) {\n", " var ViewerView = widgets.DOMWidgetView.extend({\n", " render: function() {\n", " this.cids = this.model.get('cids');\n", " this.nPages = this.cids.length;\n", " this.pid = 0;\n", " this.cxid = 0;\n", " this.cid = 0;\n", "\n", " // Insert the html payload\n", " this.$el.append(this.model.get('html'));\n", "\n", " // Initialize all labels from previous sessions\n", " this.labels = this.deserializeDict(this.model.get('_labels_serialized'));\n", " for (var i=0; i < this.nPages; i++) {\n", " this.pid = i;\n", " for (var j=0; j < this.cids[i].length; j++) {\n", " this.cxid = j;\n", " for (var k=0; k < this.cids[i][j].length; k++) {\n", " this.cid = k;\n", " if (this.cids[i][j][k] in this.labels) {\n", " this.markCurrentCandidate(false);\n", " }\n", " }\n", " }\n", " }\n", " this.pid = 0;\n", " this.cxid = 0;\n", " this.cid = 0;\n", "\n", " // Enable button functionality for navigation\n", " var that = this;\n", " this.$el.find(\"#next-cand\").click(function() {\n", " that.switchCandidate(1);\n", " });\n", " this.$el.find(\"#prev-cand\").click(function() {\n", " that.switchCandidate(-1);\n", " });\n", " this.$el.find(\"#next-context\").click(function() {\n", " that.switchContext(1);\n", " });\n", " this.$el.find(\"#prev-context\").click(function() {\n", " that.switchContext(-1);\n", " });\n", " this.$el.find(\"#next-page\").click(function() {\n", " that.switchPage(1);\n", " });\n", " this.$el.find(\"#prev-page\").click(function() {\n", " that.switchPage(-1);\n", " });\n", " this.$el.find(\"#label-true\").click(function() {\n", " that.labelCandidate(true, true);\n", " });\n", " this.$el.find(\"#label-false\").click(function() {\n", " that.labelCandidate(false, true);\n", " });\n", "\n", " // Arrow key functionality\n", " this.$el.keydown(function(e) {\n", " switch(e.which) {\n", " case 74: // j\n", " that.switchCandidate(-1);\n", " break;\n", "\n", " case 73: // i\n", " that.switchPage(-1);\n", " break;\n", "\n", " case 76: // l\n", " that.switchCandidate(1);\n", " break;\n", "\n", " case 75: // k\n", " that.switchPage(1);\n", " break;\n", "\n", " case 84: // t\n", " that.labelCandidate(true, true);\n", " break;\n", "\n", " case 70: // f\n", " that.labelCandidate(false, true);\n", " break;\n", " }\n", " });\n", "\n", " // Show the first page and highlight the first candidate\n", " this.$el.find(\"#viewer-page-0\").show();\n", " this.switchCandidate(0);\n", " },\n", "\n", " // Get candidate selector for currently selected candidate, escaping id properly\n", " getCandidate: function() {\n", " return this.$el.find(\".\"+this.cids[this.pid][this.cxid][this.cid]);\n", " }, \n", "\n", " // Color the candidate correctly according to registered label, as well as set highlighting\n", " markCurrentCandidate: function(highlight) {\n", " var cid = this.cids[this.pid][this.cxid][this.cid];\n", " var tags = this.$el.find(\".\"+cid);\n", "\n", " // Clear color classes\n", " tags.removeClass(\"candidate-h\");\n", " tags.removeClass(\"true-candidate\");\n", " tags.removeClass(\"true-candidate-h\");\n", " tags.removeClass(\"false-candidate\");\n", " tags.removeClass(\"false-candidate-h\");\n", " tags.removeClass(\"highlighted\");\n", "\n", " if (highlight) {\n", " if (cid in this.labels) {\n", " tags.addClass(String(this.labels[cid]) + \"-candidate-h\");\n", " } else {\n", " tags.addClass(\"candidate-h\");\n", " }\n", " \n", " // If un-highlighting, leave with first non-null coloring\n", " } else {\n", " var that = this;\n", " tags.each(function() {\n", " var cids = $(this).attr('class').split(/\\s+/).map(function(item) {\n", " return parseInt(item);\n", " });\n", " cids.sort();\n", " for (var i in cids) {\n", " if (cids[i] in that.labels) {\n", " var label = that.labels[cids[i]];\n", " $(this).addClass(String(label) + \"-candidate\");\n", " $(this).removeClass(String(!label) + \"-candidate\");\n", " break;\n", " }\n", " }\n", " });\n", " }\n", "\n", " // Extra highlighting css\n", " if (highlight) {\n", " tags.addClass(\"highlighted\");\n", " }\n", "\n", " // Classes for showing direction of relation\n", " if (highlight) {\n", " this.$el.find(\".\"+cid+\"-0\").addClass(\"left-candidate\");\n", " this.$el.find(\".\"+cid+\"-1\").addClass(\"right-candidate\");\n", " } else {\n", " this.$el.find(\".\"+cid+\"-0\").removeClass(\"left-candidate\");\n", " this.$el.find(\".\"+cid+\"-1\").removeClass(\"right-candidate\");\n", " }\n", " },\n", "\n", " // Cycle through candidates and highlight, by increment inc\n", " switchCandidate: function(inc) {\n", " var N = this.cids[this.pid].length\n", " var M = this.cids[this.pid][this.cxid].length;\n", " if (N == 0 || M == 0) { return false; }\n", "\n", " // Clear highlighting from previous candidate\n", " if (inc != 0) {\n", " this.markCurrentCandidate(false);\n", "\n", " // Increment the cid counter\n", "\n", " // Move to next context\n", " if (this.cid + inc >= M) {\n", " while (this.cid + inc >= M) {\n", " \n", " // At last context on page, halt\n", " if (this.cxid == N - 1) {\n", " this.cid = M - 1;\n", " inc = 0;\n", " break;\n", " \n", " // Increment to next context\n", " } else {\n", " inc -= M - this.cid;\n", " this.cxid += 1;\n", " M = this.cids[this.pid][this.cxid].length;\n", " this.cid = 0;\n", " }\n", " }\n", "\n", " // Move to previous context\n", " } else if (this.cid + inc < 0) {\n", " while (this.cid + inc < 0) {\n", " \n", " // At first context on page, halt\n", " if (this.cxid == 0) {\n", " this.cid = 0;\n", " inc = 0;\n", " break;\n", " \n", " // Increment to previous context\n", " } else {\n", " inc += this.cid + 1;\n", " this.cxid -= 1;\n", " M = this.cids[this.pid][this.cxid].length;\n", " this.cid = M - 1;\n", " }\n", " }\n", " }\n", "\n", " // Move within current context\n", " this.cid += inc;\n", " }\n", " this.markCurrentCandidate(true);\n", "\n", " // Push this new cid to the model\n", " this.model.set('_selected_cid', this.cids[this.pid][this.cxid][this.cid]);\n", " this.touch();\n", " },\n", "\n", " // Switch through contexts\n", " switchContext: function(inc) {\n", " this.markCurrentCandidate(false);\n", "\n", " // Iterate context on this page\n", " var M = this.cids[this.pid].length;\n", " if (this.cxid + inc < 0) {\n", " this.cxid = 0;\n", " } else if (this.cxid + inc >= M) {\n", " this.cxid = M - 1;\n", " } else {\n", " this.cxid += inc;\n", " }\n", "\n", " // Reset cid and set to first candidate\n", " this.cid = 0;\n", " this.switchCandidate(0);\n", " },\n", "\n", " // Switch through pages\n", " switchPage: function(inc) {\n", " this.markCurrentCandidate(false);\n", " this.$el.find(\".viewer-page\").hide();\n", " if (this.pid + inc < 0) {\n", " this.pid = 0;\n", " } else if (this.pid + inc > this.nPages - 1) {\n", " this.pid = this.nPages - 1;\n", " } else {\n", " this.pid += inc;\n", " }\n", " this.$el.find(\"#viewer-page-\"+this.pid).show();\n", "\n", " // Show pagination\n", " this.$el.find(\"#page\").html(this.pid);\n", "\n", " // Reset cid and set to first candidate\n", " this.cid = 0;\n", " this.cxid = 0;\n", " this.switchCandidate(0);\n", " },\n", "\n", " // Label currently-selected candidate\n", " labelCandidate: function(label, highlighted) {\n", " var c = this.getCandidate();\n", " var cid = this.cids[this.pid][this.cxid][this.cid];\n", " var cl = String(label) + \"-candidate\";\n", " var clh = String(label) + \"-candidate-h\";\n", " var cln = String(!label) + \"-candidate\";\n", " var clnh = String(!label) + \"-candidate-h\";\n", "\n", " // Toggle label highlighting\n", " if (c.hasClass(cl) || c.hasClass(clh)) {\n", " c.removeClass(cl);\n", " c.removeClass(clh);\n", " if (highlighted) {\n", " c.addClass(\"candidate-h\");\n", " }\n", " this.labels[cid] = null;\n", " this.send({event: 'delete_label', cid: cid});\n", " } else {\n", " c.removeClass(cln);\n", " c.removeClass(clnh);\n", " if (highlighted) {\n", " c.addClass(clh);\n", " } else {\n", " c.addClass(cl);\n", " }\n", " this.labels[cid] = label;\n", " this.send({event: 'set_label', cid: cid, value: label});\n", " }\n", "\n", " // Set the label and pass back to the model\n", " this.model.set('_labels_serialized', this.serializeDict(this.labels));\n", " this.touch();\n", " },\n", "\n", " // Serialization of hash maps, because traitlets Dict doesn't seem to work...\n", " serializeDict: function(d) {\n", " var s = [];\n", " for (var key in d) {\n", " s.push(key+\"~~\"+d[key]);\n", " }\n", " return s.join();\n", " },\n", "\n", " // Deserialization of hash maps\n", " deserializeDict: function(s) {\n", " var d = {};\n", " var entries = s.split(/,/);\n", " var kv;\n", " for (var i in entries) {\n", " kv = entries[i].split(/~~/);\n", " if (kv[1] == \"true\") {\n", " d[kv[0]] = true;\n", " } else if (kv[1] == \"false\") {\n", " d[kv[0]] = false;\n", " }\n", " }\n", " return d;\n", " },\n", " });\n", "\n", " return {\n", " ViewerView: ViewerView\n", " };\n", "});\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "sv = SentenceNgramViewer(annotator_name=\"gold\",candidates=fp,session=sess)" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "6a045cc41ce84a82ad85b729f80486cd", "version_major": 2, "version_minor": 0 }, "text/html": [ "

Failed to display Jupyter Widget of type SentenceNgramViewer.

\n", "

\n", " If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n", " that the widgets JavaScript is still loading. If this message persists, it\n", " likely means that the widgets JavaScript library is either not installed or\n", " not enabled. See the Jupyter\n", " Widgets Documentation for setup instructions.\n", "

\n", "

\n", " If you're reading this message in another frontend (for example, a static\n", " rendering on GitHub or NBViewer),\n", " it may mean that your frontend doesn't currently support widgets.\n", "

\n" ], "text/plain": [ "SentenceNgramViewer(cids=[[[11, 12, 13, 15], [3], [5]], [[27, 29], [6, 7, 8, 9, 10], [4]], [[14, 16, 22], [17, 18, 19, 20, 21, 25, 26, 28], [0]], [[23, 24], [1, 2]]], html=u'\\n\\n\\n\\n
\\n
\\n

Viewer

\\n
\\n
\\n
\\n
\\n
    \\n
  • These results indicate that Caa67(YL) catalyzes the hydration of 2-CAA to form 2-chloro-2-hydroxypropionate, which is chemically unstable and probably spontaneously dechlorinated to form pyruvate. 2-Bromoacrylate, but not other 2-CAA analogs such as acrylate and methacrylate, served as the substrate of Caa67(YL).
    • \\n\\n
  • The enzyme phenylalanine ammonia-lyase, which catalyzes the nonoxidative deamination of l-phenylalanine to trans-cinnamic acid, is ubiquitously distributed in plants.
    • \\n\\n
  • strain CNB-1 grows on 4-chloronitrobenzene (4-CNB) and nitrobenzene as sole carbon and nitrogen sources.
    • \\n
\\n
\\n\\n
\\n
    \\n
  • TG1/pBS(Kan)T3MO produced 66% p-nitrophenol and 34% m-nitrophenol from nitrobenzene and 100% p-methoxyphenol from methoxybenzene, as well as 62% 1-naphthol and 38% 2-naphthol from naphthalene; similar results were found with TG1/pBS(Kan)T4MO.
    • \\n\\n
  • Strain R4 could completely degrade 100 mg l(-1) TEA to ammonia in 32 h, and could also effectively degrade diethylamine (DEA) and ethylamine (EA) to ammonia.
    • \\n\\n
  • The catabolism of MTA in P. aeruginosa involves deamination to MTI and phosphorolysis to hypoxanthine (MTA > MTI > hypoxanthine).
    • \\n
\\n
\\n\\n
\\n
    \\n
  • Studies with cell extracts reveal the presence of inducible dehydrogenases for cyclohexanol, 6-hydroxyhexanoate and 6-oxohexanoate and a monooxygenase, that in conjunction with a lactonase converts cyclohexanone to 6-hydroxyhexanoate.
    • \\n\\n
  • The monooxygenase is therefore presumed to be of the lactone-forming type and the pathway for conversion of cyclohexanol to adipate; cyclohexanol leads to cyclohexanone leads to 1-oxa-2-oxocycloheptane leads to 6-hydroxyhexanoate leads to 6-oxohexanoate leads to adipate; for which key intermediates have been identified chromatographically, is identical with the route for the oxidation of cyclohexanol by Nocardia globerula CL1.
    • \\n\\n
  • Biotin is an enzyme cofactor indispensable to metabolic fixation of carbon dioxide in all three domains of life.
    • \\n
\\n
\\n\\n
\\n
    \\n
  • The E. coli BioC-BioH pathway employs a methylation and demethylation strategy to allow elongation of a temporarily disguised malonate moiety to a pimelate moiety by the fatty acid synthetic enzymes whereas the B. subtilis BioI-BioW pathway utilizes oxidative cleavage of fatty acyl chains.
    • \\n\\n
  • Glyoxylate biosynthesis in Saccharomyces cerevisiae is traditionally mainly ascribed to the reaction catalyzed by isocitrate lyase (Icl), which converts isocitrate to glyoxylate and succinate.
    • \\n
\\n
\\n
\\n
\\n
\\n \\n \\n \\n\\n \\n \\n\\n \\n \\n\\n \\n \\n\\n \\n \\n\\n \\n \\n
\\n
\\n \\n \\n
\\n    \\n Viewer must be focused for keyboard control\\n
Page 0
\\n
\\n
\\n')" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "sv" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(30, 30, 19, 1131)" ] }, "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(fp),len(tp),len(fn),len(tn)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Per document analysis\n" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Prediction missing for 15462889\n", "Prediction missing for 26486336\n", "Prediction missing for 26486496\n", "Prediction missing for 26486893\n", "Prediction missing for 26487473\n", "Prediction missing for 20185660\n", "Prediction missing for 17873864\n", "Prediction missing for 17873876\n", "Prediction missing for 17873861\n", "Prediction missing for 17873867\n", "Prediction missing for 17873887\n", "Prediction missing for 17873886\n", "Prediction missing for 20528367\n", "Prediction missing for 20528348\n", "Prediction missing for 20528355\n", "Prediction missing for 20528333\n", "Prediction missing for 20528329\n", "Prediction missing for 20528349\n", "Prediction missing for 20528350\n", "Prediction missing for 20528361\n", "Prediction missing for 20528347\n", "Prediction missing for 20528336\n", "Prediction missing for 20528343\n", "Prediction missing for 20528345\n", "Prediction missing for 20528339\n", "Prediction missing for 20528330\n", "Prediction missing for 20528358\n", "Prediction missing for 20528332\n", "Prediction missing for 20528363\n", "Prediction missing for 20528364\n", "Prediction missing for 20528326\n", "Prediction missing for 20528359\n", "Prediction missing for 20528352\n", "Prediction missing for 20528338\n", "Prediction missing for 20528331\n", "Prediction missing for 20528360\n", "Prediction missing for 20528328\n", "Prediction missing for 20528341\n", "Prediction missing for 18984449\n", "Prediction missing for 18984412\n", "Prediction missing for 18984422\n", "Prediction missing for 18984447\n", "Prediction missing for 18984371\n", "Prediction missing for 18984431\n", "Prediction missing for 18984424\n", "Prediction missing for 18984405\n", "Prediction missing for 18984415\n", "Prediction missing for 18984438\n", "Prediction missing for 18984385\n", "Prediction missing for 18984457\n", "Prediction missing for 18984419\n", "Prediction missing for 18984437\n", "Prediction missing for 18984460\n", "Prediction missing for 18984421\n", "Prediction missing for 18984377\n", "Prediction missing for 18984389\n", "Prediction missing for 18984390\n", "Prediction missing for 18984373\n", "Prediction missing for 18984383\n", "Prediction missing for 18984436\n", "Prediction missing for 18984451\n", "Prediction missing for 18984416\n", "Prediction missing for 18984450\n", "Prediction missing for 18984403\n", "Prediction missing for 18984391\n", "Prediction missing for 18984430\n", "Prediction missing for 18984402\n", "Prediction missing for 18984417\n", "Prediction missing for 18984455\n", "Prediction missing for 18984378\n", "Prediction missing for 18984418\n", "Prediction missing for 18984425\n", "Prediction missing for 18984426\n", "Prediction missing for 18984410\n", "Prediction missing for 18984459\n", "Prediction missing for 18984452\n", "Prediction missing for 18984406\n", "Prediction missing for 18984445\n", "Prediction missing for 18984393\n", "Prediction missing for 18984441\n", "Prediction missing for 18984443\n", "Prediction missing for 18984407\n", "Prediction missing for 18984420\n", "Prediction missing for 18984398\n", "Prediction missing for 18984439\n", "Prediction missing for 18984432\n", "Prediction missing for 18984382\n", "Prediction missing for 18984396\n", "Prediction missing for 18984442\n", "Prediction missing for 18984458\n", "Prediction missing for 18984434\n", "Prediction missing for 18984435\n", "Prediction missing for 18984404\n", "Prediction missing for 18984379\n", "Prediction missing for 18984433\n", "Prediction missing for 18984386\n", "Prediction missing for 18984428\n", "Prediction missing for 18984456\n", "Prediction missing for 18984399\n", "Prediction missing for 18984429\n", "Prediction missing for 18984374\n", "Prediction missing for 18984387\n", "Prediction missing for 18984400\n", "Prediction missing for 18984384\n", "Prediction missing for 18984411\n", "Prediction missing for 18984448\n", "Prediction missing for 18984401\n", "Prediction missing for 18984381\n", "Prediction missing for 18984394\n", "Prediction missing for 18984444\n", "Prediction missing for 18984440\n", "Prediction missing for 18984372\n", "Prediction missing for 18984461\n", "Prediction missing for 18984446\n", "Prediction missing for 18984462\n", "Prediction missing for 18984395\n", "Prediction missing for 18984427\n", "Prediction missing for 18984408\n", "Prediction missing for 18984392\n", "Prediction missing for 18984413\n", "Prediction missing for 18984397\n", "Prediction missing for 18984409\n", "Prediction missing for 18984376\n", "Prediction missing for 18984517\n", "Prediction missing for 18984492\n", "Prediction missing for 18984504\n", "Prediction missing for 18984481\n", "Prediction missing for 18984508\n", "Prediction missing for 18984518\n", "Prediction missing for 18984482\n", "Prediction missing for 18984520\n", "Prediction missing for 18984483\n", "Prediction missing for 18984480\n", "Prediction missing for 18984527\n", "Prediction missing for 18984516\n", "Prediction missing for 18984555\n", "Prediction missing for 18984487\n", "Prediction missing for 18984499\n", "Prediction missing for 18984484\n", "Prediction missing for 18984528\n", "Prediction missing for 18984495\n", "Prediction missing for 18984530\n", "Prediction missing for 18984523\n", "Prediction missing for 18984564\n", "Prediction missing for 18984485\n", "Prediction missing for 18984571\n", "Prediction missing for 18984505\n", "Prediction missing for 18984557\n", "Prediction missing for 18984572\n", "Prediction missing for 18984581\n", "Prediction missing for 18984535\n", "Prediction missing for 18984513\n", "Prediction missing for 18984494\n", "Prediction missing for 18984478\n", "Prediction missing for 18984506\n", "Prediction missing for 18984488\n", "Prediction missing for 18984503\n", "Prediction missing for 18984534\n", "Prediction missing for 18984479\n", "Prediction missing for 18984498\n", "Prediction missing for 18984507\n", "Prediction missing for 18984510\n", "Prediction missing for 18984493\n", "Prediction missing for 18984546\n", "Prediction missing for 18984547\n", "Prediction missing for 18984521\n", "Prediction missing for 18984522\n", "Prediction missing for 18984497\n", "Prediction missing for 18984531\n", "Prediction missing for 18984512\n", "Prediction missing for 18984551\n", "Prediction missing for 18984560\n", "Prediction missing for 18984578\n", "Prediction missing for 18984541\n", "Prediction missing for 18984573\n", "Prediction missing for 18984491\n", "Prediction missing for 18984569\n", "Prediction missing for 18984583\n", "Prediction missing for 18984501\n", "Prediction missing for 18984533\n", "Prediction missing for 18984496\n", "Prediction missing for 18984565\n", "Prediction missing for 18984544\n", "Prediction missing for 18984529\n", "Prediction missing for 18984525\n", "Prediction missing for 18984486\n", "Prediction missing for 18984515\n", "Prediction missing for 18984563\n", "Prediction missing for 18984519\n", "Prediction missing for 18984537\n", "Prediction missing for 18984500\n", "Prediction missing for 18984502\n", "Prediction missing for 18984576\n", "Prediction missing for 18984567\n", "Prediction missing for 18984575\n", "Prediction missing for 18984580\n", "Prediction missing for 18984549\n", "Prediction missing for 18984511\n", "Prediction missing for 18984536\n", "Prediction missing for 18984585\n", "Prediction missing for 18984489\n", "Prediction missing for 18984561\n", "Prediction missing for 18984526\n", "Prediction missing for 18984477\n", "Prediction missing for 18984509\n", "Prediction missing for 18984514\n", "Prediction missing for 18984524\n", "Prediction missing for 18984532\n", "Prediction missing for 18984588\n", "Prediction missing for 18984540\n", "Prediction missing for 18984559\n", "Prediction missing for 18984543\n", "Prediction missing for 18984586\n", "Prediction missing for 18984545\n", "Prediction missing for 18984490\n", "Prediction missing for 18984553\n", "Prediction missing for 18984570\n", "Prediction missing for 18984613\n", "Prediction missing for 18984635\n", "Prediction missing for 18984632\n", "Prediction missing for 18984640\n", "Prediction missing for 18984647\n", "Prediction missing for 18984617\n", "Prediction missing for 18984636\n", "Prediction missing for 18984624\n", "Prediction missing for 18984620\n", "Prediction missing for 18984625\n", "Prediction missing for 18984616\n", "Prediction missing for 18984644\n", "Prediction missing for 18984612\n", "Prediction missing for 18984641\n", "Prediction missing for 18984629\n", "Prediction missing for 18984606\n", "Prediction missing for 18984628\n", "Prediction missing for 18984607\n", "Prediction missing for 18984608\n", "Prediction missing for 18984634\n", "Prediction missing for 18984615\n", "Prediction missing for 18984611\n", "Prediction missing for 18984645\n", "Prediction missing for 18984631\n", "Prediction missing for 18984638\n", "Prediction missing for 18984605\n", "Prediction missing for 18984639\n", "Prediction missing for 18984623\n", "Prediction missing for 18984627\n", "Prediction missing for 18984642\n", "Prediction missing for 18984626\n", "Prediction missing for 18984621\n", "Prediction missing for 18984646\n", "Prediction missing for 18984630\n", "Prediction missing for 18984633\n", "Prediction missing for 18984610\n", "Prediction missing for 18984648\n", "Prediction missing for 18984609\n", "Prediction missing for 18984614\n", "Prediction missing for 18984643\n", "Prediction missing for 18984618\n", "Prediction missing for 18984622\n", "Prediction missing for 17645816\n", "Prediction missing for 17645799\n", "Prediction missing for 17645820\n", "Prediction missing for 17645813\n", "Prediction missing for 17645803\n", "Prediction missing for 17645817\n", "Prediction missing for 17645796\n", "Prediction missing for 17645808\n", "Prediction missing for 17645802\n", "Prediction missing for 20985818\n", "Prediction missing for 20985827\n", "Prediction missing for 16099164\n", "Prediction missing for 16099165\n", "Prediction missing for 16417650\n", "Prediction missing for 16417628\n", "Prediction missing for 16417609\n", "Prediction missing for 16417608\n", "Prediction missing for 16417624\n", "Prediction missing for 16417622\n", "Prediction missing for 16417654\n", "Prediction missing for 16417653\n", "Prediction missing for 16417638\n", "Prediction missing for 16417660\n", "Prediction missing for 16417623\n", "Prediction missing for 16417661\n", "Prediction missing for 16417615\n", "Prediction missing for 16419693\n", "Prediction missing for 16419676\n", "Prediction missing for 16419721\n", "Prediction missing for 16419712\n", "Prediction missing for 16419665\n", "Prediction missing for 16419664\n", "Prediction missing for 16419686\n", "Prediction missing for 16419669\n", "Prediction missing for 16419685\n", "Prediction missing for 16419662\n", "Prediction missing for 16419708\n", "Prediction missing for 16419687\n", "Prediction missing for 16419688\n", "Prediction missing for 16419663\n", "Prediction missing for 16419719\n", "Prediction missing for 16419699\n", "Prediction missing for 16419705\n", "Prediction missing for 16419728\n", "Prediction missing for 16419723\n", "Prediction missing for 16419656\n", "Prediction missing for 16419696\n", "Prediction missing for 16419724\n", "Prediction missing for 16419670\n", "Prediction missing for 16419690\n", "Prediction missing for 16419706\n", "Prediction missing for 16419715\n", "Prediction missing for 16419717\n", "Prediction missing for 16419659\n", "Prediction missing for 16419697\n", "Prediction missing for 16419702\n", "Prediction missing for 16419661\n", "Prediction missing for 16419698\n", "Prediction missing for 16419707\n", "Prediction missing for 16419679\n", "Prediction missing for 16419680\n", "Prediction missing for 16419678\n", "Prediction missing for 16419675\n", "Prediction missing for 16419672\n", "Prediction missing for 16419674\n", "Prediction missing for 16419704\n", "Prediction missing for 16419684\n", "Prediction missing for 16419695\n", "Prediction missing for 16419714\n", "Prediction missing for 16419709\n", "Prediction missing for 16419671\n", "Prediction missing for 16419668\n", "Prediction missing for 16419727\n", "Prediction missing for 16419710\n", "Prediction missing for 16419692\n", "Prediction missing for 16419716\n", "Prediction missing for 16419658\n", "Prediction missing for 16419718\n", "Prediction missing for 16419691\n", "Prediction missing for 16419660\n", "Prediction missing for 16419673\n", "Prediction missing for 16419666\n", "Prediction missing for 16419700\n", "Prediction missing for 16419725\n", "Prediction missing for 16419682\n", "Prediction missing for 20203770\n", "Prediction missing for 20203761\n", "Prediction missing for 20203780\n", "Prediction missing for 20203783\n", "Prediction missing for 20203786\n", "Prediction missing for 20203766\n", "Prediction missing for 20203775\n", "Prediction missing for 20203778\n", "Prediction missing for 20203779\n", "Prediction missing for 20203768\n", "Prediction missing for 20203773\n", "Prediction missing for 22218563\n", "Prediction missing for 22218570\n", "Prediction missing for 22218569\n", "Prediction missing for 22218547\n", "Prediction missing for 22218542\n", "Prediction missing for 22218554\n", "Prediction missing for 22218537\n", "Prediction missing for 22218557\n", "Prediction missing for 22218548\n", "Prediction missing for 22218540\n", "Prediction missing for 26228975\n", "Prediction missing for 26229002\n", "Prediction missing for 26228994\n", "Prediction missing for 26228988\n", "Prediction missing for 26229007\n", "Prediction missing for 26229014\n", "Prediction missing for 26229005\n", "Prediction missing for 26228982\n", "Prediction missing for 26228974\n", "Prediction missing for 26228979\n", "Prediction missing for 26229021\n", "Prediction missing for 26229016\n", "Prediction missing for 26229006\n", "Prediction missing for 26228978\n", "Prediction missing for 26228969\n", "Prediction missing for 26228983\n", "Prediction missing for 26228998\n", "Prediction missing for 26229003\n", "Prediction missing for 26229000\n", "Prediction missing for 26229008\n", "Prediction missing for 26228985\n", "Prediction missing for 26228980\n", "Prediction missing for 26228971\n", "Prediction missing for 26228970\n", "Prediction missing for 15348145\n", "Prediction missing for 15348146\n", "Prediction missing for 15348148\n", "Prediction missing for 15348150\n", "Prediction missing for 15348151\n", "Prediction missing for 15392561\n", "Prediction missing for 15392563\n", "Prediction missing for 15392564\n", "Prediction missing for 15392565\n", "Prediction missing for 15392566\n", "Prediction missing for 15433160\n", "Prediction missing for 15433166\n", "Prediction missing for 15433167\n", "Prediction missing for 15433168\n", "Prediction missing for 15433170\n", "Prediction missing for 15433171\n", "Prediction missing for 15433172\n", "Prediction missing for 15433173\n", "Prediction missing for 15433174\n", "Prediction missing for 15433175\n", "Prediction missing for 15433176\n", "Prediction missing for 15433177\n", "Prediction missing for 15433178\n", "Prediction missing for 15513630\n", "Prediction missing for 15513631\n", "Prediction missing for 15538620\n", "Prediction missing for 15543611\n", "Prediction missing for 15543613\n", "Prediction missing for 15924994\n", "Prediction missing for 15924995\n", "Prediction missing for 15925037\n", "Prediction missing for 15925039\n", "Prediction missing for 15925159\n", "Prediction missing for 15925161\n", "Prediction missing for 15925329\n", "Prediction missing for 15925330\n", "Prediction missing for 15925436\n", "Prediction missing for 15925437\n", "Prediction missing for 15925438\n", "Prediction missing for 15925439\n", "Prediction missing for 15925441\n", "Prediction missing for 15925443\n", "Prediction missing for 15925444\n", "Prediction missing for 15925445\n", "Prediction missing for 15925446\n", "Prediction missing for 15925447\n", "Prediction missing for 15925448\n", "Prediction missing for 15925449\n", "Prediction missing for 15992123\n", "Prediction missing for 15992124\n", "Prediction missing for 15992125\n", "Prediction missing for 15992126\n", "Prediction missing for 15992128\n", "Prediction missing for 15992385\n", "Prediction missing for 15992387\n", "Prediction missing for 15992388\n", "Prediction missing for 15992389\n", "Prediction missing for 15992390\n", "Prediction missing for 15992391\n", "Prediction missing for 16031018\n", "Prediction missing for 16031019\n", "Prediction missing for 16031083\n", "Prediction missing for 16031085\n", "Prediction missing for 16031086\n", "Prediction missing for 16031088\n", "Prediction missing for 16031089\n", "Prediction missing for 16031090\n", "Prediction missing for 16097954\n", "Prediction missing for 16097955\n", "Prediction missing for 16097956\n", "Prediction missing for 16097957\n", "Prediction missing for 16097958\n", "Prediction missing for 16097959\n", "Prediction missing for 16097960\n", "Prediction missing for 16097961\n", "Prediction missing for 16097962\n", "Prediction missing for 16097964\n", "Prediction missing for 16097965\n", "Prediction missing for 16097966\n", "Prediction missing for 16097967\n", "Prediction missing for 16097968\n", "Prediction missing for 16097969\n", "Prediction missing for 16097970\n", "Prediction missing for 16097972\n", "Prediction missing for 16097973\n", "Prediction missing for 16097974\n", "Prediction missing for 16099148\n", "Prediction missing for 16099149\n", "Prediction missing for 16099150\n", "Prediction missing for 16099151\n", "Prediction missing for 16099152\n", "Prediction missing for 16099154\n", "Prediction missing for 16099155\n", "Prediction missing for 16099156\n", "Prediction missing for 16099157\n", "Prediction missing for 16099158\n", "Prediction missing for 16099160\n", "Prediction missing for 16099161\n", "Prediction missing for 16099162\n", "Prediction missing for 16099163\n", "Prediction missing for 16099166\n", "Prediction missing for 16099167\n", "Prediction missing for 16099168\n", "Prediction missing for 16123151\n", "Prediction missing for 16123153\n", "Prediction missing for 16134482\n", "Prediction missing for 16134483\n", "Prediction missing for 16134484\n", "Prediction missing for 16134486\n", "Prediction missing for 16134487\n", "Prediction missing for 16134488\n", "Prediction missing for 16147750\n", "Prediction missing for 16147753\n", "Prediction missing for 16147754\n", "Prediction missing for 16147755\n", "Prediction missing for 16147756\n", "Prediction missing for 16147757\n", "Prediction missing for 16212569\n", "Prediction missing for 16315500\n", "Prediction missing for 16315501\n", "Prediction missing for 16315502\n", "Prediction missing for 16315503\n", "Prediction missing for 16315504\n", "Prediction missing for 16315506\n", "Prediction missing for 16315507\n", "Prediction missing for 16315508\n", "Prediction missing for 16315509\n", "Prediction missing for 16315510\n", "Prediction missing for 16315511\n", "Prediction missing for 16315512\n", "Prediction missing for 16315513\n", "Prediction missing for 16315515\n", "Prediction missing for 16315516\n", "Prediction missing for 16315517\n", "Prediction missing for 16315518\n", "Prediction missing for 16315519\n", "Prediction missing for 16315520\n", "Prediction missing for 16320198\n", "Prediction missing for 16320199\n", "Prediction missing for 16320200\n", "Prediction missing for 16320201\n", "Prediction missing for 16320202\n", "Prediction missing for 16320203\n", "Prediction missing for 16320204\n", "Prediction missing for 16320205\n", "Prediction missing for 16320206\n", "Prediction missing for 16320207\n", "Prediction missing for 16320209\n", "Prediction missing for 16320210\n", "Prediction missing for 16320211\n", "Prediction missing for 16320212\n", "Prediction missing for 16320213\n", "Prediction missing for 16320214\n", "Prediction missing for 16320215\n", "Prediction missing for 16320216\n", "Prediction missing for 16320217\n", "Prediction missing for 16320218\n", "Prediction missing for 16320266\n", "Prediction missing for 16320267\n", "Prediction missing for 16320268\n", "Prediction missing for 16320269\n", "Prediction missing for 16320270\n", "Prediction missing for 16320271\n", "Prediction missing for 16320272\n", "Prediction missing for 16320274\n", "Prediction missing for 16320275\n", "Prediction missing for 16320276\n", "Prediction missing for 16320278\n", "Prediction missing for 16320279\n", "Prediction missing for 16388313\n", "Prediction missing for 16388315\n", "Prediction missing for 16388316\n", "Prediction missing for 16388317\n", "Prediction missing for 16388318\n", "Prediction missing for 16407998\n", "Prediction missing for 16407999\n", "Prediction missing for 16417605\n", "Prediction missing for 16417606\n", "Prediction missing for 16417607\n", "Prediction missing for 16417610\n", "Prediction missing for 16417611\n", "Prediction missing for 16417612\n", "Prediction missing for 16417613\n", "Prediction missing for 16417614\n", "Prediction missing for 16417616\n", "Prediction missing for 16417617\n", "Prediction missing for 16417618\n", "Prediction missing for 16417619\n", "Prediction missing for 16417620\n", "Prediction missing for 16417621\n", "Prediction missing for 16417625\n", "Prediction missing for 16417626\n", "Prediction missing for 16417627\n", "Prediction missing for 16417629\n", "Prediction missing for 16417630\n", "Prediction missing for 16417632\n", "Prediction missing for 16417633\n", "Prediction missing for 16417634\n", "Prediction missing for 16417635\n", "Prediction missing for 16417636\n", "Prediction missing for 16417637\n", "Prediction missing for 16417639\n", "Prediction missing for 16417640\n", "Prediction missing for 16417641\n", "Prediction missing for 16417642\n", "Prediction missing for 16417643\n", "Prediction missing for 16417644\n", "Prediction missing for 16417646\n", "Prediction missing for 16417647\n", "Prediction missing for 16417648\n", "Prediction missing for 16417649\n", "Prediction missing for 16417651\n", "Prediction missing for 16417652\n", "Prediction missing for 16417655\n", "Prediction missing for 16417656\n", "Prediction missing for 16417657\n", "Prediction missing for 16417658\n", "Prediction missing for 16417659\n", "Prediction missing for 16419667\n", "Prediction missing for 16419677\n", "Prediction missing for 16419683\n", "Prediction missing for 16419689\n", "Prediction missing for 16419694\n", "Prediction missing for 16419701\n", "Prediction missing for 16419703\n", "Prediction missing for 16419711\n", "Prediction missing for 16419713\n", "Prediction missing for 16419720\n", "Prediction missing for 16419722\n", "Prediction missing for 16419726\n", "Prediction missing for 16419729\n", "Prediction missing for 16454060\n", "Prediction missing for 16454061\n", "Prediction missing for 16596879\n", "Prediction missing for 16596880\n", "Prediction missing for 16599590\n", "Prediction missing for 16599591\n", "Prediction missing for 16599593\n", "Prediction missing for 16599594\n", "Prediction missing for 16599595\n", "Prediction missing for 16599596\n", "Prediction missing for 16599951\n", "Prediction missing for 16599953\n", "Prediction missing for 16693579\n", "Prediction missing for 16693580\n", "Prediction missing for 16693581\n", "Prediction missing for 16693582\n", "Prediction missing for 16693583\n", "Prediction missing for 16720011\n", "Prediction missing for 16720012\n", "Prediction missing for 16720013\n", "Prediction missing for 16720014\n", "Prediction missing for 16720015\n", "Prediction missing for 16720016\n", "Prediction missing for 16720017\n", "Prediction missing for 16720018\n", "Prediction missing for 16720019\n", "Prediction missing for 16720020\n", "Prediction missing for 16720021\n", "Prediction missing for 16720022\n", "Prediction missing for 16720023\n", "Prediction missing for 16720024\n", "Prediction missing for 16720025\n", "Prediction missing for 16720026\n", "Prediction missing for 16720027\n", "Prediction missing for 16720028\n", "Prediction missing for 16720029\n", "Prediction missing for 16720030\n", "Prediction missing for 16720035\n", "Prediction missing for 16720036\n", "Prediction missing for 16720107\n", "Prediction missing for 16720110\n", "Prediction missing for 16790421\n", "Prediction missing for 16823776\n", "Prediction missing for 16862988\n", "Prediction missing for 16862989\n", "Prediction missing for 16862990\n", "Prediction missing for 16862992\n", "Prediction missing for 16862993\n", "Prediction missing for 16863847\n", "Prediction missing for 16863848\n", "Prediction missing for 16863849\n", "Prediction missing for 16863852\n", "Prediction missing for 16863853\n", "Prediction missing for 16863854\n", "Prediction missing for 16882629\n", "Prediction missing for 16882801\n", "Prediction missing for 16882802\n", "Prediction missing for 16896429\n", "Prediction missing for 16896431\n", "Prediction missing for 16935845\n", "Prediction missing for 16935846\n", "Prediction missing for 16935847\n", "Prediction missing for 16935848\n", "Prediction missing for 16935850\n", "Prediction missing for 16935851\n", "Prediction missing for 16989896\n", "Prediction missing for 16989897\n", "Prediction missing for 16989900\n", "Prediction missing for 16989901\n", "Prediction missing for 16989902\n", "Prediction missing for 16989909\n", "Prediction missing for 16989910\n", "Prediction missing for 17050361\n", "Prediction missing for 17320203\n", "Prediction missing for 17320204\n", "Prediction missing for 17320206\n", "Prediction missing for 17320207\n", "Prediction missing for 17320208\n", "Prediction missing for 17320209\n", "Prediction missing for 17320210\n", "Prediction missing for 17320212\n", "Prediction missing for 17320213\n", "Prediction missing for 17320214\n", "Prediction missing for 17320215\n", "Prediction missing for 17369275\n", "Prediction missing for 17369276\n", "Prediction missing for 17369277\n", "Prediction missing for 17369278\n", "Prediction missing for 17369279\n", "Prediction missing for 17369280\n", "Prediction missing for 17369281\n", "Prediction missing for 17369282\n", "Prediction missing for 17369283\n", "Prediction missing for 17369284\n", "Prediction missing for 17369286\n", "Prediction missing for 17369288\n", "Prediction missing for 17490528\n", "Prediction missing for 17490529\n", "Prediction missing for 17490530\n", "Prediction missing for 17490532\n", "Prediction missing for 17490534\n", "Prediction missing for 17520365\n", "Prediction missing for 17523676\n", "Prediction missing for 17523677\n", "Prediction missing for 17523678\n", "Prediction missing for 17523680\n", "Prediction missing for 17523682\n", "Prediction missing for 17554668\n", "Prediction missing for 17554670\n", "Prediction missing for 17554671\n", "Prediction missing for 17554672\n", "Prediction missing for 17554673\n", "Prediction missing for 17554674\n", "Prediction missing for 17554693\n", "Prediction missing for 17554696\n", "Prediction missing for 17554749\n", "Prediction missing for 17554751\n", "Prediction missing for 17645791\n", "Prediction missing for 17645792\n", "Prediction missing for 17645793\n", "Prediction missing for 17645794\n", "Prediction missing for 17645797\n", "Prediction missing for 17645798\n", "Prediction missing for 17645801\n", "Prediction missing for 17645804\n", "Prediction missing for 17645805\n", "Prediction missing for 17645807\n", "Prediction missing for 17645809\n", "Prediction missing for 17645810\n", "Prediction missing for 17645811\n", "Prediction missing for 17645812\n", "Prediction missing for 17645814\n", "Prediction missing for 17645815\n", "Prediction missing for 17645818\n", "Prediction missing for 17645819\n", "Prediction missing for 17645821\n", "Prediction missing for 17645822\n", "Prediction missing for 17667331\n", "Prediction missing for 17667332\n", "Prediction missing for 17667333\n", "Prediction missing for 17667334\n", "Prediction missing for 17667335\n", "Prediction missing for 17667336\n", "Prediction missing for 17667337\n", "Prediction missing for 17667338\n", "Prediction missing for 17667339\n", "Prediction missing for 17667341\n", "Prediction missing for 17667342\n", "Prediction missing for 17721230\n", "Prediction missing for 17721233\n", "Prediction missing for 17721829\n", "Prediction missing for 17721830\n", "Prediction missing for 17767471\n", "Prediction missing for 17767472\n", "Prediction missing for 17873856\n", "Prediction missing for 17873857\n", "Prediction missing for 17873859\n", "Prediction missing for 17873860\n", "Prediction missing for 17873862\n", "Prediction missing for 17873863\n", "Prediction missing for 17873865\n", "Prediction missing for 17873866\n", "Prediction missing for 17873868\n", "Prediction missing for 17873870\n", "Prediction missing for 17873871\n", "Prediction missing for 17873872\n", "Prediction missing for 17873873\n", "Prediction missing for 17873874\n", "Prediction missing for 17873875\n", "Prediction missing for 17873984\n", "Prediction missing for 17873985\n", "Prediction missing for 17873986\n", "Prediction missing for 17873987\n", "Prediction missing for 17873989\n", "Prediction missing for 17873991\n", "Prediction missing for 17975113\n", "Prediction missing for 18029048\n", "Prediction missing for 18029049\n", "Prediction missing for 18029050\n", "Prediction missing for 18029051\n", "Prediction missing for 18029053\n", "Prediction missing for 18029837\n", "Prediction missing for 18029839\n", "Prediction missing for 18029840\n", "Prediction missing for 18029841\n", "Prediction missing for 18029842\n", "Prediction missing for 18029870\n", "Prediction missing for 18029872\n", "Prediction missing for 18029877\n", "Prediction missing for 18029880\n", "Prediction missing for 18029881\n", "Prediction missing for 18029882\n", "Prediction missing for 18029883\n", "Prediction missing for 18029884\n", "Prediction missing for 18029885\n", "Prediction missing for 18029886\n", "Prediction missing for 18029888\n", "Prediction missing for 18029890\n", "Prediction missing for 18029891\n", "Prediction missing for 18029892\n", "Prediction missing for 18036511\n", "Prediction missing for 18036514\n", "Prediction missing for 18036515\n", "Prediction missing for 18036516\n", "Prediction missing for 18036517\n", "Prediction missing for 18036518\n", "Prediction missing for 18148302\n", "Prediction missing for 18152407\n", "Prediction missing for 18152408\n", "Prediction missing for 18152409\n", "Prediction missing for 18152411\n", "Prediction missing for 18152412\n", "Prediction missing for 18152413\n", "Prediction missing for 18152414\n", "Prediction missing for 18152415\n", "Prediction missing for 18152416\n", "Prediction missing for 18152418\n", "Prediction missing for 18152419\n", "Prediction missing for 18152420\n", "Prediction missing for 18152421\n", "Prediction missing for 18152422\n", "Prediction missing for 18152423\n", "Prediction missing for 18152424\n", "Prediction missing for 18152425\n", "Prediction missing for 18152426\n", "Prediction missing for 18152427\n", "Prediction missing for 18152428\n", "Prediction missing for 18152429\n", "Prediction missing for 18152430\n", "Prediction missing for 18152431\n", "Prediction missing for 18152432\n", "Prediction missing for 18152433\n", "Prediction missing for 18152434\n", "Prediction missing for 18152435\n", "Prediction missing for 18152436\n", "Prediction missing for 18152437\n", "Prediction missing for 18183443\n", "Prediction missing for 18183444\n", "Prediction missing for 18183447\n", "Prediction missing for 18183448\n", "Prediction missing for 18183449\n", "Prediction missing for 18183450\n", "Prediction missing for 18194285\n", "Prediction missing for 18194286\n", "Prediction missing for 18228936\n", "Prediction missing for 18228937\n", "Prediction missing for 18228938\n", "Prediction missing for 18228939\n", "Prediction missing for 18228940\n", "Prediction missing for 18228941\n", "Prediction missing for 18228942\n", "Prediction missing for 18228943\n", "Prediction missing for 18228945\n", "Prediction missing for 18228946\n", "Prediction missing for 18228947\n", "Prediction missing for 18281837\n", "Prediction missing for 18281840\n", "Prediction missing for 18479479\n", "Prediction missing for 18479481\n", "Prediction missing for 18479483\n", "Prediction missing for 18479486\n", "Prediction missing for 18479507\n", "Prediction missing for 18479509\n", "Prediction missing for 18479513\n", "Prediction missing for 18479514\n", "Prediction missing for 18498808\n", "Prediction missing for 18498810\n", "Prediction missing for 18498811\n", "Prediction missing for 18498813\n", "Prediction missing for 18498814\n", "Prediction missing for 18498815\n", "Prediction missing for 18659673\n", "Prediction missing for 18679778\n", "Prediction missing for 18679779\n", "Prediction missing for 18679781\n", "Prediction missing for 18679783\n", "Prediction missing for 18679784\n", "Prediction missing for 18679785\n", "Prediction missing for 18679810\n", "Prediction missing for 18679812\n", "Prediction missing for 18679813\n", "Prediction missing for 18679814\n", "Prediction missing for 18679815\n", "Prediction missing for 18679816\n", "Prediction missing for 18679817\n", "Prediction missing for 18679818\n", "Prediction missing for 18679819\n", "Prediction missing for 18679820\n", "Prediction missing for 18679821\n", "Prediction missing for 18679822\n", "Prediction missing for 18934297\n", "Prediction missing for 18934330\n", "Prediction missing for 18934333\n", "Prediction missing for 18934338\n", "Prediction missing for 18934339\n", "Prediction missing for 18934357\n", "Prediction missing for 18934359\n", "Prediction missing for 18934369\n", "Prediction missing for 18934371\n", "Prediction missing for 18934373\n", "Prediction missing for 18934375\n", "Prediction missing for 18978925\n", "Prediction missing for 18978926\n", "Prediction missing for 18978928\n", "Prediction missing for 18978929\n", "Prediction missing for 18978930\n", "Prediction missing for 18978932\n", "Prediction missing for 18984375\n", "Prediction missing for 18984388\n", "Prediction missing for 18984414\n", "Prediction missing for 18984454\n", "Prediction missing for 18984538\n", "Prediction missing for 18984539\n", "Prediction missing for 18984542\n", "Prediction missing for 18984548\n", "Prediction missing for 18984550\n", "Prediction missing for 18984552\n", "Prediction missing for 18984556\n", "Prediction missing for 18984558\n", "Prediction missing for 18984562\n", "Prediction missing for 18984566\n", "Prediction missing for 18984568\n", "Prediction missing for 18984574\n", "Prediction missing for 18984577\n", "Prediction missing for 18984579\n", "Prediction missing for 18984584\n", "Prediction missing for 18984587\n", "Prediction missing for 19052165\n", "Prediction missing for 19069309\n", "Prediction missing for 19069962\n", "Prediction missing for 19069964\n", "Prediction missing for 19069966\n", "Prediction missing for 19069967\n", "Prediction missing for 19069968\n", "Prediction missing for 19069969\n", "Prediction missing for 19081638\n", "Prediction missing for 19081639\n", "Prediction missing for 19081640\n", "Prediction missing for 19081641\n", "Prediction missing for 19081642\n", "Prediction missing for 19081643\n", "Prediction missing for 19118203\n", "Prediction missing for 19118206\n", "Prediction missing for 19118705\n", "Prediction missing for 19118708\n", "Prediction missing for 19155543\n", "Prediction missing for 19155544\n", "Prediction missing for 19155545\n", "Prediction missing for 19155547\n", "Prediction missing for 19155548\n", "Prediction missing for 19155549\n", "Prediction missing for 19215754\n", "Prediction missing for 19270593\n", "Prediction missing for 19270594\n", "Prediction missing for 19270595\n", "Prediction missing for 19270597\n", "Prediction missing for 19270598\n", "Prediction missing for 19381434\n", "Prediction missing for 19381435\n", "Prediction missing for 19381436\n", "Prediction missing for 19381437\n", "Prediction missing for 19381439\n", "Prediction missing for 19381449\n", "Prediction missing for 19381451\n", "Prediction missing for 19381457\n", "Prediction missing for 19381458\n", "Prediction missing for 19383397\n", "Prediction missing for 19572359\n", "Prediction missing for 19685494\n", "Prediction missing for 19685496\n", "Prediction missing for 19685497\n", "Prediction missing for 19685498\n", "Prediction missing for 19685500\n", "Prediction missing for 19685501\n", "Prediction missing for 19685502\n", "Prediction missing for 19685503\n", "Prediction missing for 19685504\n", "Prediction missing for 19685505\n", "Prediction missing for 19685506\n", "Prediction missing for 19873293\n", "Prediction missing for 19935026\n", "Prediction missing for 19935158\n", "Prediction missing for 19935160\n", "Prediction missing for 19935161\n", "Prediction missing for 19935163\n", "Prediction missing for 19935164\n", "Prediction missing for 19935165\n", "Prediction missing for 19935166\n", "Prediction missing for 19935167\n", "Prediction missing for 19935168\n", "Prediction missing for 19935169\n", "Prediction missing for 19935170\n", "Prediction missing for 19935171\n", "Prediction missing for 19935172\n", "Prediction missing for 19935173\n", "Prediction missing for 19935174\n", "Prediction missing for 19935175\n", "Prediction missing for 19935176\n", "Prediction missing for 19935177\n", "Prediction missing for 19935178\n", "Prediction missing for 19935179\n", "Prediction missing for 19935189\n", "Prediction missing for 19935191\n", "Prediction missing for 19950959\n", "Prediction missing for 19950960\n", "Prediction missing for 19950961\n", "Prediction missing for 19950962\n", "Prediction missing for 19950964\n", "Prediction missing for 20056389\n", "Prediction missing for 20056390\n", "Prediction missing for 20056391\n", "Prediction missing for 20056393\n", "Prediction missing for 20056394\n", "Prediction missing for 20058806\n", "Prediction missing for 20058809\n", "Prediction missing for 20109959\n", "Prediction missing for 20109960\n", "Prediction missing for 20109970\n", "Prediction missing for 20109971\n", "Prediction missing for 20109972\n", "Prediction missing for 20109973\n", "Prediction missing for 20109974\n", "Prediction missing for 20109976\n", "Prediction missing for 20109977\n", "Prediction missing for 20109978\n", "Prediction missing for 20109980\n", "Prediction missing for 20109981\n", "Prediction missing for 20109982\n", "Prediction missing for 20109983\n", "Prediction missing for 20109993\n", "Prediction missing for 20109994\n", "Prediction missing for 20147523\n", "Prediction missing for 20147525\n", "Prediction missing for 20147526\n", "Prediction missing for 20147527\n", "Prediction missing for 20147528\n", "Prediction missing for 20147529\n", "Prediction missing for 20147530\n", "Prediction missing for 20147531\n", "Prediction missing for 20147532\n", "Prediction missing for 20147533\n", "Prediction missing for 20147534\n", "Prediction missing for 20147536\n", "Prediction missing for 20203756\n", "Prediction missing for 20203757\n", "Prediction missing for 20203758\n", "Prediction missing for 20203759\n", "Prediction missing for 20203760\n", "Prediction missing for 20203762\n", "Prediction missing for 20203763\n", "Prediction missing for 20203765\n", "Prediction missing for 20203767\n", "Prediction missing for 20203769\n", "Prediction missing for 20203771\n", "Prediction missing for 20203774\n", "Prediction missing for 20203776\n", "Prediction missing for 20203781\n", "Prediction missing for 20203782\n", "Prediction missing for 20203784\n", "Prediction missing for 20203785\n", "Prediction missing for 20203787\n", "Prediction missing for 20239180\n", "Prediction missing for 20260771\n", "Prediction missing for 20260772\n", "Prediction missing for 20260773\n", "Prediction missing for 20260774\n", "Prediction missing for 20260775\n", "Prediction missing for 20260776\n", "Prediction missing for 20260925\n", "Prediction missing for 20260928\n", "Prediction missing for 20264877\n", "Prediction missing for 20264879\n", "Prediction missing for 20266243\n", "Prediction missing for 20266245\n", "Prediction missing for 20266246\n", "Prediction missing for 20266247\n", "Prediction missing for 20266248\n", "Prediction missing for 20266250\n", "Prediction missing for 20354239\n", "Prediction missing for 20398674\n", "Prediction missing for 20400856\n", "Prediction missing for 20400858\n", "Prediction missing for 20400859\n", "Prediction missing for 20400860\n", "Prediction missing for 20400861\n", "Prediction missing for 20400862\n", "Prediction missing for 20400863\n", "Prediction missing for 20400864\n", "Prediction missing for 20400865\n", "Prediction missing for 20400866\n", "Prediction missing for 20400868\n", "Prediction missing for 20400869\n", "Prediction missing for 20400870\n", "Prediction missing for 20400871\n", "Prediction missing for 20400872\n", "Prediction missing for 20400873\n", "Prediction missing for 20400874\n", "Prediction missing for 20400875\n", "Prediction missing for 20400876\n", "Prediction missing for 20516138\n", "Prediction missing for 20516139\n", "Prediction missing for 20516140\n", "Prediction missing for 20516142\n", "Prediction missing for 20516143\n", "Prediction missing for 20528325\n", "Prediction missing for 20528327\n", "Prediction missing for 20528334\n", "Prediction missing for 20528337\n", "Prediction missing for 20528340\n", "Prediction missing for 20528342\n", "Prediction missing for 20528344\n", "Prediction missing for 20528346\n", "Prediction missing for 20528353\n", "Prediction missing for 20528354\n", "Prediction missing for 20528356\n", "Prediction missing for 20528357\n", "Prediction missing for 20528362\n", "Prediction missing for 20528365\n", "Prediction missing for 20528366\n", "Prediction missing for 20528580\n", "Prediction missing for 20528583\n", "Prediction missing for 20544959\n", "Prediction missing for 20544980\n", "Prediction missing for 20544981\n", "Prediction missing for 20544982\n", "Prediction missing for 20544983\n", "Prediction missing for 20544984\n", "Prediction missing for 20544985\n", "Prediction missing for 20544986\n", "Prediction missing for 20544987\n", "Prediction missing for 20544989\n", "Prediction missing for 20544990\n", "Prediction missing for 20544992\n", "Prediction missing for 20544993\n", "Prediction missing for 20595304\n", "Prediction missing for 20618758\n", "Prediction missing for 20618759\n", "Prediction missing for 20618761\n", "Prediction missing for 20618762\n", "Prediction missing for 20618763\n", "Prediction missing for 20618764\n", "Prediction missing for 20618766\n", "Prediction missing for 20618767\n", "Prediction missing for 20618768\n", "Prediction missing for 20618769\n", "Prediction missing for 20618771\n", "Prediction missing for 20618773\n", "Prediction missing for 20618887\n", "Prediction missing for 20618889\n", "Prediction missing for 20618890\n", "Prediction missing for 20618891\n", "Prediction missing for 20618892\n", "Prediction missing for 20618893\n", "Prediction missing for 20618894\n", "Prediction missing for 20618895\n", "Prediction missing for 20618896\n", "Prediction missing for 20618897\n", "Prediction missing for 20618898\n", "Prediction missing for 20618899\n", "Prediction missing for 20740569\n", "Prediction missing for 20750616\n", "Prediction missing for 20750617\n", "Prediction missing for 20750619\n", "Prediction missing for 20750620\n", "Prediction missing for 20750621\n", "Prediction missing for 20750622\n", "Prediction missing for 20750623\n", "Prediction missing for 20750624\n", "Prediction missing for 20750625\n", "Prediction missing for 20750626\n", "Prediction missing for 20750627\n", "Prediction missing for 20750628\n", "Prediction missing for 20778987\n", "Prediction missing for 20778988\n", "Prediction missing for 20800178\n", "Prediction missing for 20800179\n", "Prediction missing for 20800338\n", "Prediction missing for 20800339\n", "Prediction missing for 20813943\n", "Prediction missing for 20813945\n", "Prediction missing for 20813946\n", "Prediction missing for 20813948\n", "Prediction missing for 20813949\n", "Prediction missing for 20813950\n", "Prediction missing for 20825175\n", "Prediction missing for 20825177\n", "Prediction missing for 20825180\n", "Prediction missing for 20825181\n", "Prediction missing for 20825187\n", "Prediction missing for 20825190\n", "Prediction missing for 20825191\n", "Prediction missing for 20825192\n", "Prediction missing for 20825193\n", "Prediction missing for 20825194\n", "Prediction missing for 20876253\n", "Prediction missing for 20876255\n", "Prediction missing for 20885950\n", "Prediction missing for 20906117\n", "Prediction missing for 20906119\n", "Prediction missing for 20985796\n", "Prediction missing for 20985797\n", "Prediction missing for 20985798\n", "Prediction missing for 20985799\n", "Prediction missing for 20985800\n", "Prediction missing for 20985801\n", "Prediction missing for 20985802\n", "Prediction missing for 20985803\n", "Prediction missing for 20985804\n", "Prediction missing for 20985805\n", "Prediction missing for 20985806\n", "Prediction missing for 20985807\n", "Prediction missing for 20985809\n", "Prediction missing for 20985810\n", "Prediction missing for 20985811\n", "Prediction missing for 20985812\n", "Prediction missing for 20985813\n", "Prediction missing for 20985814\n", "Prediction missing for 20985815\n", "Prediction missing for 20985816\n", "Prediction missing for 20985817\n", "Prediction missing for 20985819\n", "Prediction missing for 20985820\n", "Prediction missing for 20985821\n", "Prediction missing for 20985822\n", "Prediction missing for 20985823\n", "Prediction missing for 20985824\n", "Prediction missing for 20985825\n", "Prediction missing for 20985826\n", "Prediction missing for 20985828\n", "Prediction missing for 20985829\n", "Prediction missing for 20985830\n", "Prediction missing for 20985831\n", "Prediction missing for 20985832\n", "Prediction missing for 20985833\n", "Prediction missing for 20985834\n", "Prediction missing for 20985835\n", "Prediction missing for 20985837\n", "Prediction missing for 20985838\n", "Prediction missing for 21064814\n", "Prediction missing for 21133496\n", "Prediction missing for 21133538\n", "Prediction missing for 21133539\n", "Prediction missing for 21178984\n", "Prediction missing for 21431795\n", "Prediction missing for 21431806\n", "Prediction missing for 21431808\n", "Prediction missing for 21467631\n", "Prediction missing for 21467634\n", "Prediction missing for 21655667\n", "Prediction missing for 21655669\n", "Prediction missing for 21655671\n", "Prediction missing for 21655672\n", "Prediction missing for 21655673\n", "Prediction missing for 21783612\n", "Prediction missing for 21879975\n", "Prediction missing for 21962357\n", "Prediction missing for 22115524\n", "Prediction missing for 22218477\n", "Prediction missing for 22218479\n", "Prediction missing for 22218495\n", "Prediction missing for 22218497\n", "Prediction missing for 22218498\n", "Prediction missing for 22218499\n", "Prediction missing for 22218500\n", "Prediction missing for 22218501\n", "Prediction missing for 22218512\n", "Prediction missing for 22218513\n", "Prediction missing for 22218514\n", "Prediction missing for 22218515\n", "Prediction missing for 22218516\n", "Prediction missing for 22218517\n", "Prediction missing for 22218538\n", "Prediction missing for 22218539\n", "Prediction missing for 22218541\n", "Prediction missing for 22218544\n", "Prediction missing for 22218545\n", "Prediction missing for 22218546\n", "Prediction missing for 22218549\n", "Prediction missing for 22218550\n", "Prediction missing for 22218551\n", "Prediction missing for 22218552\n", "Prediction missing for 22218553\n", "Prediction missing for 22218555\n", "Prediction missing for 22218556\n", "Prediction missing for 22218558\n", "Prediction missing for 22218559\n", "Prediction missing for 22218560\n", "Prediction missing for 22218561\n", "Prediction missing for 22218562\n", "Prediction missing for 22218565\n", "Prediction missing for 22218566\n", "Prediction missing for 22218567\n", "Prediction missing for 22218571\n", "Prediction missing for 22218572\n", "Prediction missing for 22218573\n", "Prediction missing for 22218574\n", "Prediction missing for 22218575\n", "Prediction missing for 22218576\n", "Prediction missing for 22218577\n", "Prediction missing for 22218578\n", "Prediction missing for 22218579\n", "Prediction missing for 22218580\n", "Prediction missing for 22218582\n", "Prediction missing for 22218583\n", "Prediction missing for 22218585\n", "Prediction missing for 22218586\n", "Prediction missing for 22218587\n", "Prediction missing for 22218588\n", "Prediction missing for 22218593\n", "Prediction missing for 22218596\n", "Prediction missing for 22591477\n", "Prediction missing for 22803862\n", "Prediction missing for 22803863\n", "Prediction missing for 22803864\n", "Prediction missing for 22803865\n", "Prediction missing for 22803867\n", "Prediction missing for 23025814\n", "Prediction missing for 23116201\n", "Prediction missing for 23116217\n", "Prediction missing for 23116220\n", "Prediction missing for 23671337\n", "Prediction missing for 24137783\n", "Prediction missing for 24137784\n", "Prediction missing for 24137785\n", "Prediction missing for 24137786\n", "Prediction missing for 24137787\n", "Prediction missing for 24143578\n", "Prediction missing for 24143579\n", "Prediction missing for 24644995\n", "Prediction missing for 24694217\n", "Prediction missing for 24694237\n", "Prediction missing for 24694238\n", "Prediction missing for 24863628\n", "Prediction missing for 24863629\n", "Prediction missing for 24962129\n", "Prediction missing for 24962140\n", "Prediction missing for 24962141\n", "Prediction missing for 24962142\n", "Prediction missing for 24962144\n", "Prediction missing for 24962145\n", "Prediction missing for 24962146\n", "Prediction missing for 25091182\n", "Prediction missing for 25091187\n", "Prediction missing for 25091190\n", "Prediction missing for 25160674\n", "Prediction missing for 25160677\n", "Prediction missing for 25160678\n", "Prediction missing for 25160681\n", "Prediction missing for 25160682\n", "Prediction missing for 25160683\n", "Prediction missing for 25160684\n", "Prediction missing for 25160687\n", "Prediction missing for 25160688\n", "Prediction missing for 25198625\n", "Prediction missing for 25198628\n", "Prediction missing for 25597313\n", "Prediction missing for 25597314\n", "Prediction missing for 25597315\n", "Prediction missing for 25597316\n", "Prediction missing for 25597317\n", "Prediction missing for 25600353\n", "Prediction missing for 25600384\n", "Prediction missing for 25600386\n", "Prediction missing for 25627420\n", "Prediction missing for 25627421\n", "Prediction missing for 25628092\n", "Prediction missing for 25628093\n", "Prediction missing for 25628095\n", "Prediction missing for 25628096\n", "Prediction missing for 25628098\n", "Prediction missing for 25633805\n", "Prediction missing for 25633806\n", "Prediction missing for 25633817\n", "Prediction missing for 25633818\n", "Prediction missing for 25660507\n", "Prediction missing for 25664697\n", "Prediction missing for 25664698\n", "Prediction missing for 25887241\n", "Prediction missing for 25950043\n", "Prediction missing for 25950053\n", "Prediction missing for 25950055\n", "Prediction missing for 25950056\n", "Prediction missing for 25950057\n", "Prediction missing for 25950058\n", "Prediction missing for 25950059\n", "Prediction missing for 25974596\n", "Prediction missing for 25974598\n", "Prediction missing for 25974839\n", "Prediction missing for 25974840\n", "Prediction missing for 26015462\n", "Prediction missing for 26015464\n", "Prediction missing for 26216613\n", "Prediction missing for 26216614\n", "Prediction missing for 26216615\n", "Prediction missing for 26216616\n", "Prediction missing for 26216617\n", "Prediction missing for 26216618\n", "Prediction missing for 26216619\n", "Prediction missing for 26216620\n", "Prediction missing for 26216621\n", "Prediction missing for 26216622\n", "Prediction missing for 26216623\n", "Prediction missing for 26216624\n", "Prediction missing for 26216625\n", "Prediction missing for 26216626\n", "Prediction missing for 26216627\n", "Prediction missing for 26216628\n", "Prediction missing for 26216629\n", "Prediction missing for 26216630\n", "Prediction missing for 26216631\n", "Prediction missing for 26216632\n", "Prediction missing for 26216633\n", "Prediction missing for 26216634\n", "Prediction missing for 26216635\n", "Prediction missing for 26216636\n", "Prediction missing for 26216637\n", "Prediction missing for 26216639\n", "Prediction missing for 26216640\n", "Prediction missing for 26216641\n", "Prediction missing for 26216643\n", "Prediction missing for 26228965\n", "Prediction missing for 26228966\n", "Prediction missing for 26228967\n", "Prediction missing for 26228968\n", "Prediction missing for 26228972\n", "Prediction missing for 26228973\n", "Prediction missing for 26228976\n", "Prediction missing for 26228977\n", "Prediction missing for 26228981\n", "Prediction missing for 26228984\n", "Prediction missing for 26228986\n", "Prediction missing for 26228987\n", "Prediction missing for 26228989\n", "Prediction missing for 26228991\n", "Prediction missing for 26228993\n", "Prediction missing for 26228995\n", "Prediction missing for 26228996\n", "Prediction missing for 26228997\n", "Prediction missing for 26228999\n", "Prediction missing for 26229001\n", "Prediction missing for 26229004\n", "Prediction missing for 26229009\n", "Prediction missing for 26229010\n", "Prediction missing for 26229011\n", "Prediction missing for 26229012\n", "Prediction missing for 26229013\n", "Prediction missing for 26229015\n", "Prediction missing for 26229017\n", "Prediction missing for 26229018\n", "Prediction missing for 26229019\n", "Prediction missing for 26229020\n", "Prediction missing for 26229111\n", "Prediction missing for 26229112\n", "Prediction missing for 26229113\n", "Prediction missing for 26229115\n", "Prediction missing for 26229117\n", "Prediction missing for 26229118\n", "Prediction missing for 26229121\n", "Prediction missing for 26229122\n", "Prediction missing for 26229123\n", "Prediction missing for 26229124\n", "Prediction missing for 26489050\n", "Prediction missing for 26489425\n", "Prediction missing for 26491220\n" ] } ], "source": [ "dic_docs_stats = dict()\n", "dic_splits = dict()\n", "for c in labeled_cands:\n", " if c.chem1.sentence.document.id in dic_docs_stats:\n", " try:\n", " d = predictions_df.loc[c.id]\n", " true,pred = d[\"label\"],d[\"pred\"]\n", " if true==1:\n", " if pred==1:\n", " dic_docs_stats[c.chem1.sentence.document.id][\"tp\"].append(c)\n", " else:\n", " dic_docs_stats[c.chem1.sentence.document.id][\"fn\"].append(c)\n", " else:\n", " if pred==-1:\n", " dic_docs_stats[c.chem1.sentence.document.id][\"tn\"].append(c)\n", " else:\n", " dic_docs_stats[c.chem1.sentence.document.id][\"fp\"].append(c)\n", " \n", " except:\n", " print(\"Prediction missing for {}\".format(c.id))\n", " else:\n", " dic_docs_stats[c.chem1.sentence.document.id]=dict(tp=[],fp=[],fn=[],tn=[])\n", " dic_splits[c.chem1.sentence.document.id] = c.split\n", " " ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "dic_doc_stats_agreg = list()\n", "for k in dic_docs_stats.keys():\n", " d_res = dict(cid = k)\n", " for k1 in dic_docs_stats[k].keys():\n", " d_res[k1] = len(dic_docs_stats[k][k1])\n", " d_res[\"split\"] = dic_splits[k]\n", " dic_doc_stats_agreg.append(d_res)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "df = pd.DataFrame(dic_doc_stats_agreg)" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
cidfnfpsplittntp
061849600300
1547686600210
2331878700170
3155637100300
471802930011410
5812083800490
6543488700300
78716237204911
851513300300
9268757300300
107723949004150
117388176801250
12527924900300
1381921900300
1431216843011991
15357378100240
16106805400110
176812696103620
181266713201210
19301980200110
20126517900300
21493468400110
22131074900300
23572623900130
24425041400300
252686128002250
26336797000210
277776292001290
28552093400210
29210210800130
.....................
305205307500170
306160610100300
307846536900300
3084300432603250
30926364300210
310435569500300
3117967199211712
312453864700300
3138268412123251
314185241100300
315414307600300
31655771896131140
3175033833404790
3187231463201161
3195520362002230
320562327500250
321801851400210
322146379000250
323665547100300
324419684800210
32581969700110
3265775858004330
327131070700210
3282585590003360
329610866300150
33087141600300
3316607353002190
332514554002130
3332049535101180
3344982783001690
\n", "

335 rows × 6 columns

\n", "
" ], "text/plain": [ " cid fn fp split tn tp\n", "0 618496 0 0 3 0 0\n", "1 5476866 0 0 2 1 0\n", "2 3318787 0 0 1 7 0\n", "3 1556371 0 0 3 0 0\n", "4 7180293 0 0 1 141 0\n", "5 8120838 0 0 4 9 0\n", "6 5434887 0 0 3 0 0\n", "7 8716237 2 0 4 91 1\n", "8 515133 0 0 3 0 0\n", "9 2687573 0 0 3 0 0\n", "10 7723949 0 0 4 15 0\n", "11 7388176 8 0 1 25 0\n", "12 5279249 0 0 3 0 0\n", "13 819219 0 0 3 0 0\n", "14 3121684 3 0 1 199 1\n", "15 3573781 0 0 2 4 0\n", "16 1068054 0 0 1 1 0\n", "17 6812696 1 0 3 62 0\n", "18 1266713 2 0 1 21 0\n", "19 3019802 0 0 1 1 0\n", "20 1265179 0 0 3 0 0\n", "21 4934684 0 0 1 1 0\n", "22 1310749 0 0 3 0 0\n", "23 5726239 0 0 1 3 0\n", "24 4250414 0 0 3 0 0\n", "25 2686128 0 0 2 25 0\n", "26 3367970 0 0 2 1 0\n", "27 7776292 0 0 1 29 0\n", "28 5520934 0 0 2 1 0\n", "29 2102108 0 0 1 3 0\n", ".. ... .. .. ... ... ..\n", "305 2053075 0 0 1 7 0\n", "306 1606101 0 0 3 0 0\n", "307 8465369 0 0 3 0 0\n", "308 4300432 6 0 3 25 0\n", "309 263643 0 0 2 1 0\n", "310 4355695 0 0 3 0 0\n", "311 7967199 2 1 1 71 2\n", "312 4538647 0 0 3 0 0\n", "313 8268412 1 2 3 25 1\n", "314 1852411 0 0 3 0 0\n", "315 4143076 0 0 3 0 0\n", "316 5577189 6 1 3 114 0\n", "317 5033833 4 0 4 79 0\n", "318 7231463 2 0 1 16 1\n", "319 5520362 0 0 2 23 0\n", "320 5623275 0 0 2 5 0\n", "321 8018514 0 0 2 1 0\n", "322 1463790 0 0 2 5 0\n", "323 6655471 0 0 3 0 0\n", "324 4196848 0 0 2 1 0\n", "325 819697 0 0 1 1 0\n", "326 5775858 0 0 4 33 0\n", "327 1310707 0 0 2 1 0\n", "328 2585590 0 0 3 36 0\n", "329 6108663 0 0 1 5 0\n", "330 871416 0 0 3 0 0\n", "331 6607353 0 0 2 19 0\n", "332 514554 0 0 2 13 0\n", "333 2049535 1 0 1 18 0\n", "334 4982783 0 0 1 69 0\n", "\n", "[335 rows x 6 columns]" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [], "source": [ "df[\"n_cands\"] = df[[\"tp\",\"fp\",\"tn\",\"fn\"]].sum(axis = 1)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "\n", "%matplotlib notebook" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "recalls = df[\"tp\"]/(df[\"tp\"]+df[\"fn\"])\n", "recalls.loc[recalls.isna()] = -1" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "precisions = df[\"tp\"]/(df[\"tp\"]+df[\"fp\"])\n", "precisions = precisions.loc[(precisions.notna()|(df[\"tp\"]+df[\"fn\"]>0))]\n", "precisions.loc[precisions.isna()] = -1" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAEmBJREFUeJzt3X+s3XV9x/Hn2yK4cV0p4u5qJRYypiM0gj1hbCbzXvxVIRHMmIOgKxNTdWJc5pJVzTI3Z4bLkGRsiXbCYBvj6lDSDnCkQu+IieBaV2mBIAW7ja72Dgud1zEm+t4f53vNod7T8/2eX7f97PlITu4535+v+zmnr/vt937PuZGZSJKOfS9Y6gCSpOGw0CWpEBa6JBXCQpekQljoklQIC12SCmGhS1IhLHRJKoSFLkmFOG6cOzvllFNy9erVfa37ve99jxNPPHG4gYbAXM2YqxlzNXO05oLBsu3YsePJzHxpzwUzc2y3tWvXZr+2bdvW97qjZK5mzNWMuZo5WnNlDpYN2J41OtZTLpJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFcJCl6RCWOiSVIixvvV/ELv2HeKKjXcsyb73Xn3hkuxXkprwCF2SCmGhS1IhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFaJnoUfEiyLiaxHxjYh4MCL+oJp+WkTcHxF7IuJzEXH86ONKkrqpc4T+LHB+Zr4aOBtYFxHnAZ8Ers3MnwWeAq4cXUxJUi89Cz3b5quHL6xuCZwP3FpNvwm4eCQJJUm11DqHHhHLImInMAdsBR4Dns7M56pFngBWjSaiJKmOyMz6C0ecBNwG/B5wY3W6hYg4FfhSZp61yDobgA0Ak5OTa2dmZvoKOnfwEAee6WvVga1ZtbzrvPn5eSYmJsaYph5zNWOuZszV3CDZpqend2Rmq9dyjf6maGY+HRHbgF8EToqI46qj9JcD+7qsswnYBNBqtXJqaqrJLn/kups3c82upfkTqHsvn+o6b3Z2ln6/p1EyVzPmasZczY0jW52rXF5aHZkTET8BvBF4GNgGXFItth7YPKqQkqTe6hzyrgRuiohltH8AfD4zb4+Ih4CZiPgj4F+A60eYU5LUQ89Cz8wHgHMWmf44cO4oQkmSmvOdopJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRA9Cz0iTo2IbRHxUEQ8GBEfrKZ/LCL2RcTO6nbB6ONKkro5rsYyzwEfysyvR8SLgR0RsbWad21m/uno4kmS6upZ6Jm5H9hf3f9uRDwMrBp1MElSM43OoUfEauAc4P5q0lUR8UBE3BARK4acTZLUQGRmvQUjJoB/Aj6RmV+MiEngSSCBjwMrM/Ndi6y3AdgAMDk5uXZmZqavoHMHD3Hgmb5WHdiaVcu7zpufn2diYmKMaeoxVzPmasZczQ2SbXp6ekdmtnotV6vQI+KFwO3AXZn5qUXmrwZuz8yzjrSdVquV27dv77m/xVx382au2VXnlP/w7b36wq7zZmdnmZqaGl+YmszVjLmaMVdzg2SLiFqFXucqlwCuBx7uLPOIWNmx2NuA3f0ElSQNR51D3tcC7wR2RcTOatpHgMsi4mzap1z2Au8ZSUJJUi11rnL5ChCLzLpz+HEkSf3ynaKSVAgLXZIKYaFLUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFaJnoUfEqRGxLSIeiogHI+KD1fSTI2JrRDxafV0x+riSpG7qHKE/B3woM88EzgPeHxFnAhuBuzPzDODu6rEkaYn0LPTM3J+ZX6/ufxd4GFgFXATcVC12E3DxqEJKknqLzKy/cMRq4F7gLODfMvOkanoATy08PmydDcAGgMnJybUzMzN9BZ07eIgDz/S16sDWrFredd78/DwTExNjTFOPuZoxVzPmam6QbNPT0zsys9VruePqbjAiJoAvAL+Vmf/V7vC2zMyIWPQnQ2ZuAjYBtFqtnJqaqrvL57nu5s1cs6t23KHae/lU13mzs7P0+z2NkrmaMVcz5mpuHNlqXeUSES+kXeY3Z+YXq8kHImJlNX8lMDeaiJKkOupc5RLA9cDDmfmpjllbgPXV/fXA5uHHkyTVVeccxmuBdwK7ImJnNe0jwNXA5yPiSuBfgbePJqIkqY6ehZ6ZXwGiy+zXDzeOJKlfvlNUkgphoUtSISx0SSqEhS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRAWuiQVomehR8QNETEXEbs7pn0sIvZFxM7qdsFoY0qSeqlzhH4jsG6R6ddm5tnV7c7hxpIkNdWz0DPzXuDgGLJIkgYwyDn0qyLigeqUzIqhJZIk9SUys/dCEauB2zPzrOrxJPAkkMDHgZWZ+a4u624ANgBMTk6unZmZ6Svo3MFDHHimr1UHtmbV8q7z5ufnmZiYGGOaeszVjLmaMVdzg2Sbnp7ekZmtXssd18/GM/PAwv2I+Evg9iMsuwnYBNBqtXJqaqqfXXLdzZu5ZldfcQe29/KprvNmZ2fp93saJXM1Y65mzNXcOLL1dcolIlZ2PHwbsLvbspKk8eh5yBsRtwBTwCkR8QTw+8BURJxN+5TLXuA9I8woSaqhZ6Fn5mWLTL5+BFkkSQPwnaKSVAgLXZIKYaFLUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSpEzz8SLUmlWL3xjiXb943rThz5PjxCl6RC9Cz0iLghIuYiYnfHtJMjYmtEPFp9XTHamJKkXuocod8IrDts2kbg7sw8A7i7eixJWkI9Cz0z7wUOHjb5IuCm6v5NwMVDziVJaqjfc+iTmbm/uv9tYHJIeSRJfYrM7L1QxGrg9sw8q3r8dGae1DH/qcxc9Dx6RGwANgBMTk6unZmZ6Svo3MFDHHimr1UHtmbV8q7z5ufnmZiYGGOaeszVjLmaOVZz7dp3aIxpnu+05cv6HrPp6ekdmdnqtVy/ly0eiIiVmbk/IlYCc90WzMxNwCaAVquVU1NTfe3wups3c82upbnKcu/lU13nzc7O0u/3NErmasZczRyrua5Y4ssWRz1m/Z5y2QKsr+6vBzYPJ44kqV91Llu8Bfgq8MqIeCIirgSuBt4YEY8Cb6geS5KWUM9zGJl5WZdZrx9yFknSAHynqCQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQljoklQIC12SCtHzj0RL/x/s2neIKzbesST73nv1hUuyX5XHI3RJKoSFLkmFGOiUS0TsBb4L/AB4LjNbwwglSWpuGOfQpzPzySFsR5I0AE+5SFIhIjP7XzniW8BTQAKfycxNiyyzAdgAMDk5uXZmZqavfc0dPMSBZ/qOOpA1q5Z3nTc/P8/ExMQY09RjrmZ8fTVzrObate/QGNM832nLl/U9ZtPT0zvqnNIetNBXZea+iPhpYCvwgcy8t9vyrVYrt2/f3te+rrt5M9fsWpqrLI90Wdns7CxTU1PjC1OTuZrx9dXMsZpr9RJdmgpw47oT+x6ziKhV6AOdcsnMfdXXOeA24NxBtidJ6l/fhR4RJ0bEixfuA28Cdg8rmCSpmUH+jzkJ3BYRC9v5u8z8x6GkkiQ11nehZ+bjwKuHmEWSNAAvW5SkQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiH6/iPRkoZj9cY7us770JrnuOII8wex9+oLR7LdOo70PQ9ilON1LPAIXZIKMVChR8S6iHgkIvZExMZhhZIkNdd3oUfEMuAvgLcAZwKXRcSZwwomSWpmkCP0c4E9mfl4Zv4vMANcNJxYkqSmBin0VcC/dzx+opomSVoCkZn9rRhxCbAuM99dPX4n8AuZedVhy20ANlQPXwk80mfWU4An+1x3lMzVjLmaMVczR2suGCzbKzLzpb0WGuSyxX3AqR2PX15Ne57M3ARsGmA/AETE9sxsDbqdYTNXM+ZqxlzNHK25YDzZBjnl8s/AGRFxWkQcD1wKbBlOLElSU30foWfmcxFxFXAXsAy4ITMfHFoySVIjA71TNDPvBO4cUpZeBj5tMyLmasZczZirmaM1F4whW9+/FJUkHV18678kFeKoKvSI+NWIeDAifhgRXX8b3O0jB6pf0N5fTf9c9cvaYeQ6OSK2RsSj1dcViywzHRE7O27/ExEXV/NujIhvdcw7e1y5quV+0LHvLR3Tl3K8zo6Ir1bP9wMR8Wsd84Y6Xr0+oiIiTqi+/z3VeKzumPfhavojEfHmQXL0keu3I+KhanzujohXdMxb9DkdU64rIuI/O/b/7o5566vn/dGIWD/mXNd2ZPpmRDzdMW+U43VDRMxFxO4u8yMi/qzK/UBEvKZj3nDHKzOPmhvw87SvVZ8FWl2WWQY8BpwOHA98Azizmvd54NLq/qeB9w0p158AG6v7G4FP9lj+ZOAg8JPV4xuBS0YwXrVyAfNdpi/ZeAE/B5xR3X8ZsB84adjjdaTXS8cyvwl8urp/KfC56v6Z1fInAKdV21k2xlzTHa+h9y3kOtJzOqZcVwB/vsi6JwOPV19XVPdXjCvXYct/gPaFGiMdr2rbvwy8BtjdZf4FwJeAAM4D7h/VeB1VR+iZ+XBm9nrj0aIfORARAZwP3FotdxNw8ZCiXVRtr+52LwG+lJn/PaT9d9M0148s9Xhl5jcz89Hq/n8Ac0DPN070oc5HVHTmvRV4fTU+FwEzmflsZn4L2FNtbyy5MnNbx2voPtrv9Ri1QT7S483A1sw8mJlPAVuBdUuU6zLgliHt+4gy817aB3DdXAT8dbbdB5wUESsZwXgdVYVeU7ePHHgJ8HRmPnfY9GGYzMz91f1vA5M9lr+UH38xfaL679a1EXHCmHO9KCK2R8R9C6eBOIrGKyLOpX3U9VjH5GGNV52PqPjRMtV4HKI9PqP8eIum276S9lHegsWe03Hm+pXq+bk1IhbeYHhUjFd1auo04J6OyaMarzq6ZR/6eI39D1xExJeBn1lk1kczc/O48yw4Uq7OB5mZEdH10qDqJ+8a2tfnL/gw7WI7nvalS78L/OEYc70iM/dFxOnAPRGxi3Zp9W3I4/U3wPrM/GE1ue/xKlFEvANoAa/rmPxjz2lmPrb4FobuH4BbMvPZiHgP7f/dnD+mfddxKXBrZv6gY9pSjtfYjL3QM/MNA26i20cOfIf2f2WOq46yFv0ogn5yRcSBiFiZmfurApo7wqbeDtyWmd/v2PbC0eqzEfFXwO+MM1dm7qu+Ph4Rs8A5wBdY4vGKiJ8C7qD9w/y+jm33PV6LqPMRFQvLPBERxwHLab+ean28xQhzERFvoP1D8nWZ+ezC9C7P6TAKqmeuzPxOx8PP0v6dycK6U4etOzuETLVydbgUeH/nhBGOVx3dsg99vI7FUy6LfuRAtn/LsI32+WuA9cCwjvi3VNurs90fO3dXldrCeeuLgUV/Gz6KXBGxYuGURUScArwWeGipx6t67m6jfW7x1sPmDXO86nxERWfeS4B7qvHZAlwa7atgTgPOAL42QJZGuSLiHOAzwFszc65j+qLP6Rhzrex4+Fbg4er+XcCbqnwrgDfx/P+pjjRXle1VtH/B+NWOaaMcrzq2AL9eXe1yHnCoOmgZ/ngN+ze+g9yAt9E+j/QscAC4q5r+MuDOjuUuAL5J+yfsRzumn077H9we4O+BE4aU6yXA3cCjwJeBk6vpLeCzHcutpv1T9wWHrX8PsIt2Mf0tMDGuXMAvVfv+RvX1yqNhvIB3AN8Hdnbczh7FeC32eqF9Cuet1f0XVd//nmo8Tu9Y96PVeo8Abxny671Xri9X/w4WxmdLr+d0TLn+GHiw2v824FUd676rGsc9wG+MM1f1+GPA1YetN+rxuoX2VVrfp91fVwLvBd5bzQ/afwzosWr/rY51hzpevlNUkgpxLJ5ykSQtwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQ/wdjzvEPbDEDdQAAAABJRU5ErkJggg==\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "precisions.hist()" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [], "source": [ "precision_df = pd.DataFrame([df.iloc[precisions.index][\"split\"],precisions]).T" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [], "source": [ "precision_df.columns = [\"split\",\"precision\"]" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/matthieu/Documents/Snorkel/snorkel/.snorkelenv.gpu/local/lib/python2.7/site-packages/ggplot/utils.py:81: FutureWarning: pandas.tslib is deprecated and will be removed in a future version.\n", "You can access Timestamp as pandas.Timestamp\n", " pd.tslib.Timestamp,\n", "/home/matthieu/Documents/Snorkel/snorkel/.snorkelenv.gpu/local/lib/python2.7/site-packages/ggplot/stats/smoothers.py:4: FutureWarning: The pandas.lib module is deprecated and will be removed in a future version. These are private functions and can be accessed from pandas._libs.lib instead\n", " from pandas.lib import Timestamp\n", "/home/matthieu/Documents/Snorkel/snorkel/.snorkelenv.gpu/local/lib/python2.7/site-packages/statsmodels/tsa/statespace/tools.py:12: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n", " from . import _statespace\n", "/home/matthieu/Documents/Snorkel/snorkel/.snorkelenv.gpu/local/lib/python2.7/site-packages/statsmodels/tsa/statespace/tools.py:12: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192, got 176\n", " from . import _statespace\n", "/home/matthieu/Documents/Snorkel/snorkel/.snorkelenv.gpu/local/lib/python2.7/site-packages/statsmodels/compat/pandas.py:56: FutureWarning: The pandas.core.datetools module is deprecated and will be removed in a future version. Please use the pandas.tseries module instead.\n", " from pandas.core import datetools\n", "/home/matthieu/Documents/Snorkel/snorkel/.snorkelenv.gpu/local/lib/python2.7/site-packages/statsmodels/tsa/kalmanf/kalmanfilter.py:33: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n", " from . import kalman_loglike\n", "/home/matthieu/Documents/Snorkel/snorkel/.snorkelenv.gpu/local/lib/python2.7/site-packages/statsmodels/tsa/kalmanf/kalmanfilter.py:33: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192, got 176\n", " from . import kalman_loglike\n", "/home/matthieu/Documents/Snorkel/snorkel/.snorkelenv.gpu/local/lib/python2.7/site-packages/scipy/signal/_max_len_seq.py:8: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n", " from ._max_len_seq_inner import _max_len_seq_inner\n", "/home/matthieu/Documents/Snorkel/snorkel/.snorkelenv.gpu/local/lib/python2.7/site-packages/scipy/signal/_upfirdn.py:36: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n", " from ._upfirdn_apply import _output_len, _apply\n", "/home/matthieu/Documents/Snorkel/snorkel/.snorkelenv.gpu/local/lib/python2.7/site-packages/scipy/signal/spectral.py:10: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n", " from ._spectral import lombscargle\n", "/home/matthieu/Documents/Snorkel/snorkel/.snorkelenv.gpu/local/lib/python2.7/site-packages/statsmodels/tsa/regime_switching/markov_switching.py:30: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n", " from statsmodels.tsa.regime_switching._hamilton_filter import (\n", "/home/matthieu/Documents/Snorkel/snorkel/.snorkelenv.gpu/local/lib/python2.7/site-packages/statsmodels/tsa/regime_switching/markov_switching.py:30: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192, got 176\n", " from statsmodels.tsa.regime_switching._hamilton_filter import (\n", "/home/matthieu/Documents/Snorkel/snorkel/.snorkelenv.gpu/local/lib/python2.7/site-packages/statsmodels/nonparametric/kde.py:26: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n", " from .linbin import fast_linbin\n", "/home/matthieu/Documents/Snorkel/snorkel/.snorkelenv.gpu/local/lib/python2.7/site-packages/statsmodels/nonparametric/kde.py:26: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192, got 176\n", " from .linbin import fast_linbin\n", "/home/matthieu/Documents/Snorkel/snorkel/.snorkelenv.gpu/local/lib/python2.7/site-packages/statsmodels/nonparametric/smoothers_lowess.py:11: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n", " from ._smoothers_lowess import lowess as _lowess\n", "/home/matthieu/Documents/Snorkel/snorkel/.snorkelenv.gpu/local/lib/python2.7/site-packages/statsmodels/nonparametric/smoothers_lowess.py:11: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192, got 176\n", " from ._smoothers_lowess import lowess as _lowess\n" ] } ], "source": [ "from ggplot import *" ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqQAAAH9CAYAAADML24LAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XtwW/Wd//+XLMmK7ESRSBTnnrCEGudCEpwbbYlrSDIMBMptSlm6QGC7hHKZdtuddpedXTrtd2mXdminZYZtd4duod2U2zQsoXRDa5wSwBCnNrSBEpLmhnNRgx3jC7rY+v3BL06URLakc+TPkc7zMdMplo7O6/PWOXrzRrJ1POl0Oi0AAADAkArTCwAAAIC7MZACAADAKAZSAAAAGMVACgAAAKMYSAEAAGAUAykAAACMYiCFcbfeeqsmTZqk+fPnn/H+dDqte+65R3PmzNH555+v7du3j/IKAcBZ9u/fr8bGRs2dO1fz5s3T97///dO2oXeilDCQwrhbbrlFzz//fNb7f/WrX2nnzp3auXOnfvSjH+mOO+4YxdUBgPP4fD5997vf1Y4dO/Tqq6/qoYce0o4dOzK2oXeilDCQwriVK1fqrLPOynr/xo0bddNNN8nj8WjFihXq6urSwYMHR3GFAOAsU6ZM0QUXXCBJGjdunOrq6vTee+9lbEPvRClhIIXjvffee5oxY8bQz9OnTz+t8QKAW+3Zs0e///3vtXz58ozb6Z0oJQykAACUqJ6eHl177bX63ve+p1AoZHo5QMEYSOF406ZN0/79+4d+PnDggKZNm2ZwRQBgXjKZ1LXXXqsbb7xR11xzzWn30ztRShhI4XhXXnmlfvrTnyqdTuvVV1/V+PHjNWXKFNPLAgBj0um0brvtNtXV1env//7vz7gNvROlxGd6AcANN9ygF198UX/5y180ffp0ff3rX1cymZQkrV+/Xpdddpmee+45zZkzR1VVVXrkkUcMrxgAzNq6daseffRRLViwQIsWLZIk/du//Zv27dsnid6J0uNJp9Np04sAAACAe/GRPQAAAIxiIAUAAIBRDKQAAAAwioEUAAAARjGQAgAAwKhR/9qnjo6OvLb3+/2KRqOKxWJDXwWUr0AgoHg8XtBjTea7ufZyyHdz7Sbyp06dWlBWqXBb73Tb+eukfDfXXsr5hWY7pXe64h3SigqzZZrMd3PtpvPdXLsT8mEdrx935ru5dvLNcWfVAAAAcAwGUgAAABiV0++QtrS0qK2tTUeOHNH8+fN19dVXD923e/dubdq0SceOHdP06dN11VVXKRwOF23BAFAK6JsAkLuc3iEdN26cVq5cqcWLF2fc3tvbq1/84he6+OKL9dWvflVTp07VE088UZSFAkApoW8CQO5yGkjnzp2ruro6BYPBjNvfeustRaNRzZs3T36/X5/61Kd0+PBhxWKxoiwWAEoFfRMAcmfpa59isZgmT5489HNlZaUikYhisZii0ai6u7vV09OT8ZhEIqHq6urcF+jzZfx/Ibxer/x+f0GPNZnv5trLId/NtTsh36lG6puS6J2Gs92e7+baSzm/1PumpVUnEglVVVVl3DZmzJih789qbW1Vc3Nzxv0NDQ1qbGzMOysSiRS+UBuYzHdz7W7Pd3PtTsgvhpH6pkTvLIdst+e7uXbT+aZrL5SlgbSysvK0L2+Nx+MKBAKSpPr6etXW1mbcn0gk8vpoyufzKRKJqLOzU6lUqqB1WvmSW5P5bq69HPLdXLuJ/OPvLjrdSH1ToneaznZ7vptrL+X8QrOd0jstDaTRaFTt7e1DPycSCb3//vtDxYVCIYVCoYzHdHR0FHT1glQqVfBVD3w+X8GPdUK+m2sv5Xw31+6EfKcaqW9K9E7T2W7Pd3Pt5ZBfqn0zpz9qGhgYUDKZVDqdVjqdVjKZ1MDAgOrq6nTkyBHt2LFDyWRSzc3Nqqmpccy0DQCm0DcBIHc5vUO6ZcuWjN9neuONN4Z+n+kzn/mMnnvuOT399NOaNm2arrvuuqItFgBKBX0TAHKX00Da2NiY9ZfpzznnHN199922LgoASh19EwByx6VDAQAAYBQDKQAAAIxiIAUAAIBRDKQAAAAwioEUAAAARjGQAgAAwCgGUgAAABjFQAoAAACjGEgBAABgFAMpAAAAjGIgBQAAgFGedDqdHs3Ao0ePqqIi9znY4/GosrJSiURChS61oqJCg4ODBT3WZL6bay+HfDfXbiI/EokUlFUq3NY73Xb+OinfzbWXcn6h2U7pnb7RDozH43lt7/f7FQ6H1dvbq2QyWVBmMBhUf39/QY81me/m2ssh3821m8h3SlMtFrf1Tredv07Kd3PtpZxfaLZTeicf2QMAAMAoBlIAAAAYxUAKAAAAoxhIAQAAYBQDKQAAAIxiIAUAAIBRDKQAAAAwioEUAAAARjGQAgAAwCgGUgAAABjFQAoAAACjLF/LvrOzU5s2bdKBAwfk9Xo1d+5cXXrppfJ6vXasDwDKEr0TAE6w/A7ppk2bVF1drS9/+ctav3699u7dq9dff92OtQFA2aJ3AsAJlt8h7erq0rJly+T3++X3+zVnzhzFYjFJUnd3t3p6ejK2TyQSqq6uzn2BPl/G/xfC6/XK7/cX9FiT+W6uvRzy3Vy7E/Kdjt7p3Gy357u59lLOL/W+6Umn02krO9i2bZv27duntWvX6sMPP9Sjjz6qiy++WHV1dWpqalJzc3PG9g0NDWpsbMwrY8mSJVaWmJNt27YVPQMAjhuN3gkApcLyQBqLxfT000/r0KFDSqfTWrhwoa666ip5PB7b/it/9erVVpaYk1/96ldZ8yORiDo7O5VKpQradyAQUDwez/txJrPJt57v5tpN5Eej0YKyTBmN3llqx9Ap2W7Pd3PtpZxfaLZTeqel93UHBwf12GOPqb6+XrfddpsSiYQ2btyozZs3a82aNQqFQgqFQhmP6ejoUDKZtLToYhhpTalUquB1+3w+SzWbzCa/8Hw31+6EfCcbzd5ZqsfQ9Pnj5nw3114O+aXaNy39UVN/f7+OHTumZcuWyefzqaqqSosWLdLOnTvtWh8AlB16JwBksjSQVldXKxwO6/XXX9fAwID6+/vV3t6umpoau9YHAGWH3gkAmSz/Kdb111+v559/Xlu3bpXH49HZZ5+tSy+91I61AUDZoncCwAmWB9IpU6Zo3bp1dqwFAFyD3gkAJ3DpUAAAABjFQAoAAACjGEgBAABgFAMpAAAAjGIgBQAAgFEMpAAAADCKgRQAAABGMZACAADAKAZSAAAAGMVACgAAAKM86XQ6PZqBR48eVUVF7nOwx+PRqlWririij2zevDlrfmVlpRKJhAp9qioqKjQ4OJj340xmk2893821m8iPRCIFZZWKQnpnqR1Dp2S7Pd/NtZdyfqHZTumdlq9ln694PJ7X9n6/v0grydTf3581PxwOq7e3V8lksqB9B4PBrPsfjsls8q3nu7l2E/lOaarFUkjvLLVj6JRst+e7ufZSzi802ym9k4/sAQAAYBQDKQAAAIxiIAUAAIBRDKQAAAAwioEUAAAARjGQAgAAwCgGUgAAABjFQAoAAACjGEgBAABgFAMpAAAAjGIgBQAAgFG2Xcv+zTffVHNzs44dO6axY8fqqquu0qxZs+zaPQCUHfomAHzEloF0165deuGFF3Tddddp2rRp6unpsWO3AFC26JsAcIItA2lTU5MaGho0Y8YMSVIoFJIkdXd3n9ZkE4mEqqurc1+gz7Y3cYfl9/uHzbeyDq/Xm3X/wzGZTb71fDfX7oR8p8vWNyV7e2epHkPT54+b891ceynnl3rf9KTT6bSVHQwODuqb3/ymGhsbtX37dqVSKZ133nlas2aNXnrpJTU3N2ds39DQoMbGxrwylixZYmWJOdm2bVvRMwBAGr5v+v1+NTU12dI7AaBUWB6je3p6NDg4qB07dujWW29VRUWFNmzYoC1btmjp0qWqra3N2D6RSCgWi+W+wFGa9LOtyefzKRKJqLOzU6lUqqB9BwIBxePxvB9nMpt86/lurt1EfjQaLSjLhOH65iWXXKL6+npbemepHUOnZLs93821l3J+odlO6Z2Wp73jbysvX75c48aNkyRdeOGFQ4315I+hJKmjo0PJZNJqrO1GWlMqlSp43T6fz1LNJrPJLzzfzbU7Id/JcumbdvXOUj2Gps8fN+e7ufZyyC/Vvmn5a5+CweBpjRMAkB19EwAy2fI9pIsWLdJrr72mnp4e9ff369VXX9XHPvYxO3YNAGWJvgkAJ9jyC5oNDQ3q6+vTD37wA/l8Ps2bN08XXXSRHbsGgLJE3wSAE2wZSL1er9auXau1a9fasTsAKHv0TQA4gUuHAgAAwCgGUgAAABjFQAoAAACjGEgBAABgFAMpAAAAjGIgBQAAgFEMpAAAADCKgRQAAABGMZACAADAKAZSAAAAGGXLpUPzEQgEVFGR+xzs8XiKuJoTgsFg1vy+vj75/X75fIU9XRUVFVn3PxyT2eRbz3dz7U7ILzeF9M4lS5bklbF58+aMn3n9uDPfzbWXcn6p981RX3E8Hs9re7/fX6SVZOrv78+aHw6H1dvbq2QyWdC+g8Fg1v0Px2Q2+dbz3Vy7ifxIJFJQVqkYjd556vHi9ePOfDfXXsr5hWY7pXfykT0AAACMYiAFAACAUQykAAAAMIqBFAAAAEYxkAIAAMAoBlIAAAAYxUAKAAAAoxhIAQAAYBQDKQAAAIxiIAUAAIBRDKQAAAAwyraB9OjRo/rGN76hp556yq5dAkDZo3cCgI0D6aZNmzRt2jS7dgcArkDvBADJZ8dO3nzzTY0ZM0bRaFTvv//+0O3d3d3q6enJ2DaRSKi6ujr3BfpsWeKI/H7/sPlW1uH1erPufzgms8m3nu/m2p2QXwqc1jtPPV68ftyZ7+baSzm/1Pum5VV/+OGHampq0s0336zt27dn3Nfa2qrm5uaM2xoaGtTY2Gg11nbRaHTY+yORyCitxFnZ5HPsT7VkyZJRyd62bZvx+ovJib1zpD5YCF4/7sx3c+2m803XXijLA2lTU5MuuOACjR8//rT76uvrVVtbm3FbIpFQLBbLfYGjNOlnW5PP51MkElFnZ6dSqVRB+w4EAorH43k/zmQ2+dbz3Vy7XfLJL8YwVUxO7J2n7p/Xjzvz3Vx7KecXmu2U3mlp2jt48KB2796t22+//Yz3h0IhhUKhjNs6OjqUTCatxBbFSGtKpVIFr9vn81mq2WQ2+YXnu7l2u5jOLxan9s5T98/rx535bq69HPJLtW9aGkj37Nmjrq4uPfjgg5I++i/4dDqthx9+WOvXr7dlgQBQbuidAJDJ0kBaX1+v+fPnD/388ssvq6urS2vXrrW8MAAoV/ROAMhkaSCtrKxUZWVlxs8+ny+vvwQFALehdwJAJlv/YsiJfz0PAE5H7wTgdlw6FAAAAEYxkAIAAMAoBlIAAAAYxUAKAAAAoxhIAQAAYBQDKQAAAIxiIAUAAIBRDKQAAAAwioEUAAAARjGQAgAAwChbLx2ai0AgoIqK3Odgj8dTxNWcEAwGs+b39fXJ7/fL5yvs6aqoqMi6/+GYzHZK/pIlSwp6bD42b96cNd9K/aX+3FvNt8p0vtOMRu889XwxeQ6ZPn/dnO/m2ks53wl924pRX3E8Hs9re7/fX6SVZOrv78+aHw6H1dvbq2QyWdC+g8Fg1v0Px2S2U/JHQ7GOfak/91bzraqqqsorPxKJFHlFZo1G7zz1fDF5Dpk+f92c7+baSzm/0Gyn9E4+sgcAAIBRDKQAAAAwioEUAAAARjGQAgAAwCgGUgAAABjFQAoAAACjGEgBAABgFAMpAAAAjGIgBQAAgFEMpAAAADCKgRQAAABGWb6WfSqV0qZNm7R792719/crEolo1apVOvfcc+1YHwCUHfomAGSyPJAODg4qFArplltu0fjx47Vz50498cQTuuOOOxSJROxYIwCUFfomAGSyPJBWVlaqsbFx6Ofa2lqFw2EdPHiQxgoAZ0DfBIBMlgfSU/X09Ojo0aOKRqPq7u5WT09Pxv2JRELV1dW5L9Bn+xLPyO/3D5tvZR1erzfr/odjMttJ+cVWrGNfDs/9aB2DkdZR7k7um5KM9c5TzxeT55Dp89fN+W6uvZTzndK3C2XrqgcGBvTUU09p0aJFikajampqUnNzc8Y2DQ0NGe8MOMXxfxFkY/JdC9PvmJjOLzaOPfkmndo3Jam1tdVI7xzptVAIXj/WLVmyJO/HbNu2jefepfmmay+UbQPp4OCgnn76aXm9Xl122WWSpPr6etXW1mZsl0gkFIvFcl/gKE362dbk8/kUiUTU2dmpVCpV0L4DgYDi8XjejzOZ7ZT80VCsY1/qz73VfDvkk1+MYarYztQ3JXO989T9mzyHTJ+/pZx/HM+9u/ILzXZK77Tl3/jpdFrPPPOMent7deONN8rr9UqSQqGQQqFQxrYdHR1KJpN2xNpqpDWlUqmC1+3z+SzVbDLbCfnFVqxjXw7PvZV8O5jOL6ZsfVMy1ztP3b/Jc8j0+evmfDfXXg75pdo3bfke0meffVaxWEw33HBDwb93AQBuQt8EgBMsv0Pa1dWl1tZWeb1efec73xm6/YorrtD5559vdfcAUHbomwCQyfJAGg6Hdd9999mwFABwB/omAGTi0qEAAAAwioEUAAAARjGQAgAAwCgGUgAAABjFQAoAAACjGEgBAABgFAMpAAAAjGIgBQAAgFEMpAAAADCKgRQAAABGWb50aL4CgYAqKnKfgz0eTxFXc0IwGMya39fXJ7/fL5+vsKeroqIi6/6HYzLbKfmjoVjH3sRzv3r16oLynMjquVduRqN3nnq+muwfpd67TOZL1l4/pVy7m/PtyDZp1Fccj8fz2t7v9xdpJZn6+/uz5ofDYfX29iqZTBa072AwmHX/wzGZ7ZT80VCsY2/6uS91VVVVedUfiUSKvCKzRqN3nnq+mjyHTb9+Sjlfyv/1Y1e26drdnF9otlN6Jx/ZAwAAwCgGUgAAABjFQAoAAACjGEgBAABgFAMpAAAAjGIgBQAAgFEMpAAAADCKgRQAAABGMZACAADAKAZSAAAAGMVACgAAAKNsuZZ9X1+fnnnmGe3atUtVVVW65JJLdP7559uxawAoS/RNADjBloH0ueeek9fr1Ve+8hUdOnRIP//5zzV58mRNmjTJjt0DQNmhbwLACZY/sk8kEtqxY4caGxsVCAQ0a9Ys1dbWqr293Y71AUDZoW8CQCbL75AePXpUFRUVmjhx4tBtNTU12rt3r7q7u9XT05OxfSKRUHV1de4L9NnyJu6I/H7/sPlW1uH1erPufzgms52UX2zFOvamn/tyUK71D9c3JRnrnaeerybPYdOvn1LOP3Udo5ltunY355f6vzcsrzqRSCgQCGTcNmbMGMXjcbW2tqq5uTnjvoaGBjU2Nua8/+7ubj3wwAOqr69XKBSyuty8dXd3q6mpSfX19YpEIq7Jdko+xz6/7G3bttmW39ra6srnfjQM1zcl0TtLOLsY+fm+rnnu3ZlvunarLA+klZWVQ030uHg8rkAgoPr6etXW1mbcN3bs2Lz239PTo+bmZtXW1hppqibz3Vy72/PdXLsT8ottuL4pid5Zwtluz3dz7abzTdduleWBdMKECRocHNTRo0c1YcIESdKhQ4cUjUYVCoVK8kkBgGIarm9KoncCcB3Lf9RUWVmpuro6NTU1KZFIaN++ffrTn/6khQsX2rE+ACg79E0AyGTLb75efvnl2rhxox544AEFg0FdfvnlfHUJAAyDvgkAJ3jvu++++6zuxO/3a8GCBVq5cqUuvPBC1dTU2LC0j6TTaVVWVmr27Nmn/RHAaDCZ7+ba3Z7v5tqdkD8aitk3JfPPIa8fd+a7uXbT+aZrt8qTTqfTphcBAAAA93LEl1W1tLSora1NR44c0fz583X11VcPu/0rr7yil156SclkUnPnztXatWuHvners7NTGzdu1IEDBzR+/HhddtllOuecc4bdXz6X8HvssceGvitQkgYGBjRx4kR94QtfkCQ9+OCD6u3tlcfjkSTNmDFDN910k235TU1N+t3vfiev1zt02x133KGzzjpLknTw4EE988wzisViikajuvLKKzVlyhRbsrdu3aq2tjYdO3ZMVVVVWrp0qT7xiU8M3Z9L7bnmpdNpvfDCC9q+fbsk6YILLtCqVauG9p1vnfnm21GrlXy7j3O++cU4z/N5ndv9Gi9Xbu6dJvtmvvn0ztLunSb7puSe3umIgXTcuHFauXKldu3apWQyOey27777rl566SXdfPPNGjdunDZs2KCmpiatXr1akvTUU09p+vTpuvHGG7Vz5049/vjjuueee4b9Qul8LuH3uc99LuPnRx55RGeffXbGbTfccENeBznfSwjOmzdP11577Wm3p1IpbdiwQStWrNDSpUu1bds2bdiwQXfffXfWL8rNJzudTuvqq69WTU2NOjs79eijjyoUCmnBggU5155rXmtrq95++22tX79eHo9HP/3pTxUOh7V06dKC6sw3345areRL9h7nfPOLcZ7n+jovxmu8XLm5d5rsm/nm0ztLu3ea7JuSe3qn5b+yt8PcuXNVV1enYDA44rZtbW1avHixJk2apGAwqIaGBrW1tUmS/vKXv+jgwYNqbGyU3+/X3LlzVVNTox07dmTdn5VL+HV2dmrfvn2W/jLWzksI7tmzR4ODg1qxYoV8Pp9WrFihdDqtP//5z7Zkf/KTn9TUqVPl9Xo1ceJE1dbWav/+/UWpta2tTRdeeKHGjx+vUCikj3/840PHOd86C8m3WqvV/OGMRv0ns+M8l3J/ndv9Gi9nbu2dJvtmIfn0ztLtnab7puSe3umId0jzEYvFdN555w39XFNTo97eXvX19SkWiykSiWT8Mm9NTY1isVjW/Y10Cb/htLe3a+bMmaddEeHpp59WOp3W5MmTtWbNGk2ePNnW/HfeeUff+ta3NG7cOC1btkxLly6V9NFzU1NTM/RxwPF9xWIxnXvuubbWnk6ntW/fPtXX1+dcez55sVgs47EnH8d867RabyG12pFv13EuNP84O87zfNj9GsdHyql3muybVmund5ZW7yyVvimVfu8suYH01EvujRkzRtJHVznJdjm+7u7unPd3/DGnXkXlTNrb27Vy5cqM26699lpNmTJF6XRaLS0tevTRR3XXXXdl/S+bfPPnzZun+vp6jR07VgcOHNDjjz+uMWPGaMGCBXnvy0rtL774otLptBYvXpxz7fnknek4JxIJpdPpgtdd6OMKqdVqvp3HuZD8k9lxnufD7tc4PlJOvdNk3ywk/2T0ztLqnaXSN8+01lLrnUUfSB955JGs/yUxY8YM3XbbbXnt79RL7h3/50AgcMbL8bW1temDDz4449vrM2bM0GWXXTbsJfyy2bt3r3p6ejR37tyM22fOnDn0zxdddJGam5v17W9/+4z7KCT/5N9ZmTlzppYvX64dO3ZowYIFp9V/8nN/6nWxrdTe0tKi9vZ2rVu3LuP3bk6tva2tTfv27Ru6BOJIl0s82ZmOc2VlpTweT177GW6fuTyu0Fqt5udznHOpI9/843I9z3OpP1f5vsZzqb0Uubl3vvvuu+rt7T1j7yx23zye39/ff1rfLDT/OHpn6fXOUumbZ1prqfXOog+k69ats3V/0WhUhw8f1vz58yV9dLm96upqVVVVKRqNqrOzM+NJjkQiWrly5dBb9qdKJBLDXsIvm/b2dtXV1Y14MMPhsFatWpXxNrod+cd5PB4d/+auaDSql19+Wel0Wh6PR+vWrdODDz6otWvXnvHjiEKyt2/frpdeeknr1q3T+PHjc16bNPLlEk92/DhPnz79tO1OrVOSDh8+nPUYF5JvtVY78rPtf7Tql3I/z3OpP1f5vsYPHTqU8QcT5cLNvXPdunX64Q9/mLV3FrNvSh/9i/2aa67J+jEuvdM9vbNU+qZU+r3TEX/UNDAwoGQyqXQ6rXQ6rWQyqYGBgTNuu3DhQm3fvl1HjhxRf3+/tmzZokWLFkmSJk6cqMmTJ+vFF19UMpnUW2+9pcOHD5/2XyknK+QSfslkUn/84x+Hco/r6urSvn37lEqllEwmtXXrVvX19WX8V5HV/Lffflv9/f1Kp9M6cOCAWlpahhr27NmzVVFRoZaWFqVSKbW0tEjSaX/lV2j2G2+8od/85je66aabhr5CI5/a88lbuHChXnnlFXV3d6u7u1uvvPLK0POdb52F5Fut1Wq+nce5kHzJ3vNcyv11bvdrvJy5tXea7JuF5NM7c6/Var7pY21335Tc0zsd8cX4TU1Np3000tDQoMbGRnV1demhhx7SnXfeqXA4LEl6+eWXtXXr1qzfs/XLX/5S7733Xl7fpbdx40bt3r1bwWBQq1atGvqOsb179+qxxx7TvffeO7T9m2++qRdeeEFf/OIXM345+siRI3ryySfV2dkpn8+nyZMna9WqVZo2bZpt+U8++aR27dqlVCqlUCikpUuXasWKFUP7Ovk71iZOnKhPf/rTI34Paa7Z3/ve99Td3Z3x/W7nn3++rrjiipxrz5Z3alY6ndbmzZszvktv9erVZ/wuvVzqzDffjlqt5Nt9nPPNl+w/z7O9zhcvXlz013i5cnPvNNk3882nd5Z27zTZNyX39E5HDKQAAABwL0d8ZA8AAAD3YiAFAACAUQykAAAAMIqBFAAAAEYxkAIAAMAoBlIAAAAYxUAKAAAAoxhIAQAAYBQDKQAAAIxiIAUAAIBRDKQAAAAwioEUAAAARjGQAgAAwCgGUgAAABjFQAoAAACjGEgBAABgFAMpAAAAjGIgBQAAgFEMpAAAADCKgRQAAABGMZACAADAKAZSAAAAGMVACgAAAKMYSAEAAGAUAykAAACMYiAFAACAUQykAAAAMIqBFAAAAEYxkAIAAMAoBlIAAAAYxUAKAAAAoxhIAQAAYBQDKQAAAIxiIAUAAIBRDKQAAAAwioEUAAAARjGQAgAAwCgGUgAAABjFQAoAAACjGEgBAABgFAMpAAAAjGIgBQAAgFEMpAAAADCKgRQAAABGMZACAADAKAZSAAAAGMVACgAAAKMYSAEAAGAUAykAAACMYiAFAACAUQykAAAAMIojYWGGAAAgAElEQVSBFAAAAEYxkAIAAMAoBlIAAAAYxUAKAAAAoxhIAQAAYBQDKYz78MMPtWzZMi1cuFDz5s3Tv/7rv562TTwe1/XXX685c+Zo+fLl2rNnz+gvFAAcZmBgQIsXL9batWtPu4++iVLCQArjAoGAfvvb36q9vV1tbW16/vnn9eqrr2Zs81//9V+KRCJ699139aUvfUlf/epXDa0WAJzj+9//vurq6s54H30TpYSBFMZ5PB6NHTtWkpRMJpVMJuXxeDK22bhxo26++WZJ0nXXXaff/OY3SqfTo75WAHCKAwcOaNOmTfrbv/3bM95P30QpYSCFIwwMDGjRokWaNGmSVq9ereXLl2fc/95772nGjBmSJJ/Pp/Hjx+vo0aMmlgoAjvDFL35R//7v/66KijP/q5y+iVLCQApH8Hq9amtr04EDB/Taa6/pD3/4g+klAYBjPfvss5o0aZLq6+tNLwWwBQMpHCUcDquxsVHPP/98xu3Tpk3T/v37JUmpVErHjh3ThAkTTCwRAIzbunWrnnnmGc2ePVuf/exn9dvf/laf+9znMrahb6KUMJDCuFgspq6uLklSf3+/Nm/erPPOOy9jmyuvvFL//d//LUl68skndfHFF5/2e6YA4Bb333+/Dhw4oD179mjDhg26+OKL9dhjj2VsQ99EKfGZXgBw8OBB3XzzzRoYGNDg4KA+85nPaO3atfqXf/kXLVmyRFdeeaVuu+02/c3f/I3mzJmjs846Sxs2bDC9bABwHPomSpUnzZ/cAQAAwCA+sgcAAIBRDKQAAAAwioEUAAAARjGQAgAAwCgGUgAAABg16l/71NHRkdf2fr9f0WhUsVhMyWSyoMxAIKB4PF7QY03mu7n2csh3c+0m8qdOnVpQVqlwW+902/nrpHw3117K+YVmO6V3uuId0mzX+XVDvptrN53v5tqdkA/reP24M9/NtZNvjjurBgAAgGMwkAIAAMConH6HtKWlRW1tbTpy5Ijmz5+vq6++eui+3bt3a9OmTTp27JimT5+uq666SuFwuGgLBoBSQN8EgNzl9A7puHHjtHLlSi1evDjj9t7eXv3iF7/QxRdfrK9+9auaOnWqnnjiiaIsFABKCX0TAHKX00A6d+5c1dXVKRgMZtz+1ltvKRqNat68efL7/frUpz6lw4cPKxaLFWWxAFAq6JsAkDtLX/sUi8U0efLkoZ8rKysViUQUi8UUjUbV3d2tnp6ejMckEglVV1fnvkCfL+P/C+H1euX3+wt6rMl8N9deDvlurt0J+U41Ut+URO80nO32fDfXXsr5pd43La06kUioqqoq47YxY8YMfX9Wa2urmpubM+5vaGhQY2NjXjlLliyxssycbNu2bdj7I5FI0dfgxGzyOfZuzi+GkfqmRO+0i+nzx835bq7ddL7p2gtlaSCtrKw87ctb4/G4AoGAJKm+vl61tbUZ9ycSibw+mhqtST/bmnw+nyKRiDo7O5VKpQrad6Ffcmsym3zr+W6u3UT+8XcXnW6kvinROyX3nb9Oyndz7aWcX2i2U3qnpY4VjUbV3t4+9HMikdD7778/VFwoFFIoFMp4TEdHR8FXLyimkdaUSqUKXrfP57NUs8ls8gvPd3PtTsh3qpH6pkTvlMyfP27Od3Pt5ZBfqn0zpz9qGhgYUDKZVDqdVjqdVjKZ1MDAgOrq6nTkyBHt2LFDyWRSzc3Nqqmpccy0DQCm0DcBIHc5vUO6ZcuWjN9neuONN4Z+n+kzn/mMnnvuOT399NOaNm2arrvuuqItFgBKBX0TAHKX00Da2NiY9ZfpzznnHN199922LgoASh19EwByx6VDAQAAYBQDKQAAAIxiIAUAAIBRDKQAAAAwioEUAAAARjGQAgAAwCgGUgAAABjFQAoAAACjGEgBAABgFAMpAAAAjMrp0qF2CgQCqqjIfQ72eDxFXM0JwWAwa35fX5/8fr98vsKeroqKiqz7H47JbPKt57u5difklxu39U7T54+b891ceynnl3rfHPUVx+PxvLb3+/1FWkmm/v7+rPnhcFi9vb1KJpMF7TsYDGbd/3BMZpNvPd/NtZvIj0QiBWWVCrf1Tredv07Kd3PtpZxfaLZTeicf2QMAAMAoBlIAAAAYxUAKAAAAoxhIAQAAYBQDKQAAAIxiIAUAAIBRDKQAAAAwioEUAAAARjGQAgAAwCgGUgAAABjFQAoAAACjGEgBAABglM/qDjo7O7Vp0yYdOHBAXq9Xc+fO1aWXXiqv12vH+gCgLNE7AeAEy++Qbtq0SdXV1fryl7+s9evXa+/evXr99dftWBsAlC16JwCcYPkd0q6uLi1btkx+v19+v19z5sxRLBaTJHV3d6unpydj+0Qioerq6twX6LO8xJz4/f5h862sw+v1Zt3/cExmk2893821OyHf6eidwzN9/rg53821l3J+qfdNy6tesWKF/vCHP2j27Nn68MMPtXPnTl188cWSpNbWVjU3N2ds39DQoMbGRquxtotGo8PeH4lERmklzsomn2Pv5vxioncWn+nzx835bq7ddL7p2gtleSCdNWuWWltbdf/99yudTmvhwoU677zzJEn19fWqra3N2D6RSAy9C5DTAkdp0s+2Jp/Pp0gkos7OTqVSqYL2HQgEFI/H836cyWzyree7uXYT+SMNRk5D7xye285fJ+W7ufZSzi802ym901LHGhwc1GOPPab6+nrddtttSiQS2rhxozZv3qw1a9YoFAopFAplPKajo0PJZNLSoothpDWlUqmC1+3z+SzVbDKb/MLz3Vy7E/KdjN45MtPnj5vz3Vx7OeSXat+09EdN/f39OnbsmJYtWyafz6eqqiotWrRIO3futGt9AFB26J0AkMnSQFpdXa1wOKzXX39dAwMD6u/vV3t7u2pqauxaHwCUHXonAGSy/EtG119/vZ5//nlt3bpVHo9HZ599ti699FI71gYAZYveCQAnWB5Ip0yZonXr1tmxFgBwDXonAJzApUMBAABgFAMpAAAAjGIgBQAAgFEMpAAAADCKgRQAAABGMZACAADAKAZSAAAAGMVACgAAAKMYSAEAAGAUAykAAACMsnzp0HwFAgFVVOQ+B3s8niKu5oRgMJg1v6+vT36/Xz5fYU9XRUVF1v0Px2Q2+dbz3Vy7E/LLjdt6p+nzx835bq69lPNLvW+O+orj8Xhe2/v9/iKtJFN/f3/W/HA4rN7eXiWTyYL2HQwGs+5/OCazybee7+baTeRHIpGCskqF23qn285fJ+W7ufZSzi802ym9k4/sAQAAYBQDKQAAAIxiIAUAAIBRDKQAAAAwioEUAAAARjGQAgAAwCgGUgAAABjFQAoAAACjGEgBAABgFAMpAAAAjGIgBQAAgFG2Xcv+zTffVHNzs44dO6axY8fqqquu0qxZs+zaPQCUHfomAHzEloF0165deuGFF3Tddddp2rRp6unpsWO3AFC26JsAcIItA2lTU5MaGho0Y8YMSVIoFJIkdXd3n9ZkE4mEqqurc1+gz7Y3cYfl9/uHzbeyDq/Xm3X/wzGZTb71fDfX7oR8p8vWNyV6p2T+/HFzvptrL+X8Uu+bllc9ODiojo4O1dbW6vvf/75SqZTOO+88rVmzRq2trWpubs7YvqGhQY2NjVZjbReNRoe9PxKJjNJKnJVNPsfezfnFMlzf9Pv99E6bmD5/3Jzv5tpN55uuvVCWB9Kenh4NDg5qx44duvXWW1VRUaENGzZoy5YtWrp0qWprazO2TyQSisViuS9wlCb9bGvy+XyKRCLq7OxUKpUqaN+BQEDxeDzvx5nMJt96vptrN5E/0mDkJMP1zUsuuUT19fWu751uO3+dlO/m2ks5v9Bsp/ROyx3r+NvKy5cv17hx4yRJF1544VBjPfljKEnq6OhQMpm0Gmu7kdaUSqUKXrfP57NUs8ls8gvPd3PtTsh3slz6ptt7p+nzx835bq69HPJLtW9a/tqnYDB4WuMEAGRH3wSATLZ8D+miRYv02muvqaenR/39/Xr11Vf1sY99zI5dA0BZom8CwAm2/JJRQ0OD+vr69IMf/EA+n0/z5s3TRRddZMeuAaAs0TcB4ARbBlKv16u1a9dq7dq1duwOAMoefRMATuDSoQAAADCKgRQAAABGMZACAADAKAZSAAAAGMVACgAAAKMYSAEAAGAUAykAAACMYiAFAACAUQykAAAAMIqBFAAAAEbZcunQfAQCAVVU5D4HezyeIq7mhGAwmDW/r69Pfr9fPl9hT1dFRUXW/Q/HZDb51vPdXLsT8suN23qn6fPHzflurr2Y+atXry5on/nYsmVLyfbNUV9xPB7Pa3u/31+klWTq7+/Pmh8Oh9Xb26tkMlnQvoPBYNb9D8dkNvnW891cu4n8SCRSUFapcFvvdNv566R8N9fuhHwrqqqq8s52Su/kI3sAAAAYxUAKAAAAoxhIAQAAYBQDKQAAAIxiIAUAAIBRDKQAAAAwioEUAAAARjGQAgAAwCgGUgAAABjFQAoAAACjGEgBAABglG0D6dGjR/WNb3xDTz31lF27BICyR+8EABsH0k2bNmnatGl27Q4AXIHeCQCSz46dvPnmmxozZoyi0ajef//9odu7u7vV09OTsW0ikVB1dXXuC/TZssQR+f3+YfOtrMPr9Wbd/3BMZpNvPd/NtTshvxTQO7Mzff64Od/NtTsh36pS7ZuWV/3hhx+qqalJN998s7Zv355xX2trq5qbmzNua2hoUGNjo9VY20Wj0WHvj0Qio7QSZ2WTz7F3c34x0TuLz/T54+Z8N9duOt907YWyPJA2NTXpggsu0Pjx40+7r76+XrW1tRm3JRIJxWKx3Bc4SpN+tjX5fD5FIhF1dnYqlUoVtO9AIKB4PJ7340xmk2893821m8gfaTByGnrn8Nx2/jop3821OyHfqnyzndI7LXWsgwcPavfu3br99tvPeH8oFFIoFMq4raOjQ8lk0kpsUYy0plQqVfC6fT6fpZpNZpNfeL6ba3dCvpPRO0dm+vxxc76ba3dCvlWl2jctDaR79uxRV1eXHnzwQUkf/Rd8Op3Www8/rPXr19uyQAAoN/ROAMhkaSCtr6/X/Pnzh35++eWX1dXVpbVr11peGACUK3onAGSyNJBWVlaqsrIy42efz5fXX4ICgNvQOwEgk62/9e7EvwAFAKejdwJwOy4dCgAAAKMYSAEAAGAUAykAAACMYiAFAACAUQykAAAAMIqBFAAAAEYxkAIAAMAoBlIAAAAYxUAKAAAAoxhIAQAAYJStlw7NRSAQUEVF7nOwx+Mp4mpOCAaDWfP7+vrk9/vl8xX2dFVUVGTd/3BMZpNvPd/NtTshv9y4rXeaPn/cnO/m2p2Qb0Up981RX3E8Hs9re7/fX6SVZOrv78+aHw6H1dvbq2QyWdC+g8Fg1v0Px2Q2+dbz3Vy7ifxIJFJQVqlwW+902/nrpHw31+6EfCuqqqryznZK7+QjewAAABjFQAoAAACjGEgBAABgFAMpAAAAjGIgBQAAgFEMpAAAADCKgRQAAABGMZACAADAKAZSAAAAGMVACgAAAKMYSAEAAGCU5WvZp1Ipbdq0Sbt371Z/f78ikYhWrVqlc8891471AUDZoW8CQCbLA+ng4KBCoZBuueUWjR8/Xjt37tQTTzyhO+64Q5FIxI41AkBZoW8CQCbLA2llZaUaGxuHfq6trVU4HNbBgwfl9XrV09OTsX0ikVB1dXXuC/RZXmJO/H7/sPlW1uH1erPufzgms8m3nu/m2p2Q72TD9c1IJKLu7m7X907T54+b891cuxPyrSrVvmn7qnt6enT06FFFo1G1traqubk54/6GhoaMRuwU0Wh02PtNvmth+h0T8jn2J1uyZMmoZG/bts14/aPl5L4pid5pE9Pnj5vz3Vy76XzTtRfK1oF0YGBATz31lBYtWqRoNKr6+nrV1tZmbJNIJBSLxXJf4ChN+tnW5PP5FIlE1NnZqVQqVdC+A4GA4vF43o8zmU2+9Xw3126XfPJHGoyc6tS+KYneKfPnr5vz3Vy7E/KtyjfbKb3Tto41ODiop59+Wl6vV5dddpkkKRQKKRQKZWzX0dGhZDJpV6xtRlpTKpUqeN0+n89SzSazyS88382128V0frGdqW9K9E7J/Pnr5nw31+6EfKtKtW/a8rVP6XRazzzzjHp7e3X99dfL6/XasVsAKFv0TQA4wZaB9Nlnn1UsFtMNN9xQ8C8CA4Cb0DcB4ATLH9l3dXWptbVVXq9X3/nOd4Zuv+KKK3T++edb3T0AlB36JgBksjyQhsNh3XfffTYsBQDcgb4JAJm4dCgAAACMYiAFAACAUQykAAAAMIqBFAAAAEYxkAIAAMAoBlIAAAAYxUAKAAAAoxhIAQAAYBQDKQAAAIxiIAUAAIBRli8dmq9AIKCKitznYI/HU8TVnBAMBrPm9/X1ye/3y+cr7OmqqKjIuv/hmMx2Sv6SJUsKeqwTbd68OedtnfDcW823ynS+07itd5o+f92c7+banZBvRSn3zVFfcTwez2t7v99fpJVk6u/vz5ofDofV29urZDJZ0L6DwWDW/Q/HZLZT8stJPs+DE557q/lWVVVV5ZUfiUSKvCKz3NY7TZ+/bs53c+1OyLci374pOad38pE9AAAAjGIgBQAAgFEMpAAAADCKgRQAAABGMZACAADAKAZSAAAAGMVACgAAAKMYSAEAAGAUAykAAACMYiAFAACAUQykAAAAMMqWa9n39fXpmWee0a5du1RVVaVLLrlE559/vh27BoCyRN8EgBNsGUife+45eb1efeUrX9GhQ4f085//XJMnT9akSZPs2D0AlB36JgCcYPkj+0QioR07dqixsVGBQECzZs1SbW2t2tvb7VgfAJQd+iYAZLL8DunRo0dVUVGhiRMnDt1WU1OjvXv3qru7Wz09PRnbJxIJVVdX575Any1v4o7I7/cPm29lHV6vN+v+h2My20n55SKf58Epz73pY2A6v1iG65uS6J0yf/66Od/NtTsh36pS7ZuWV51IJBQIBDJuGzNmjOLxuFpbW9Xc3JxxX0NDgxobG3Pef3d3tx544AHV19crFApZXW7euru71dTUpPr6ekUiEddkOyXf9LFvbW01ku+E5z5b/rZt24zml4Ph+qYkemcJZ7s93821j5Rf7N5punarLA+klZWVQ030uHg8rkAgoPr6etXW1mbcN3bs2Lz239PTo+bmZtXW1hppqibz3Vy72/PdXLsT8ottuL4pid5Zwtluz3dz7abzTdduleWBdMKECRocHNTRo0c1YcIESdKhQ4cUjUYVCoVK8kkBgGIarm9KoncCcB3Lf9RUWVmpuro6NTU1KZFIaN++ffrTn/6khQsX2rE+ACg79E0AyGTLb75efvnl2rhxox544AEFg0FdfvnlfHUJAAyDvgkAJ3jvu++++6zuxO/3a8GCBVq5cqUuvPBC1dTU2LC0j6TTaVVWVmr27Nmn/RHAaDCZ7+ba3Z7v5tqdkD8aitk3JfPPIa8fd+a7uXbT+aZrt8qTTqfTphcBAAAA93LEl1W1tLSora1NR44c0fz583X11VcPu/0rr7yil156SclkUnPnztXatWuHvners7NTGzdu1IEDBzR+/HhddtllOuecc4bdXz6X8HvssceGvitQkgYGBjRx4kR94QtfkCQ9+OCD6u3tlcfjkSTNmDFDN910k235TU1N+t3vfiev1zt02x133KGzzjpLknTw4EE988wzisViikajuvLKKzVlyhRbsrdu3aq2tjYdO3ZMVVVVWrp0qT7xiU8M3Z9L7bnmpdNpvfDCC9q+fbsk6YILLtCqVauG9p1vnfnm21GrlXy7j3O++cU4z/N5ndv9Gi9Xbu6dJvtmvvn0ztLunSb7puSe3umIgXTcuHFauXKldu3apWQyOey27777rl566SXdfPPNGjdunDZs2KCmpiatXr1akvTUU09p+vTpuvHGG7Vz5049/vjjuueee4b9Qul8LuH3uc99LuPnRx55RGeffXbGbTfccENeBznfSwjOmzdP11577Wm3p1IpbdiwQStWrNDSpUu1bds2bdiwQXfffXfWL8rNJzudTuvqq69WTU2NOjs79eijjyoUCmnBggU5155rXmtrq95++22tX79eHo9HP/3pTxUOh7V06dKC6sw3345areRL9h7nfPOLcZ7n+jovxmu8XLm5d5rsm/nm0ztLu3ea7JuSe3qn5b+yt8PcuXNVV1enYDA44rZtbW1avHixJk2apGAwqIaGBrW1tUmS/vKXv+jgwYNqbGyU3+/X3LlzVVNTox07dmTdn5VL+HV2dmrfvn2W/jLWzksI7tmzR4ODg1qxYoV8Pp9WrFihdDqtP//5z7Zkf/KTn9TUqVPl9Xo1ceJE1dbWav/+/UWpta2tTRdeeKHGjx+vUCikj3/840PHOd86C8m3WqvV/OGMRv0ns+M8l3J/ndv9Gi9nbu2dJvtmIfn0ztLtnab7puSe3umId0jzEYvFdN555w39XFNTo97eXvX19SkWiykSiWT8Mm9NTY1isVjW/Y10Cb/htLe3a+bMmaddEeHpp59WOp3W5MmTtWbNGk2ePNnW/HfeeUff+ta3NG7cOC1btkxLly6V9NFzU1NTM/RxwPF9xWIxnXvuubbWnk6ntW/fPtXX1+dcez55sVgs47EnH8d867RabyG12pFv13EuNP84O87zfNj9GsdHyql3muybVmund5ZW7yyVvimVfu8suYH01EvujRkzRtJHVznJdjm+7u7unPd3/DGnXkXlTNrb27Vy5cqM26699lpNmTJF6XRaLS0tevTRR3XXXXdl/S+bfPPnzZun+vp6jR07VgcOHNDjjz+uMWPGaMGCBXnvy0rtL774otLptBYvXpxz7fnknek4JxIJpdPpgtdd6OMKqdVqvp3HuZD8k9lxnufD7tc4PlJOvdNk3ywk/2T0ztLqnaXSN8+01lLrnUUfSB955JGs/yUxY8YM3XbbbXnt79RL7h3/50AgcMbL8bW1temDDz4449vrM2bM0GWXXTbsJfyy2bt3r3p6ejR37tyM22fOnDn0zxdddJGam5v17W9/+4z7KCT/5N9ZmTlzppYvX64dO3ZowYIFp9V/8nN/6nWxrdTe0tKi9vZ2rVu3LuP3bk6tva2tTfv27Ru6BOJIl0s82ZmOc2VlpTweT177GW6fuTyu0Fqt5udznHOpI9/843I9z3OpP1f5vsZzqb0Uubl3vvvuu+rt7T1j7yx23zye39/ff1rfLDT/OHpn6fXOUumbZ1prqfXOog+k69ats3V/0WhUhw8f1vz58yV9dLm96upqVVVVKRqNqrOzM+NJjkQiWrly5dBb9qdKJBLDXsIvm/b2dtXV1Y14MMPhsFatWpXxNrod+cd5PB4d/+auaDSql19+Wel0Wh6PR+vWrdODDz6otWvXnvHjiEKyt2/frpdeeknr1q3T+PHjc16bNPLlEk92/DhPnz79tO1OrVOSDh8+nPUYF5JvtVY78rPtf7Tql3I/z3OpP1f5vsYPHTqU8QcT5cLNvXPdunX64Q9/mLV3FrNvSh/9i/2aa67J+jEuvdM9vbNU+qZU+r3TEX/UNDAwoGQyqXQ6rXQ6rWQyqYGBgTNuu3DhQm3fvl1HjhxRf3+/tmzZokWLFkmSJk6cqMmTJ+vFF19UMpnUW2+9pcOHD5/2XyknK+QSfslkUn/84x+Hco/r6urSvn37lEqllEwmtXXrVvX19WX8V5HV/Lffflv9/f1Kp9M6cOCAWlpahhr27NmzVVFRoZaWFqVSKbW0tEjSaX/lV2j2G2+8od/85je66aabhr5CI5/a88lbuHChXnnlFXV3d6u7u1uvvPLK0POdb52F5Fut1Wq+nce5kHzJ3vNcyv11bvdrvJy5tXea7JuF5NM7c6/Var7pY21335Tc0zsd8cX4TU1Np3000tDQoMbGRnV1demhhx7SnXfeqXA4LEl6+eWXtXXr1qzfs/XLX/5S7733Xl7fpbdx40bt3r1bwWBQq1atGvqOsb179+qxxx7TvffeO7T9m2++qRdeeEFf/OIXM345+siRI3ryySfV2dkpn8+nyZMna9WqVZo2bZpt+U8++aR27dqlVCqlUCikpUuXasWKFUP7Ovk71iZOnKhPf/rTI34Paa7Z3/ve99Td3Z3x/W7nn3++rrjiipxrz5Z3alY6ndbmzZszvktv9erVZ/wuvVzqzDffjlqt5Nt9nPPNl+w/z7O9zhcvXlz013i5cnPvNNk3882nd5Z27zTZNyX39E5HDKQAAABwL0d8ZA8AAAD3YiAFAACAUQykAAAAMIqBFAAAAEYxkAIAAMAoBlIAAAAYxUAKAAAAoxhIAQAAYBQDKQAAAIxiIAUAAIBRDKQAAAAwioEUAAAARjGQAgAAwCgGUgAAABjFQAoAAACjGEgBAABgFAMpAPz/fvazn2nNmjUjbrd+/Xp94xvfGIUVAYA7eNLpdNr0IgAAAOBevEMKoKykUinTSwAA5ImBFEBJmD17tu6//37NnTtXkUhE69at04cffqgXX3xR06dP17e//W1NnjxZ69atkyQ9++yzWrRokcLhsD7+8Y/rjTfeGNrX/v37dc011ygajWrChAm66667JEk/+clP9MlPflKSlE6n9aUvfUmTJk1SKBTSggUL9Ic//EGSdMstt+if//mfh/b34x//WHPmzNFZZ52lK6+8Uh0dHUP3eTwePfzwwzr33HMVDod15513ig+mACATAymAkvGzn/1Mv/71r7Vr1y698847+uY3vylJOnTokN5//33t3btXP/rRj/T73/9et956q/7jP/5DR48e1e23364rr7xS8XhcAwMDWrt2rWbNmqU9e/bovffe02c/+9nTsv7v//5PW7Zs0TvvvKNjx47p8ccf14QJE07b7re//a3+8R//UY8//rgOHjyoWbNmnba/Z599Vq+//rreeOMNPf744/ILYC4AAAL4SURBVPr1r39dnCcIAEoUAymAknHXXXdpxowZOuuss3Tvvffqf/7nfyRJFRUV+vrXv65AIKBgMKgf/ehHuv3227V8+XJ5vV7dfPPNCgQCevXVV/Xaa6+po6NDDzzwgKqrqzVmzJihd0VP5vf79cEHH+jtt99WOp1WXV2dpkyZctp2P/vZz3TrrbfqggsuUCAQ0P33369XXnlFe/bsGdrma1/7msLhsGbOnKnGxka1tbUV7TkCgFLEQAqgZMyYMWPon2fNmjX00Xg0GtWYMWOG7tu7d6+++93vKhwOD/1v//796ujo0P79+zVr1iz5fL5hsy6++GLddddduvPOOzVp0iT93d/9nbq7u0/brqOjQ7NmzRr6eezYsZowYYLee++9odsmT5489M9VVVXq6enJv3gAKGMMpABKxv79+4f+ed++fZo6daqkj35P82QzZszQvffeq66urqH/9fX16YYbbtCMGTO0b9++nP746Z577lFra6t27Nihd955Rw888MBp20ydOlV79+4d+rm3t1dHjx7VtGnTCi0TAFyHgRRAyXjooYd04MABvf/++/p//+//6frrrz/jdp///Of18MMPq6WlRel0Wr29vdq0aZM++OADLVu2TFOmTNHXvvY19fb26sMPP9TWrVtP28frr7+ulpYWJZPJoY/2KypOb5k33HCDHnnkEbW1tSkej+uf/umftHz5cs2ePdvu8gGgbDGQAigZf/3Xf601a9bor/7qr3TOOedk/KX7yZYsWaIf//jHuuuuuxSJRDRnzhz95Cc/kSR5vV797//+r959913NnDlT06dP1y9+8YvT9tHd3a3Pf/7zikQimjVrliZMmKB/+Id/OG27VatW6Rvf+IauvfZaTZkyRbt27dKGDRtsrRsAyh1fjA+gJMyePVv/+Z//qVWrVpleCgDAZrxDCgAAAKMYSAEAAGAUH9kDAADAKN4hBQAAgFEMpAAAADCKgRQAAABGMZACAADAKAZSAAAAGMVACgAAAKP+P7BhTrB/nuPJAAAAAElFTkSuQmCC\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ggplot(data=precision_df,aesthetics=aes(x = \"precision\"))+geom_histogram()+facet_wrap(\"split\")" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAFGtJREFUeJzt3X+QXlV9x/H3t6RiZVsIjd3GkDE4TX+gjGB2KNaZdlerIM4YnFIah2pQnKhFx5najlGnrf3BlHZKmRFbbVooWCkrRZmk/CjFkNVxRlBikQQYZME4JA1JMZhxFanQb/94Tpxr2N3n9+56eL9mntl7zzn33u89z/LZm/v8IDITSVK9fmKxC5AkDZdBL0mVM+glqXIGvSRVzqCXpMoZ9JJUOYNekirXNugj4vkR8eWI+FpE3BcRf1raT46IuyJiOiI+HRHPK+3HlvXp0r9muKcgSZpPJ1f0TwGvzsyXA6cBZ0fEmcBfAZdn5i8ATwAXlfEXAU+U9svLOEnSIoluPhkbES8Avgi8G7gZ+PnMfDoiXgl8JDPPiojbyvKXImIZ8BjwwpznQCtWrMg1a9b0dALf/e53Oe6443radpiWal2wdGuzru5YV3dqrGvnzp2PZ+YL2w7MzLYP4BjgHmCG1hX6CmC60b8a2F2WdwMnNfoeBlbMt/9169Zlr3bs2NHztsO0VOvKXLq1WVd3rKs7NdYF3J0dZHi3V/QnADcCfwRcna3bM0TEauDWzHxZROwGzs7MvaXvYeBXM/Pxo/a1CdgEMDo6um5ycrLjOppmZmYYGRnpadthWqp1wdKtzbq6Y13dqbGuiYmJnZk51nZgJ38Nmg/gj4E/BB4HlpW2VwK3leXbgFeW5WVlXMy3T6/oF9ZSrc26umNd3amxLjq8ou/kXTcvLFfyRMRPAa8FHgB2AOeVYRuBrWV5W1mn9N9RCpIkLYJlHYxZCVwTEcfQepfO9Zl5U0TcD0xGxF8A/wVcWcZfCfxLREwDh4ANQ6hbktShtkGfmfcCp8/S/ghwxizt3wd+eyDVSZL65idjJalyBr0kVc6gl6TKGfSSVLlO3nWzpO3ad5gLN9+8KMfec+kbFuW4ktQNr+glqXIGvSRVzqCXpMoZ9JJUOYNekipn0EtS5Qx6SaqcQS9JlTPoJalyBr0kVc6gl6TKGfSSVDmDXpIqZ9BLUuUMekmqnEEvSZUz6CWpcga9JFXOoJekyhn0klQ5g16SKmfQS1Ll2gZ9RKyOiB0RcX9E3BcR7yvtH4mIfRFxT3mc09jmgxExHREPRsRZwzwBSdL8lnUw5mng/Zn51Yj4aWBnRNxe+i7PzL9pDo6IU4ANwEuBFwGfi4hfzMxnBlm4JKkzba/oM3N/Zn61LH8HeABYNc8m64HJzHwqM78BTANnDKJYSVL3urpHHxFrgNOBu0rTeyLi3oi4KiKWl7ZVwKONzfYy/x8GSdIQRWZ2NjBiBPg8cElmfjYiRoHHgQT+HFiZmW+PiI8Bd2bmp8p2VwK3ZuYNR+1vE7AJYHR0dN3k5GRPJ3Dw0GEOPNnTpn07ddXxc/bNzMwwMjKygNV0bqnWZl3dsa7u1FjXxMTEzswcazeuk3v0RMRPAp8Brs3MzwJk5oFG/z8CN5XVfcDqxuYnlbYfkZlbgC0AY2NjOT4+3kkpz3LFtVu5bFdHpzFwey4Yn7NvamqKXs9p2JZqbdbVHevqznO5rk7edRPAlcADmfm3jfaVjWFvAnaX5W3Ahog4NiJOBtYCXx5cyZKkbnRyKfwq4C3Aroi4p7R9CHhzRJxG69bNHuCdAJl5X0RcD9xP6x07F/uOG0laPG2DPjO/CMQsXbfMs80lwCV91CVJGhA/GStJlTPoJalyBr0kVc6gl6TKGfSSVDmDXpIqZ9BLUuUMekmqnEEvSZUz6CWpcga9JFXOoJekyhn0klQ5g16SKmfQS1LlDHpJqpxBL0mVM+glqXIGvSRVzqCXpMoZ9JJUOYNekipn0EtS5Qx6SaqcQS9JlTPoJalyBr0kVa5t0EfE6ojYERH3R8R9EfG+0n5iRNweEQ+Vn8tLe0TERyNiOiLujYhXDPskJElz6+SK/mng/Zl5CnAmcHFEnAJsBrZn5lpge1kHeD2wtjw2AR8feNWSpI61DfrM3J+ZXy3L3wEeAFYB64FryrBrgHPL8nrgk9lyJ3BCRKwceOWSpI50dY8+ItYApwN3AaOZub90PQaMluVVwKONzfaWNknSIojM7GxgxAjweeCSzPxsRHw7M09o9D+Rmcsj4ibg0sz8YmnfDnwgM+8+an+baN3aYXR0dN3k5GRPJ3Dw0GEOPNnTpn07ddXxc/bNzMwwMjKygNV0bqnWZl3dsa7u1FjXxMTEzswcazduWSc7i4ifBD4DXJuZny3NByJiZWbuL7dmDpb2fcDqxuYnlbYfkZlbgC0AY2NjOT4+3kkpz3LFtVu5bFdHpzFwey4Yn7NvamqKXs9p2JZqbdbVHevqznO5rk7edRPAlcADmfm3ja5twMayvBHY2mh/a3n3zZnA4cYtHknSAuvkUvhVwFuAXRFxT2n7EHApcH1EXAR8Ezi/9N0CnANMA98D3jbQiiVJXWkb9OVee8zR/ZpZxidwcZ91SZIGxE/GSlLlDHpJqpxBL0mVM+glqXIGvSRVzqCXpMoZ9JJUOYNekipn0EtS5Qx6SaqcQS9JlTPoJalyBr0kVc6gl6TKGfSSVDmDXpIqZ9BLUuUMekmqnEEvSZUz6CWpcga9JFXOoJekyhn0klQ5g16SKmfQS1LlDHpJqpxBL0mVaxv0EXFVRByMiN2Nto9ExL6IuKc8zmn0fTAipiPiwYg4a1iFS5I608kV/dXA2bO0X56Zp5XHLQARcQqwAXhp2ebvI+KYQRUrSepe26DPzC8Ahzrc33pgMjOfysxvANPAGX3UJ0nqUz/36N8TEfeWWzvLS9sq4NHGmL2lTZK0SCIz2w+KWAPclJkvK+ujwONAAn8OrMzMt0fEx4A7M/NTZdyVwK2ZecMs+9wEbAIYHR1dNzk52dMJHDx0mANP9rRp305ddfycfTMzM4yMjCxgNZ1bqrVZV3esqzs11jUxMbEzM8fajVvWy84z88CR5Yj4R+CmsroPWN0YelJpm20fW4AtAGNjYzk+Pt5LKVxx7VYu29XTafRtzwXjc/ZNTU3R6zkN21Ktzbq6Y13deS7X1dOtm4hY2Vh9E3DkHTnbgA0RcWxEnAysBb7cX4mSpH60vRSOiOuAcWBFROwF/gQYj4jTaN262QO8EyAz74uI64H7gaeBizPzmeGULknqRNugz8w3z9J85TzjLwEu6acoSdLg+MlYSaqcQS9JlTPoJalyBr0kVc6gl6TKGfSSVDmDXpIqZ9BLUuUMekmqnEEvSZUz6CWpcga9JFXOoJekyhn0klQ5g16SKmfQS1LlDHpJqpxBL0mVM+glqXIGvSRVzqCXpMoZ9JJUOYNekipn0EtS5Qx6SaqcQS9JlTPoJalybYM+Iq6KiIMRsbvRdmJE3B4RD5Wfy0t7RMRHI2I6Iu6NiFcMs3hJUnudXNFfDZx9VNtmYHtmrgW2l3WA1wNry2MT8PHBlClJ6lXboM/MLwCHjmpeD1xTlq8Bzm20fzJb7gROiIiVgypWktS9Xu/Rj2bm/rL8GDBallcBjzbG7S1tkqRFEpnZflDEGuCmzHxZWf92Zp7Q6H8iM5dHxE3ApZn5xdK+HfhAZt49yz430bq9w+jo6LrJycmeTuDgocMceLKnTft26qrj5+ybmZlhZGRkAavp3FKtzbq6Y13dqbGuiYmJnZk51m7csp72DgciYmVm7i+3Zg6W9n3A6sa4k0rbs2TmFmALwNjYWI6Pj/dUyBXXbuWyXb2eRn/2XDA+Z9/U1BS9ntOwLdXarKs71tWd53Jdvd662QZsLMsbga2N9reWd9+cCRxu3OKRJC2CtpfCEXEdMA6siIi9wJ8AlwLXR8RFwDeB88vwW4BzgGnge8DbhlCzJKkLbYM+M988R9drZhmbwMX9FiVJGhw/GStJlTPoJalyBr0kVc6gl6TKGfSSVDmDXpIqZ9BLUuUMekmqnEEvSZUz6CWpcga9JFXOoJekyhn0klQ5g16SKmfQS1LlDHpJqpxBL0mVM+glqXIGvSRVzqCXpMoZ9JJUOYNekipn0EtS5Qx6SaqcQS9JlTPoJalyBr0kVW5ZPxtHxB7gO8AzwNOZORYRJwKfBtYAe4DzM/OJ/sqUJPVqEFf0E5l5WmaOlfXNwPbMXAtsL+uSpEUyjFs364FryvI1wLlDOIYkqUP9Bn0C/xkROyNiU2kbzcz9ZfkxYLTPY0iS+hCZ2fvGEasyc19E/BxwO/BeYFtmntAY80RmLp9l203AJoDR0dF1k5OTPdVw8NBhDjzZ06Z9O3XV8XP2zczMMDIysoDVdG6p1mZd3bGu7tRY18TExM7GbfM59fVibGbuKz8PRsSNwBnAgYhYmZn7I2IlcHCObbcAWwDGxsZyfHy8pxquuHYrl+3q6zR6tueC8Tn7pqam6PWchm2p1mZd3bGu7jyX6+r51k1EHBcRP31kGXgdsBvYBmwswzYCW/stUpLUu34uhUeBGyPiyH7+NTP/IyK+AlwfERcB3wTO779MSVKveg76zHwEePks7d8CXtNPUZKkwfGTsZJUOYNekipn0EtS5Qx6SaqcQS9JlVucTxpJP0Z27TvMhZtvXvDj7rn0DQt+TNXJK3pJqpxBL0mVM+glqXIGvSRVzqCXpMoZ9JJUOYNekipn0EtS5Qx6SaqcQS9JlTPoJalyBr0kVc6gl6TKGfSSVDmDXpIqZ9BLUuUMekmqnEEvSZUz6CWpcga9JFXOoJekyg0t6CPi7Ih4MCKmI2LzsI4jSZrfsmHsNCKOAf4OeC2wF/hKRGzLzPuHcTxJg7Nm8819bf/+U5/mwh73sefSN/R17F71e879uPrs44Z+jKEEPXAGMJ2ZjwBExCSwHjDopQ61C59+AlXPLcO6dbMKeLSxvre0SZIWWGTm4HcacR5wdma+o6y/BfjVzHxPY8wmYFNZ/SXgwR4PtwJ4vI9yh2Wp1gVLtzbr6o51dafGul6cmS9sN2hYt272Aasb6yeVth/KzC3Aln4PFBF3Z+ZYv/sZtKVaFyzd2qyrO9bVnedyXcO6dfMVYG1EnBwRzwM2ANuGdCxJ0jyGckWfmU9HxHuA24BjgKsy875hHEuSNL9h3bohM28BbhnW/hv6vv0zJEu1Lli6tVlXd6yrO8/ZuobyYqwkaenwKxAkqXI/FkEfEb8dEfdFxP9FxJyvTs/1tQvlReG7SvunywvEg6jrxIi4PSIeKj+XzzJmIiLuaTy+HxHnlr6rI+Ibjb7TFqquMu6ZxrG3NdoXc75Oi4gvlef73oj4nUbfQOer3dd0RMSx5fyny3ysafR9sLQ/GBFn9VNHD3X9fkTcX+Zne0S8uNE363O6QHVdGBH/0zj+Oxp9G8vz/lBEbFzgui5v1PT1iPh2o2+Y83VVRByMiN1z9EdEfLTUfW9EvKLRN9j5yswl/wB+hdZ77aeAsTnGHAM8DLwEeB7wNeCU0nc9sKEsfwJ494Dq+mtgc1neDPxVm/EnAoeAF5T1q4HzhjBfHdUFzMzRvmjzBfwisLYsvwjYD5ww6Pma7/elMeb3gE+U5Q3Ap8vyKWX8scDJZT/HLGBdE43foXcfqWu+53SB6roQ+Ngs254IPFJ+Li/LyxeqrqPGv5fWm0OGOl9l378OvALYPUf/OcCtQABnAncNa75+LK7oM/OBzGz3gaoffu1CZv4vMAmsj4gAXg3cUMZdA5w7oNLWl/11ut/zgFsz83sDOv5cuq3rhxZ7vjLz65n5UFn+b+Ag0PYDIT2Y9fdlnnpvAF5T5mc9MJmZT2XmN4Dpsr8FqSszdzR+h+6k9TmVYetkvuZyFnB7Zh7KzCeA24GzF6muNwPXDejY88rML9C6sJvLeuCT2XIncEJErGQI8/VjEfQdmutrF34W+HZmPn1U+yCMZub+svwYMNpm/Aae/Ut2Sfln2+URcewC1/X8iLg7Iu48cjuJJTRfEXEGrau0hxvNg5qvTr6m44djynwcpjU/w/yKj273fRGtq8IjZntOF7Ku3yrPzw0RceRDk0tivsotrpOBOxrNw5qvTsxV+8Dna2hvr+xWRHwO+PlZuj6cmVsXup4j5quruZKZGRFzvoWp/KU+ldZnC474IK3Aex6tt1h9APizBazrxZm5LyJeAtwREbtohVnPBjxf/wJszMz/K809z1eNIuJ3gTHgNxrNz3pOM/Ph2fcwcP8OXJeZT0XEO2n9a+jVC3TsTmwAbsjMZxptizlfC2bJBH1m/mafu5jraxe+ReufRMvKVdmzvo6h17oi4kBErMzM/SWYDs6zq/OBGzPzB419H7m6fSoi/hn4g4WsKzP3lZ+PRMQUcDrwGRZ5viLiZ4Cbaf2Rv7Ox757naxZtv6ajMWZvRCwDjqf1+9TJtsOsi4j4TVp/PH8jM5860j7HczqI4Orka02+1Vj9J1qvyRzZdvyobacGUFNHdTVsAC5uNgxxvjoxV+0Dn6+abt3M+rUL2Xp1Ywet++MAG4FB/QthW9lfJ/t91r3BEnZH7oufC8z66vww6oqI5UdufUTECuBVwP2LPV/lubuR1r3LG47qG+R8dfI1Hc16zwPuKPOzDdgQrXflnAysBb7cRy1d1RURpwP/ALwxMw822md9ThewrpWN1TcCD5Tl24DXlfqWA6/jR/9lO9S6Sm2/TOuFzS812oY5X53YBry1vPvmTOBwuZgZ/HwN+pXmYTyAN9G6T/UUcAC4rbS/CLilMe4c4Ou0/iJ/uNH+Elr/IU4D/wYcO6C6fhbYDjwEfA44sbSPAf/UGLeG1l/pnzhq+zuAXbQC61PAyELVBfxaOfbXys+LlsJ8Ab8L/AC4p/E4bRjzNdvvC61bQW8sy88v5z9d5uMljW0/XLZ7EHj9gH/f29X1ufLfwZH52dbuOV2guv4SuK8cfwfwy41t317mcRp420LWVdY/Alx61HbDnq/raL1r7Ae08usi4F3Au0p/0PofND1cjj/W2Hag8+UnYyWpcjXdupEkzcKgl6TKGfSSVDmDXpIqZ9BLUuUMekmqnEEvSZUz6CWpcv8Pa9JclMO4CpYAAAAASUVORK5CYII=\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "recalls.hist()" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [], "source": [ "recalls_df = pd.DataFrame([df.iloc[recalls.index][\"split\"],recalls]).T\n", "recalls_df.columns = [\"split\",\"recall\"]" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqQAAAH9CAYAAADML24LAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XtwXOV9//HP3rWSvdZirw02xrgYhC/4gmwwJKAIMKHGUFwolyTlEmhjppO0hEzzR6YpbX6TNEATaCYdppNLE2hCQuypKXLogFFkro6RKxvGkBobGxsZrMiSN7qu9vL7w7Vg8YU92j3nOfvo/foHtLva8/34PHr01TlnzxMoFAoFAQAAAIYETRcAAACA8Y2GFAAAAEbRkAIAAMAoGlIAAAAYRUMKAAAAo2hIAQAAYBQNKYz7/Oc/r6lTp2rBggXHfb5QKOhLX/qS5syZo4ULF2rr1q0eVwgA/rJv3z41Nzdr3rx5mj9/vh5++OFjXsPciWpCQwrjbr/9dj399NMnfP7Xv/61du7cqZ07d+rf/u3fdPfdd3tYHQD4Tzgc1j//8z9rx44deuWVV/T9739fO3bsKHoNcyeqCQ0pjLv00kt1yimnnPD59evX69Zbb1UgENDy5cvV29urAwcOeFghAPjLaaedpvPPP1+SNHHiRM2dO1fvvvtu0WuYO1FNaEjhe++++65mzpw5+vXpp59+zMQLAOPVnj179D//8z+68MILix5n7kQ1oSEFAKBK9fX16frrr9dDDz2kRCJhuhxgzGhI4XszZszQvn37Rr/ev3+/ZsyYYbAiADBvZGRE119/vT772c/qT//0T495nrkT1YSGFL537bXX6qc//akKhYJeeeUVTZo0SaeddprpsgDAmEKhoDvvvFNz587Vl7/85eO+hrkT1SRsugDglltu0W9+8xv9/ve/1+mnn65/+Id/0MjIiCRpzZo1WrlypTZs2KA5c+aotrZWP/7xjw1XDABmvfjii3r00Ud13nnnafHixZKkb37zm3rnnXckMXei+gQKhULBdBEAAAAYvzhlDwAAAKNoSAEAAGAUDSkAAACMoiEFAACAUTSkAAAAMMrz2z51dnY6en0kElEqlVJXV9forYDcEIvFNDw87Nr7S/Zk8SqHRBYnbBlf0tiyTJ8+3dWaTHM6d0r2jDnJnixe/fxMnDhRf/jDH6o6i23jy6/7xC9zJ0dI/08waM8/BVn8yZYstuQYD2zaV7Zk8SpHPp93fRu27BPJmyzsk5Or3soBAABgBRpSAAAAGFX2NaQ9PT1qaWnR/v37FQqFNG/ePF111VUKhUKVqA8ArPTjH/9Y+/fvHz3Flkgk9MUvftFwVQBgRtkNaUtLi+rq6nTvvfdqaGhIjz76qLZs2aLly5dXoj4AsNbKlSvV2NhougwAMK7sU/a9vb2aP3/+6CfI5syZo66urkrUBgAAgHGg7COky5cv1+uvv64zzzxTQ0ND2rlzpy677DJJUjqdVl9fX9HrM5mM6urqSi8wHC76r1tCoZAikYir27Ali1c5JLI4Ycv4krzdLyZt3LhRzz77rKZMmaLLLrtMs2fPllSZuVOyZ8xJ9mTx6ucnEAhUfRbbxpcN+8RNgUKhUCjnDbq6urRu3Tq99957KhQKWrRoka677joFAgG1traqra2t6PVNTU1qbm52tI2lS5eWU2JJXn31Vde3AQBH7d+/X6lUSqFQSK+//ro2bNigNWvW6JRTTqnY3Inxi9+bqDZlNaT5fF4PP/ywGhsbdfHFFyuTyWj9+vWaPHmyrrzyyoodIV2xYsVYSyzZc8895/rNisPhsJLJpHp6epTNZl3bjts3+PUqh0QWJ2wZX9LYsqRSKVdrctujjz6qc845RxdeeGHFjpDaMuYke7J49fNjw+9N28ZXTU2NhoaGfDe+/DJ3lnXseHBwUIcPH9YFF1ygcDiscDisxYsX67nnntOVV16pRCKhRCJR9D2dnZ2ur7gwFrlczrO6stmsq9sKh8OeZHE7h0SWsbBlfEne7Be/CAQCOnp8oFJzpy1jTrIni5c/P27z6vemLeOrUCgwvk6irA811dXVqb6+Xlu2bFEul9Pg4KC2bdumadOmVao+ALDO4OCg3nrrLY2MjCiXy2n79u3au3ev5syZY7o0ADCi7Ktrb7rpJj399NN68cUXFQgENHv2bF111VWVqA0ArJTP5/Xcc8/p97//vQKBgKZMmaKbb75ZU6ZMMV0aABhRdkN62mmn6Y477qhELQAwLtTV1ekv//IvTZcBAL7B0qEAAAAwioYUAAAARtGQAgAAwCgaUgAAABhFQwoAAACjaEgBAABgFA0pAAAAjKIhBQAAgFE0pAAAADCKhhQAAABG0ZACAADAqLLXsgcAuC8WiykYdHYMIRgMKh6Pu1SRFAgENDAwoEgkonDY3V8ntmRxO4d0JIsXbNknkjdZcrmcFePLLTSkAFAFhoeHHX9PPB7X4OCgC9UcEYlEVF9fr/7+fo2MjLi2HcmeLG7nkI5k8UI+n7din0jejK9oNKqhoSHfja9kMulSNc5wyh4AAABG0ZACAADAKM9P2Tu9DsqWa2Eke65Rsu26HrI4Y9PPCgDAHzyf6Z1eB2XLtTCSPdco2XZdD1mc8eoaOKdZ/HIdFADAOU7ZAwAAwCgaUgAAABhFQwoAAACjaEgBAABgFA0pAAAAjKIhBQAAgFE0pAAAADCKhhQADOru7tY3vvENrV271nQpAGAMDSkAGNTS0qIZM2aYLgMAjGJNPgAw5LXXXlNNTY1SqZQOHTo0+ng6nVZfX1/RazOZjOrq6hy9fygUcnW1u6PLunqxvKstWdzOIXmzPyR79onkTZZAIGDF+HILDSkAGDA0NKTW1lbddttt2rp1a9Fz7e3tamtrK3qsqalJzc3NXpZYMpuWbbUpi9sSiYQn27Fpn8TjcdMl+BYNKQAY0NraqvPPP1+TJk065rnGxkY1NDQUPZbJZNTV1eVoG7FYTMPDw2XVeTLhcFjJZFI9PT3KZrOubUeyJ4vbOSTvjpCm02kr9onkzfiqqanR0NCQ78ZXKpVyqRpnaEgBwGMHDhzQ7t279YUvfOG4zycSiWOOPnV2dmpkZMTRdsLhsOPvGYtsNuv6dmzJ4lUOL+RyOSv2ieTNfikUCoyvk6AhBQCP7dmzR729vfrud78r6cjRz0KhoEceeURr1qwxXB0AeI+GFAA81tjYqAULFox+/dJLL6m3t1erVq0yWBUAmFOxhvS1115TW1ubDh8+rAkTJui6667TrFmzKvX2AGCNaDSqaDRa9HU4HHb8KXoAsEVFGtJdu3bp2Wef1Q033KAZM2Ycc7sSAMCJ+fXT8wDglYo0pK2trWpqatLMmTMlfXAriErcS8+W+6lJ9tznzrZ7w334v26yJYtNPysAAH8oe7bP5/Pq7OxUQ0ODHn74YWWzWZ177rm68sorq+peel7dT02y555qtuSQyOJXNmUBAJxY2Q1pX1+f8vm8duzYoc9//vMKBoN6/PHHtWnTJi1btqzse+nZcj81yZ773Nl2bziyOOPVfRSdZvHLvfQAAM6V3e0dPXV34YUXauLEiZKkiy66SJs2bdLll19ekXvpecGr+6lJ9tznzpZ7w0lkccLL+9x5sV8AAOYFy32DeDzu6eluAAAA2KXshlSSFi9erN/+9rfq6+vT4OCgXnnlFZ1zzjmVeGsAAABYriIXaDY1NWlgYEDf+973FA6HNX/+fF1yySWVeGsAAABYriINaSgU0qpVq1hlBAAAAI5V5JQ9AAAAMFY0pAAAADCKZVAAoArEYjEFg86OIQSDQcXjcZcqkgKBgAYGBhSJRFy/Z7QtWdzOIR3J4gVb9onkTZZcLmfF+HILDSkAVIGxLEYQj8c1ODjoQjVHRCIR1dfXq7+/3/X7xdqSxe0cklxf2veofD5vxT6RvBlf0WhUQ0NDvhtfflkRj1P2AAAAMIqGFAAAAEbRkAIAAMAoGlIAAAAYRUMKAAAAo2hIAQAAYBQNKQAAAIyiIQUAAIBRNKQAAAAwioYUAAAARnm+dKjT9ZhtWZNXsmetZNvWFyaLMzb9rAAA/MHzmd7pesy2rMkr2bNWsm3rC5PFGa/W4naaxS/rMZdq7dq1evvtt5XJZDRhwgR94hOfUGNjo+myAMAIDj0AgAGXXHKJ/uRP/kThcFhdXV3693//d5122mmaPn266dIAwHM0pABgwNSpU0f/PxAIKBAI6NChQ5o+fbrS6bT6+vqKXp/JZFRXV+doG6FQyNWzTEcvp/DisgpbsridQ/Jmf0j27BPJmyyBQMCK8eUWGlIAMOSpp55SR0eHstmsTj31VJ199tmSpPb2drW1tRW9tqmpSc3NzSbK/FjVdrnEydiUxW2JRMKT7di0T9y+/r6a0ZACgCGrVq3SypUrtW/fPu3Zs2f06EljY6MaGhqKXpvJZNTV1eXo/WOxmOPr9p0Ih8NKJpPq6elRNpt1bTuSPVncziF5d4Q0nU5bsU8kb8ZXTU2NhoaGfDe+UqmUS9U4Q0MKAAYFg0HNmjVL27dv15YtW7R8+XIlEoljjj51dnY6/rBaOBx2/cN6kpTNZl3fji1ZvMrhhVwuZ8U+kbzZL4VCgfF1EtyHFAB8IJ/Pq6enx3QZAGAEDSkAeKyvr0+vvfaahoeHlc/n9dZbb+n111/X7NmzTZcGAEZwyh4APBYIBPTqq6/qqaeeUqFQUH19va666iqde+65pksDACNoSAHAY3V1dbrjjjtMlwEAvsEpewAAABhFQwoAAACjaEgBAABgFA0pAAAAjKIhBQAAgFE0pAAAADCqYg1pd3e3vvGNb2jt2rWVeksAAACMAxVrSFtaWjRjxoxKvR0AAADGiYo0pK+99ppqampY9g4AAACOlb1S09DQkFpbW3Xbbbdp69atRc+l02n19fUVPZbJZFRXV1d6gWFvFpMKhUKKRCKubuNoFrczuZ3FqxwSWZywZXxJ3u4XAIB5Zc/2ra2tOv/88zVp0qRjnmtvb1dbW1vRY01NTWpubi53sxWXSCQ821YymfRsW26yJYdEFr+yKUu5YrGYgkFnJ7WCwaDi8bhLFUmBQEADAwOKRCKu//FgSxa3c0hHsnjBln0ieZMll8tZMb7cUta/yoEDB7R792594QtfOO7zjY2NamhoKHosk8moq6ur9AI9OkKSTqc1PDzs6jbC4bCSyaR6enqUzWZd204sFnM1i1c5JLI4Ycv4ksaWJZVKuVqTaWP5N4/H4xocHHShmiMikYjq6+vV39+vkZER17Yj2ZPF7RySXD+DcVQ+n7din0jejK9oNKqhoSHfjS+//OFfVre3Z88e9fb26rvf/a6kI81moVDQI488ojVr1iiRSBxz5LGzs9P1gTUWuVzOs7qy2ayr2wqHw55kcTuHRJaxsGV8Sd7sFwCAeWU1pI2NjVqwYMHo1y+99JJ6e3u1atWqsgsDAADA+FBWQxqNRhWNRou+DofDjj60BAAAgPGtohdo+vHDSgAAAPA3lg4FAACAUTSkAAAAMIqGFAAAAEbRkAIAAMAoGlIAAAAYRUMKAAAAo2hIAQAAYJQ3C8UDAEZls1m1tLRo9+7dGhwcVDKZ1BVXXKGzzz7bdGkAYAQNKQB4LJ/PK5FI6Pbbb9ekSZO0c+dOPfHEE7r77ruVTCZNlwcAnqMhBQCPRaPRopXtGhoaVF9frwMHDiiZTCqdTquvr6/oezKZjONlmUOhkCKRSEVqPp5wOFz0XzfZksXtHJI3+0OyZ59I3mQJBAJWjC+30JACgGF9fX3q7u5WKpWSJLW3t6utra3oNU1NTb5dntmmo7o2ZXFbIpHwZDs27ZN4PG66BN+iIQUAg3K5nNauXavFixePNqSNjY1qaGgoel0mk1FXV5ej947FYhoeHq5YrR8VDoeVTCbV09OjbDbr2nYke7K4nUPy7ghpOp22Yp9I3oyvmpoaDQ0N+W58HZ13TKMhBQBD8vm81q1bp1AopJUrV44+nkgkjjn61NnZqZGREUfvHw6HHX/PWGSzWde3Y0sWr3J4IZfLWbFPJG/2S6FQYHydBLd9AgADCoWCnnzySfX39+umm25SKBQyXRIAGENDCgAGPPXUU+rq6tItt9xStR9CAIBK8fyUfSwWUzBYeh8cCARcrOYDwWDQ9YuNA4GABgYGFIlEXL3Gx+0sXuWQyOKELeNL8na/mNDb26v29naFQiE9+OCDo49fc801WrhwocHKAMAMz2d6pxfbenXkIJ/Pa3Bw0NVtRCIR1dfXq7+/39VrPOLxuKtZvMohkcUJW8aXNLYs1fRJ3Pr6et13332mywAA3+CUPQAAAIyiIQUAAIBRNKQAAAAwioYUAAAARtGQAgAAwCgaUgAAABhFQwoAAACjaEgBAABgFA0pAAAAjKIhBQAAgFH2LRINABaKxWIKBp0dQwgGg4rH4y5VJAUCAQ0MDCgSiSgcdvfXiS1Z3M4hHcniBVv2ieRNllwuZ8X4cgsNKQBUgeHhYcffE4/HNTg46EI1R0QiEdXX16u/v18jIyOubUeyJ4vbOaQjWbyQz+et2CeSN+MrGo1qaGjId+MrmUy6VI0znLIHAACAUTSkAAAAMIqGFAAAAEaVfQ1pNptVS0uLdu/ercHBQSWTSV1xxRU6++yzK1EfAAAALFd2Q5rP55VIJHT77bdr0qRJ2rlzp5544gndfffdvrlQFgAAAP5VdkMajUbV3Nw8+nVDQ4Pq6+t14MABhUIh9fX1Fb0+k8morq6u9AJdvtXDUaFQyPVPJh7N4nYmt7N4lUMiixO2jC/J2/0CADCv4rN9X1+furu7lUql1N7erra2tqLnm5qaihpYv0gkEp5ty5Yjx7bkkMjixNKlS119/6NeffVVq/YLAODEKtqQ5nI5rV27VosXL1YqlVJjY6MaGhqKXpPJZNTV1VV6gR4dIUmn02O6z58T4XBYyWRSPT09ymazrm0nFou5msWrHBJZnPAyi1ecZEmlUi5XAwBwS8W6vXw+r3Xr1ikUCmnlypWSjhx1/OiRx87OTtdvcDsWuVzOs7qy2ayr2wqHw55kcTuHRJax8CKLV2zKAgA4sYrc9qlQKOjJJ59Uf3+/brrpJoVCoUq8LQAAAMaBijSkTz31lLq6unTLLbd4tmQZAAAA7FD2Kfve3l61t7crFArpwQcfHH38mmuu0cKFC8t9ewCw0ubNm9XR0aGDBw9qwYIFWr16temSAMCYshvS+vp63XfffRUoBQDGj4kTJ+rSSy/Vrl27uE4WwLjHTf4AwIB58+ZJ8u8HPQHASzSkAOAz6XS67EVFJHsWY5DsyeLlwhJus2WfSN5kCQQCVowvt9CQAoDPVNOiIhILS4xXXi0oY9M+icfjpkvwLRpSAPCZSiwqItm1GIMtWdzOIdmzoIxt46umpkZDQ0O+G19+WVSEhhQAfKZSi4rYtBiDLVm8yuEFrxaUsWV8FQoFxtdJ0JACgAG5XE75fF6FQkGFQkEjIyMKBoMsLAJgXKIhBQADNm3aVHSd6Pbt2319nSgAuImGFAAMaG5upvkEgP9TkaVDAQAAgLGiIQUAAIBRNKQAAAAwioYUAAAARtGQAgAAwCgaUgAAABjl+W2fYrGYgsHS++BAIOBiNR8IBoOurzEbCAS0dOlSV7chSRs3bnQ1SyAQ0MDAgCKRiOtL1K1YscLV9/eSTfvFCzZlAQCcnOczvdM1ViORiEuVFMvn8xocHHR1G7ZkiUQiqq+vV39/f9UuUWYC+8WZ2tpaR1mSyaTLFQEA3MIpewAAABhFQwoAAACjuDgLAKqA0+vvJfevjffyumVbsnj1eQUv2LJPJG+y5HI5K8aXW2hIAaAKOL3+XpLi8bg11y3bksXtHBKfVxgLL8ZXNBrV0NCQ78aXX66/55Q9AAAAjKIhBQAAgFE0pAAAADCKhhQAAABG0ZACAADAKBpSAAAAGEVDCgAAAKNoSAEAAGAUDSkAAACMoiEFAACAUTSkAAAAMKoia9kPDAzoySef1K5du1RbW6vLL79cCxcurMRbA4CVmDcB4AMVaUg3bNigUCikr3zlK3rvvff0s5/9TKeeeqqmTp1aibcHAOswbwLAB8o+ZZ/JZLRjxw41NzcrFotp1qxZamho0LZt2ypRHwBYh3kTAIqVfYS0u7tbwWBQU6ZMGX1s2rRp2rt3r9LptPr6+open8lkVFdXV3qB4YocxP1YoVBIkUjE1W3YkuVoDq/y2IL94pxNWT7sZPOmpIrMnZJdY86WLPyuKZ1t4ysQCFgxvtxS9r9MJpNRLBYreqympkbDw8Nqb29XW1tb0XNNTU1qbm4u+f3T6bQeeOABNTY2KpFIlFuuUbZkSafTam1tVWNjo5LJpKvbevXVV119/3Q6rfb29qrfJ5J3+8XtfSJ5O8ZMONm8Kakic6cXbNpPtmXhd42/pNNpvfLKK1ZkcUvZDWk0Gh2dRI8aHh5WLBZTY2OjGhoaip6bMGGCo/fv6+tTW1ubGhoaqvoHS7Iniy05JLL4lU1Zjudk86akisydXrBpP5HFf2zJIdmVxS1lN6STJ09WPp9Xd3e3Jk+eLEl67733lEqllEgk+IcHgI842bwpibkTwLhT9oeaotGo5s6dq9bWVmUyGb3zzjv63e9+p0WLFlWiPgCwDvMmABSryNW1V199tdavX68HHnhA8XhcV199NbcuAYCTYN4EgA+E7rvvvvvKfZNIJKLzzjtPl156qS666CJNmzatAqUdUSgUFI1GdeaZZx7zIYBqY0sWW3JIZPErm7KciJvzplds2k9k8R9bckh2ZXFLoFAoFEwXAQAAgPHLFzf5K3UJvUKhoGeffVZbt26VJJ1//vm64oorFAgEvC75hErN8vbbb6utrU0HDhxQTU2N7rnnHgPVnlypWV588UV1dHTo8OHDqq2t1bJly/SJT3zCQMUnVmqWl19+WZs3b9bAwICi0agWLFigFStWKBQKGaj6WE6Xm8xms3rkkUc0PDyse++918NKP16pWVpbW/X8888X7YO7775bp5xyipfl4v+MZclTv45DJ1n8Ns+Nx9+bftsHH2XT/GyCLxrSUpfQa29v15tvvqk1a9YoEAjopz/9qerr67Vs2TJDlR+r1CyRSERLlizRggUL9Pzzzxuq9uRKzVIoFLR69WpNmzZNPT09evTRR5VIJHTeeecZqvxYpWZpaGjQ4sWLFY/HNTAwoF/+8pfavHmzLr74YkOVF3O63ORLL72k2traY24x5AdOssyfP1/XX3+9gSrxUWNZ8tSv49BJFr/Nc+Px96bf9sFH2TQ/m1D2p+zL5WQJvY6ODl100UWaNGmSEomELr74YnV0dBio+vicZDn99NO1aNEi394g10mWT37yk5o+fbpCoZCmTJmihoYG7du3z0DVx+ckyymnnKJ4PD76dSAQ0KFDh7ws94ScLjfZ09Oj7du365JLLvG40o/H0pnVaSz7za/j0GkWP81z4/X3pp/2wUfZND+bYrwhPdESel1dXce8tqurS6eeeurHvs4UJ1n8bqxZCoWC3nnnndH7KfqB0yzbt2/XN7/5Td1///16//33tXTpUq9KPSmnOTZs2KDLL7/cl8tvOs3yv//7v/qnf/onff/739eWLVu8KhMfMZZ5wa/jsJz52vQ8x+9N8/vgo2yan00x/i/xcUvoney1NTU1ymQyKhQKvrgexkkWvxtrlt/85jcqFApasmSJm+U54jTLwoULtXDhQnV3d2vbtm2O1w93i5Mcb7zxhgqFgubOnau3337bqxJL5iTL/Pnz1djYqAkTJmj//v365S9/qZqaGt+cphtPnP4s+XkcljNfm57n+L1pfh98lE3zsynGG9KPW0LvZK8dHh5WNBr1xQ+V5CyL340ly+bNm7Vt2zbdcccdvvqrb6z7ZfLkyUqlUmppadHNN9/sZoklKTVHJpPRM888o89+9rNelueIk33y4euvzjjjDF144YXasWMHDakLfvzjH2vv3r3HfW7mzJlauXJlyfvN9DisZJYP88M8N95/b/phH3yUTfOzKcb35MctofdhqVRK77//vk4//fSTvs4UJ1n8zmmWrVu36oUXXtAdd9yhSZMmeVnqxypnv+TzefX09LhdYklKzdHd3a3e3l796Ec/kiTlcjkNDw/rgQce0F133eWL65bL2SeBQEDcrc4dd9xxx0mfz2QyJe830+OwklmO8ss8N55/b/plH3yUTfOzKcavIXWyhN6iRYv08ssvK51OK51O6+WXX9bixYsNVH18TrLk83mNjIwon89LkkZGRpTNZr0u+YScZNm+fbs2btyoW2+91Ze34nGSpb29XX19fZKkgwcP6oUXXtDs2bO9Lvm4Ss0xdepU3XPPPVqzZo3WrFmja6+9VnV1dVqzZo1vJnAn++TNN9/U4OCgCoWC9u/fr82bN+vcc881UDWc7De/j0Ony7f6aZ4br783/bQPPsqm+dkUX9wYf2BgQOvXr9fu3bsVj8d1xRVXaOHChdq7d68ee+wxfe1rX5N05CLmZ555puh+aitWrPDNqQep9Cxvv/22fvKTnxR976xZsz72r3ovlZrloYceUjqdLrpP5MKFC3XNNdeYKv0YpWb5z//8T+3cuVOZTEa1tbWaP3++mpubFYlEDCc4otQcH/b2229r3bp1vrvPXalZfvWrX2nXrl3KZrNKJBJatmyZli9fbrj68etE+01S1Y1DJ1n8Ns+Nx9+bftsHH2XT/GyCLxpSAAAAjF/GT9kDAABgfKMhBQAAgFE0pAAAADCKhhQAAABG0ZACAADAKBpSAAAAGEVDCgAAAKNoSAEAAGAUDSkAAACMoiEFAACAUTSkAAAAMIqGFAAAAEbRkAIAAMAoGlIAAAAYRUMKAAAAo2hIAQAAYBQNKQAAAIyiIQUAAIBRNKQAAAAwioYUAAAARtGQAgAAwCgaUgAAABhFQwoAAACjaEgBAABgFA0pAAAAjKIhBQAAgFE0pAAAADCKhhQAAABG0ZACAADAKBpSAAAAGEVDCgAAAKNoSAEAAGAUDSkAAACMoiEFAACAUTSkAAAAMIqGFAAAAEbRkAIAAMAoGlIAAAAYRUMKAAAAo2hIAQAAYBQNKQAAAIz/DhffAAAgAElEQVSiIQUAAIBRNKQAAAAwioYUAAAARtGQAgAAwCgaUgAAABhFQwoAAACjaEgBAABgFA0pAAAAjKIhBQAAgFE0pAAAADCKhhQAAABG0ZACAADAKBpSAAAAGEVDCgAAAKNoSAEAAGAUDSmMGxoa0gUXXKBFixZp/vz5+vu///tjXjM8PKybbrpJc+bM0YUXXqg9e/Z4XygA+Ewul9OSJUu0atWqY55j3kQ1oSGFcbFYTM8995y2bdumjo4OPf3003rllVeKXvPDH/5QyWRSb731lu655x599atfNVQtAPjHww8/rLlz5x73OeZNVBMaUhgXCAQ0YcIESdLIyIhGRkYUCASKXrN+/XrddtttkqQbbrhBGzduVKFQ8LxWAPCL/fv3q6WlRXfddddxn2feRDWhIYUv5HI5LV68WFOnTtWKFSt04YUXFj3/7rvvaubMmZKkcDisSZMmqbu720SpAOALf/M3f6P7779fweDxf5Uzb6Ka0JDCF0KhkDo6OrR//3799re/1euvv266JADwraeeekpTp05VY2Oj6VKAiqAhha/U19erublZTz/9dNHjM2bM0L59+yRJ2WxWhw8f1uTJk02UCADGvfjii3ryySd15pln6uabb9Zzzz2nz33uc0WvYd5ENaEhhXFdXV3q7e2VJA0ODuqZZ57RueeeW/Saa6+9Vj/5yU8kSb/61a902WWXHXOdKQCMF9/61re0f/9+7dmzR48//rguu+wyPfbYY0WvYd5ENQmbLgA4cOCAbrvtNuVyOeXzed14441atWqVvv71r2vp0qW69tprdeedd+rP//zPNWfOHJ1yyil6/PHHTZcNAL7DvIlqFSjwkTsAAAAYxCl7AAAAGEVDCgAAAKNoSAEAAGAUDSkAAACMoiEFAACAUZ7f9qmzs9PR6yORiFKplLq6ujQyMuJSVVIsFtPw8LBr7y/Zk8WrHBJZnLBlfEljyzJ9+nRXazKNubP6szDfOMf4Kt1Yc/hl7uQI6f850VrA1Ygs/mRLFltyoDJsGg9k8R9bckh2ZXED/zoAAAAwioYUAAAARrF0KAC4YPPmzero6NDBgwe1YMECrV69evS53bt3q6WlRYcPH9bpp5+u6667TvX19QarBQCzOEIKAC6YOHGiLr30Ui1ZsqTo8f7+fv3iF7/QZZddpq9+9auaPn26nnjiCUNVAoA/0JACgAvmzZunuXPnKh6PFz3+xhtvKJVKaf78+YpEIvrUpz6l999/X11dXYYqBQDzOGUPAB7q6urSqaeeOvp1NBpVMplUV1eXUqmUJCmdTquvr6/o+zKZjOrq6kreTjgcLvqvW0KhkCKRiKvbsCWLVzkke7Iwvkrn5fhyg+dVx2IxR7c+CAQCWrp0qYsVHbFx48ZjjmRUWiAQ0MDAgCKRiKsDJhgMuprFqxwSWZywZXxJ3u4Xr2UyGdXW1hY9VlNTU3R/wvb2drW1tRW9pqmpSc3NzY625cXc+eqrr7q+jaOSyaRn23KTLTkksvhRtebwfKZ3elNYt/8yOiqfz2twcNDVbUQiEdXX16u/v9/Vm+/G43FXs3iVQyKLE7aML2lsWaplEo5Go8fMg8PDw4rFYqNfNzY2qqGhoeg1mUzG0Wl9rxr5dDrt+o3Lw+Gwksmkenp6lM1mXduO2zcu9yqHZE8WL26Mb0uWseY4embGNLsOPQCAz6VSKW3btm3060wmo0OHDhX9UkgkEkokEkXf19nZ6fofTWORy+U8qyubzbq6rXA47EkWt3NI9mTxKodkTxYvxpcb+FATALjgaKNWKBRUKBQ0MjKiXC6nuXPn6uDBg9qxY4dGRkbU1tamadOm+eYoBQCYwBFSAHDBpk2biq4D3b59++h1oDfeeKM2bNigdevWacaMGbrhhhsMVgoA5tGQAoALmpubT/ghpLPOOktf/OIXPa4IAPyLU/YAAAAwioYUAAAARtGQAgAAwCgaUgAAABhFQwoAAACjSvqU/ebNm9XR0aGDBw9qwYIFWr169ehzu3fvVktLiw4fPqzTTz9d1113nerr610rGAAAAHYp6QjpxIkTdemll2rJkiVFj/f39+sXv/iFLrvsMn31q1/V9OnT9cQTT7hSKAAAAOxUUkM6b948zZ07V/F4vOjxN954Q6lUSvPnz1ckEtGnPvUpvf/++47WWwYAAMD4VtaN8bu6unTqqaeOfh2NRpVMJtXV1aVUKqV0Oq2+vr6i78lkMqqrqyu9wLA39+4PhUKKRCKubuNoFrczuZ3FqxwSWZywZXxJ3u4XAIB5Zc32mUxGtbW1RY/V1NRoeHhYktTe3l60dJ6k0aXz/CaRSHi2rWQy6dm23GRLDoksfmVTFgDAiZXVkEaj0dHm86jh4WHFYjFJUmNjoxoaGoqez2Qyjk7pe3WEJJ1OH5Ol0sLhsJLJpHp6epTNZl3bTiwWczWLVzkksjhhy/iSxpYllUq5WpNpsVhMwWDpN0YJBAIuVvOBYDB4zOVclRYIBDQwMKBIJOLq7wS3s3iVQ7InC+OrdF6OLzeUVXEqldK2bdtGv85kMjp06NDoL4ZEInHMkcfOzk6NjIyUs1lX5HI5z+rKZrOubiscDnuSxe0cElnGwpbxJXmzX6qF0z8C3L6s4qh8Pq/BwUFXtxGJRFRfX6/+/n5Xx0M8Hnc1i1c5JHuyuJ1DsifLWHP45UxUSX9uH23WCoWCCoWCRkZGlMvlNHfuXB08eFA7duzQyMiI2traNG3aNOuPVAAAAKBySjpCumnTpqJrQbdv3z56LeiNN96oDRs2aN26dZoxY4ZuuOEG14oFAACAfUpqSJubm0/4QaSzzjpLX/ziFytaFAAAAMYPlg4FAACAUTSkAAAAMIqGFAAAAEbRkAIAAMAoGlIAAAAYRUMKAAAAo2hIAQAAYBQNKQAAAIyiIQUAAIBRJa3UVEmxWEzBYOl9cCAQcLGaDwSDQcXjcVe3EQgENDAwoEgkonDYvX96t7N4lUMiixO2jC/J2/1iSk9Pj1paWrR//36FQiHNmzdPV111lUKhkOnSAMBzns/0w8PDjl4fiURcqqRYPp/X4OCgq9uIRCKqr69Xf3+/RkZGXNtOPB53NYtXOSSyOGHL+JLGliWZTLpaU6W1tLSorq5O9957r4aGhvToo49qy5YtWr58uenSAMBzdh56AACf6+3t1QUXXKBIJKJIJKI5c+aoq6tLkpROp9XX11f0+kwmo7q6upLf36sjy6FQyPUDB0ezuJ3J7Sxe5ZDsycL4Kp2X48sN1Vk1AFS55cuX6/XXX9eZZ56poaEh7dy5U5dddpkkqb29XW1tbUWvb2pqUnNzs4lSTyqRSHi2rWo7Cn4ituSQyOJH1ZqDhhQADJg1a5ba29v1rW99S4VCQYsWLdK5554rSWpsbFRDQ0PR6zOZzOgR1FJ4dZQknU47vhTLqXA4rGQyqZ6eHmWzWde2E4vFXM3iVQ7Jnixu55DsyTLWHKlUyrWanKAhBQCP5fN5PfbYY2psbNSdd96pTCaj9evX65lnntGVV16pRCJxzJHHzs5O169zHotcLudZXdls1tVthcNhT7K4nUOyJ4tXOSR7sngxvtzAbZ8AwGODg4M6fPiwLrjgAoXDYdXW1mrx4sXauXOn6dIAwAgaUgDwWF1dnerr67VlyxblcjkNDg5q27ZtmjZtmunSAMAITtkDgAE33XSTnn76ab344osKBAKaPXu2rrrqKtNlAYARNKQAYMBpp52mO+64w3QZAOALnLIHAACAUTSkAAAAMKrsU/asxwwAAIBylH2E9MPrMa9Zs0Z79+7Vli1bKlEbAAAAxoGyj5CyHnPpWC/XObKUzpbxJVX/mswAAGfKnu1Zj9m5al1n9qNsySGRxa9sygIAOLGyG1LWYy7deF8vdyzIUjpbxpc0tix+WY8ZAOBcWd0e6zGPDevllo4sztkyvqTqXZMZAOBMWR9qYj1mAAAAlKushpT1mAEAAFCusi/QZD1mAHBfLBZTMFj6MYRAIOBiNR8IBoOKx+OubiMQCGhgYECRSMTVzxW4ncWrHJI9WRhfpfNyfLmh7IpZjxkA3Of0g2Ru35rrqHw+r8HBQVe3EYlEVF9fr/7+flevKY7H465m8SqHZE8Wt3NI9mQZaw6/3M2EpUMBAABgFA0pAAAAjKIhBQAAgFE0pAAAADCKhhQAAABG0ZACAADAKBpSAAAAGEVDCgAAAKM8v5U/q42wGoQTZCmdLeNLqv4VRwAAzng+07PayPhdDWIsyFI6W8aXNLYsflltBADgHKfsAQAAYBTnwgDAkNdee01tbW06fPiwJkyYoOuuu06zZs0yXRYAeI6GFAAM2LVrl5599lndcMMNmjFjhvr6+kyXBADG0JACgAGtra1qamrSzJkzJUmJRGL0uXQ6fUyDmslkVFdXV/L7e/VhsFAo5Pq1/kezuJ3J7Sxe5ZDsycL4Kp2X48sN1Vk1AFSxfD6vzs5ONTQ06OGHH1Y2m9W5556rK6+8UpFIRO3t7Wprayv6nqamJjU3Nxuq+MQ+3Ei7zZYPrtmSQyKLH1VrDhpSAPBYX1+f8vm8duzYoc9//vMKBoN6/PHHtWnTJl1++eVqbGxUQ0ND0fdkMhl1dXWVvA2vjpKk02nHd09xKhwOK5lMqqenR9ls1rXtxGIxV7N4lUOyJ4vbOSR7sow1RyqVcq0mJ2hIAcBjR0/bXXjhhZo4caIk6aKLLhptSBOJxDFHHjs7O12/NdlY5HI5z+rKZrOubiscDnuSxe0ckj1ZvMoh2ZPFi/HlBm77BAAei8fjnp7qBgC/oyEFAAMWL16s3/72t+rr69Pg4KBeeeUVnXPOOabLAgAjOGUPAAY0NTVpYGBA3/ve9xQOhzV//nxdcsklpssCACMq1pByg2cAKF0oFNKqVau0atUq06UAgHEVaUi5wTMAAADGqiIN6Ylu8MzNnYtx813nyFI6W8aXVP03eAYAOFP2bH+yGzxzc+fjq9ab1n6ULTkksviVTVkAACdWdkN6shs8L1u2jJs7f8h4v/nuWJCldLaML2lsWfxyc2cAgHNld3ul3OD5w7i5MzffdYIsztkyvqTqvcEzAMCZsu9Dyg2eAQAAUI6K3BifGzwDAABgrCpygSY3eAYAAMBYVaQh5QbPAAAAGCvWsgcAAIBR3HUaAKpALBZTMFj6MYRAIOBiNR8IBoOKx+OubiMQCGhgYECRSMTVWwG6ncWrHJI9WRhfpfNyfLmh+ioGgHHI6b1f3V5N66h8Pq/BwUFXtxGJRFRfX6/+/n5XbwMWj8ddzeJVDsmeLG7nkOzJMtYcflmAhFP2AAAAMIqGFAAAAEbRkAIAAMAoGlIAAAAYRUMKAAAAozz/lD23LuHWEk6QpXS2jC+p+m9fAgBwxvOZnluXjN9bS4wFWUpny/iSxpbFL7cuAQA4xyl7AAAAGEVDCgAAAKNoSAEAAGAUDSkAAACMoiEFAIO6u7v1jW98Q2vXrjVdCgAYQ0MKAAa1tLRoxowZpssAAKO4wR8AGPLaa6+ppqZGqVRKhw4dGn08nU6rr6+v6LWZTEZ1dXUlv7dX928NhUKu357vaBa3M7mdxasckj1ZGF+l83J8uaE6qwaAKjc0NKTW1lbddttt2rp1a9Fz7e3tamtrK3qsqalJzc3NXpZYkkQi4dm2bLnXrC05JLL4UbXmoCEFAANaW1t1/vnna9KkScc819jYqIaGhqLHMpmMurq6Sn5/r46SpNNpxwueOBUOh5VMJtXT06NsNuvadmKxmKtZvMoh2ZPF7RySPVnGmiOVSrlWkxM0pADgsQMHDmj37t36whe+cNznE4nEMUceOzs7XV9NbCxyuZxndWWzWVe3FQ6HPcnidg7Jnixe5ZDsyeLF+HJDxRrS7u5u/eu//qvmzZun66+/vlJvCwDW2bNnj3p7e/Xd735X0pGjn4VCQY888ojWrFljuDoA8F7FGlI+KQoApWlsbNSCBQtGv37ppZfU29urVatWGawKAMypSEPKJ0VLwyf5nCNL6WwZX1L1f1r040SjUUWj0aKvw+Gwo7kRAGxS9mzPJ0Wdq9ZPwH2ULTkksviVTVlOxo9zIgB4qeyGlE+Klm68f5JvLMhSOlvGlzS2LH75pCgAwLmyuj0+KTo2fJKvdGRxzpbxJVXvp0UBAM6U1ZDySVEAAACUq6yGlE+KAgAAoFxlNaR8UhQAAADlqugnhvikKAAAAJwKmi4AAAAA4xsNKQAAAIyiIQUAAIBRNKQAAAAwys6FogHAMrFYTMFg6ccQAoGAi9V8IBgMKh6Pu7qNQCCggYEBRSIRV1fvczuLVzkke7LYNL5WrFjh2nsftWnTJk/Glxuqr2IAGIecLtcaiURcqqRYPp/X4OCgq9uIRCKqr69Xf3+/qyt3xeNxV7N4lUOyJ4vbOSRv94vbamtrHedIJpMuVlQ6TtkDAADAKBpSAAAAGOX5KXuug6r+a264DmpsbMli088KAMAfPJ/puQ6q+q+54TqosbEli1+v6fLLdVAAAOc4ZQ8AAACjaEgBAABgFA0pAAAAjKIhBQAAgFE0pAAAADCKhhQAAABG0ZACAADAKO44DQAey2azamlp0e7duzU4OKhkMqkrrrhCZ599tunSAMAIGlIA8Fg+n1cikdDtt9+uSZMmaefOnXriiSd09913c4N/AOMSDSkAeCwajaq5uXn064aGBtXX1+vAgQNKJpNKp9Pq6+sr+p5MJqO6urqSt+HVkquhUMj1FfWOZnE7k9tZvMoh2ZPFpvHllWrNUXbVnHoCgPL09fWpu7tbqVRKktTe3q62trai1zQ1NRU1sX6RSCQ825YtR49tySGRxY+qNUfZDSmnngBg7HK5nNauXavFixePNqSNjY1qaGgoel0mk1FXV1fJ7+vVUZJ0Oq3h4WFXtxEOh5VMJtXT06NsNuvadmKxmKtZvMoh2ZPF7RySt/vFC05zHJ13TCt7xjrZqadQKMRppw+x5RQHp53GxpYsNv2smJbP57Vu3TqFQiGtXLly9PFEInHMkcfOzk6NjIx4XeLHyuVyntWVzWZd3VY4HPYki9s5JHuyeJVD8ma/eKFac1R8tv/wqSdOOx2fLUeOvcixdOlS17fx6quvWrNPJHvGl2RXlo8qFAp68skn1d/fr89+9rMKhUKmSwIAYyrakH701BOnnYrZcorDttMbkvNTHGNhy37x6yk0v5x2KtVTTz2lrq4u3Xrrra4fcQYAv6tYt3e8U0+cdjo+W05xVOtpgePhFFrpOIVWvt7eXrW3tysUCunBBx8cffyaa67RwoULDVYGAGZUpCHl1BMAlK6+vl733Xef6TIAwDcqsnTo0VNPt9xyC6eeAAAA4EjZR0g59QQAAIBylN2QcuoJAAAA5ajIKXsAAABgrGhIAQAAYBQNKQAAAIyiIQUAAIBRNKQAAAAwioYUAAAARtGQAgAAwKiKrWUPAHBPLBZTMFj6MYRAIOBiNR9YsWKFJ9vZtGmTIpGIwmH3fm15kcWLHJIUDAYVj8dde/9AIKCBgQHXs7idQ/IuixeqOUf1VQwA49Dw8LCj19u2jHNtba36+/s1MjJiupSyeJUjHo9rcHDQtfePRCKqr693PYvbOSTvsnhhLOMrmUy6WFHpPG9I/fpXvk1/hdnyl7FXvMpiy36x6YgYAMAfPJ/p/fpXfj6ft+avMFv+MvYKRyz8yel+8ctf+QAA5/hQEwAAAIyiIQUAAIBRNKQAAAAwioYUAAAARtGQAgAAwCgaUgAAABhFQwoAAACjaEgBAABgFA0pAAAAjKrISk0DAwN68skntWvXLtXW1uryyy/XwoULK/HWAGAl5k0A+EBFGtINGzYoFArpK1/5it577z397Gc/06mnnqqpU6dW4u0BwDrMmwDwgbJP2WcyGe3YsUPNzc2KxWKaNWuWGhoatG3btkrUBwDWYd4EgGJlHyHt7u5WMBjUlClTRh+bNm2a9u7dq3Q6rb6+vqLXZzIZ1dXVlV5guCIHcT9WKBRSJBJxdRtHs7idye0sXuXwkhdZ2C/O2ZTlw042b0qqqrnTS7ZkYr4pnU2/m71SrTnKrjqTySgWixU9VlNTo+HhYbW3t6utra3ouaamJjU3N5f8/ul0Wg888IAaGxuVSCTKLdeodDqt1tZWNTY2KplMmi5nzLzM8eqrr7r6/rbsE8m7LG7vE8mu/XI8J5s3JTF3foQtY9umcU0W5xhfJ1d2QxqNRkcn0aOGh4cVi8XU2NiohoaGoucmTJjg6P37+vrU1tamhoaGqp9UbcliSw6JLH5lU5bjOdm8KYm58yNsyWJLDoksflTtOcpuSCdPnqx8Pq/u7m5NnjxZkvTee+8plUopkUhU5T8KALjpZPOmJOZOAONO2R9qikajmjt3rlpbW5XJZPTOO+/od7/7nRYtWlSJ+gDAOsybAFCsIle+Xn311Vq/fr0eeOABxeNxXX311dy6BABOgnkTAD4Quu++++4r900ikYjOO+88XXrppbrooos0bdq0CpR2RKFQUDQa1ZlnnnnMhwCqjS1ZbMkhkcWvbMpyIm7Om5Jd/4a2ZLElh0QWP6r2HIFCoVAwXQQAAADGL1/crKrUJfQKhYKeffZZbd26VZJ0/vnn64orrlAgEPC65BMqNcuLL76ojo4OHT58WLW1tVq2bJk+8YlPGKj4xJwubZjNZvXII49oeHhY9957r4eVfjwnWTo7O/X000/rwIEDikajuuSSS7R8+XKPKz6+UnNks1n9+te/1ptvvqlcLqczzjhDq1at8tUHZTZv3qyOjg4dPHhQCxYs0OrVq0/42pdfflkvvPCCRkZGNG/ePK1atapq77VXScydzJ1usmXelOyZO22eN31RWalL6LW3t+vNN9/UmjVrFAgE9NOf/lT19fVatmyZocqPVWqWQqGg1atXa9q0aerp6dGjjz6qRCKh8847z1Dlx3K6tOFLL72k2traY25n4welZunv79djjz2mq666SvPmzVMul1M6nTZU9bFKzfHKK69o//79uvvuuxWLxfRf//Vf2rBhg26++WZDlR9r4sSJuvTSS7Vr1y6NjIyc8HVvvfWWXnjhBd12222aOHGiHn/8cbW2tmrFihUeVutPzJ3MnW6yZd6U7Jk7bZ43y/6UfbmcLKHX0dGhiy66SJMmTVIikdDFF1+sjo4OA1Ufn5Msn/zkJzV9+nSFQiFNmTJFDQ0N2rdvn4Gqj8/p0oY9PT3avn27LrnkEo8r/XhOsrz88suaM2eOFi5cqHA4rFgsNnorHtOc5Ojt7dVZZ52lCRMmKBKJaMGCBerq6jJQ9YnNmzdPc+fOVTweP+nrOjo6tGTJEk2dOlXxeFxNTU2++rk3hbmTudNNtsybkl1zp83zpvGG9ERL6B1vAHR1denUU0/92NeZ4iTLhxUKBb3zzju++gF2mmXDhg26/PLLfXk6wEmW/fv3Kx6P6wc/+IHuv/9+/exnP1Nvb6+X5Z6QkxxLlizRvn37lE6nlclktH37ds2ZM8fLcivmeD/3/f39GhgYMFiVecydzJ1usmXelMbn3FmN86bxn4CPW0LvZK+tqalRJpNRoVDwxbVQTrJ82G9+8xsVCgUtWbLEzfIccZLljTfeUKFQ0Ny5c/X22297VWLJnGRJp9M6cOCAbr31Vk2dOlXPPPOM1q5dqzvvvNOrck/ISY7JkycrkUjoO9/5jgKBgKZNm6aVK1d6VWpFHe/nXjqyslFtba2psoxj7mTudJMt86Y0PufOapw3jR8h/bgl9E722uHhYUWjUV9MqJKzLEdt3rxZ27Zt02c+8xlf/YVcapZMJqNnnnlGf/zHf+xleY442S+RSERz587VjBkzFIlE9KlPfUr79u3T0NCQV+WekJMcLS0tyuVy+tu//Vt97Wtf09y5c/Uf//EfXpVaUcf7uZdUlbc1qSTmTuZON9kyb0rjc+6sxnnTeEP64SX0jvrwEnoflkql9P7773/s60xxkkWStm7dqhdeeEG33nqrJk2a5FWZJSk1S3d3t3p7e/WjH/1IDzzwgH7xi1+or69PDzzwgHp6erwu+7ic7JdK3wuykpzkeO+997R48WLV1tYqHA7rggsu0Lvvvqv+/n4vS66I4/3c19XV+favfK8wdzJ3usmWeVMan3NnNc6bxhtSJ0voLVq0SC+//LLS6bTS6bRefvllLV682EDVx+cky/bt27Vx40bdeuutOuWUUwxUe3KlZpk6daruuecerVmzRmvWrNG1116ruro6rVmzxje/KJzsl8WLF+vNN9/UgQMHlMvltGnTJp1xxhmjpztMcpJjxowZ2rZtm4aGhpTL5bRlyxZNnDhRdXV1Bio/vlwup5GRERUKBRUKBY2MjCiXyx3zukWLFmnr1q06ePCgBgcHtWnTJl/93JvC3Mnc6SZb5k3JrrnT5nnTFzfGHxgY0Pr167V7927F43FdccUVWrhwofbu3avHHntMX/va1yQduYD9mWeeKbqX3ooVK3xz2kkqPctDDz2kdDqtUCg0+r0LFy7UNddcY6r0Y5Sa5cPefvttrVu3zlf30pOcZdmyZYs2bdqkkZERnXHGGbr66qt98QtCKj3HwMCAfv3rX2vXrl3K5XKaOnWqPv3pT+v00083nOADra2tamtrK3qsqalJS5Ys0fe//3391V/9lerr6yUduS3Oiy++WDX30/MKc+cRzJ3usGXelOyZO22eN33RkAIAAGD8Mn7KHgAAAOMbDSkAAACMoiEFAACAUTSkAAAAMIqGFAAAAEbRkAIAAMAoGlIAAAAYRUMKAAAAo2hIAQAAYBQNKQAAAIyiIQUAAIBRNKQAAAAwioYUAAAARtGQAgAAwCgaUgAAABhFQwoAAACjaEgBYAzuu+8+fe5zn5Mk7dmzR4FAQNls1nBVAFCdaEgBAABgFA0pAKtx1BIA/I+GFIB1zjzzTH3729/WwoULVVdXp3feeSqmk8sAAAJ3SURBVEfXX3+9UqmUZs+erX/5l38ZfW0ul9M3v/lNnXXWWZo4caIaGxu1b98+SdJf//Vfa+bMmUokEmpsbNTzzz9vKhIAWI2GFICVfv7zn6ulpUWHDh3S6tWrtWjRIr377rvauHGjHnroIf33f/+3JOk73/mOfv7zn2vDhg1Kp9P60Y9+pNraWknSsmXL1NHRoUOHDukzn/mM/uzP/kxDQ0MmYwGAlWhIAVjpS1/6kmbOnKnXX39dXV1d+vrXv65oNKo/+qM/0l/8xV/o8ccflyT94Ac/0P/7f/9PDQ0NCgQCWrRokSZPnixJ+tznPqfJkycrHA7r3nvv1fDwsH73u9+ZjAUAVgqbLgAA3DBz5kxJ0t69e9XZ2an6+vrR53K5nC655BJJ0r59+3TWWWcd9z0efPBB/fCHP1RnZ6cCgYDS6bR+//vfu188AIwzNKQArBQIBCQdaUxnz56tnTt3Hvd1M2fO1K5du7RgwYKix59//nndf//92rhxo+bPn69gMKhkMqlCoeB67QAw3nDKHoDVLrjgAk2cOFHf/va3NTg4qFwup9dff11btmyRJN111136u7/7O+3cuVOFQkHbt29Xd3e3/vCHPygcDiuVSimbzeof//EflU6nDacBADvRkAKwWigU0lNPPaWOjg7Nnj1bU6ZM0V133aXDhw9Lkr785S/rxhtv1JVXXqlEIqE777xTg4OD+vSnP62rrrpK55xzjmbNmqWamprRywAAAJUVKHD+CQAAAAZxhBQAAABG0ZACAADAKBpSAAAAGEVDCgAAAKNoSAEAAGAUDSkAAACMoiEFAACAUTSkAAAAMIqGFAAAAEb9f3In6q1p7rhpAAAAAElFTkSuQmCC\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "" ] }, "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ggplot(data=recalls_df.loc[recalls_df.recall>=0,:],aesthetics=aes(x = \"recall\"))+geom_histogram()+facet_wrap(\"split\",scales = \"free\")" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
splitrecall
03.0-1.000000
12.0-1.000000
21.0-1.000000
33.0-1.000000
41.0-1.000000
54.0-1.000000
63.0-1.000000
74.00.333333
83.0-1.000000
93.0-1.000000
104.0-1.000000
111.00.000000
123.0-1.000000
133.0-1.000000
141.00.250000
152.0-1.000000
161.0-1.000000
173.00.000000
181.00.000000
191.0-1.000000
203.0-1.000000
211.0-1.000000
223.0-1.000000
231.0-1.000000
243.0-1.000000
252.0-1.000000
262.0-1.000000
271.0-1.000000
282.0-1.000000
291.0-1.000000
.........
3051.0-1.000000
3063.0-1.000000
3073.0-1.000000
3083.00.000000
3092.0-1.000000
3103.0-1.000000
3111.00.500000
3123.0-1.000000
3133.00.500000
3143.0-1.000000
3153.0-1.000000
3163.00.000000
3174.00.000000
3181.00.333333
3192.0-1.000000
3202.0-1.000000
3212.0-1.000000
3222.0-1.000000
3233.0-1.000000
3242.0-1.000000
3251.0-1.000000
3264.0-1.000000
3272.0-1.000000
3283.0-1.000000
3291.0-1.000000
3303.0-1.000000
3312.0-1.000000
3322.0-1.000000
3331.00.000000
3341.0-1.000000
\n", "

335 rows × 2 columns

\n", "
" ], "text/plain": [ " split recall\n", "0 3.0 -1.000000\n", "1 2.0 -1.000000\n", "2 1.0 -1.000000\n", "3 3.0 -1.000000\n", "4 1.0 -1.000000\n", "5 4.0 -1.000000\n", "6 3.0 -1.000000\n", "7 4.0 0.333333\n", "8 3.0 -1.000000\n", "9 3.0 -1.000000\n", "10 4.0 -1.000000\n", "11 1.0 0.000000\n", "12 3.0 -1.000000\n", "13 3.0 -1.000000\n", "14 1.0 0.250000\n", "15 2.0 -1.000000\n", "16 1.0 -1.000000\n", "17 3.0 0.000000\n", "18 1.0 0.000000\n", "19 1.0 -1.000000\n", "20 3.0 -1.000000\n", "21 1.0 -1.000000\n", "22 3.0 -1.000000\n", "23 1.0 -1.000000\n", "24 3.0 -1.000000\n", "25 2.0 -1.000000\n", "26 2.0 -1.000000\n", "27 1.0 -1.000000\n", "28 2.0 -1.000000\n", "29 1.0 -1.000000\n", ".. ... ...\n", "305 1.0 -1.000000\n", "306 3.0 -1.000000\n", "307 3.0 -1.000000\n", "308 3.0 0.000000\n", "309 2.0 -1.000000\n", "310 3.0 -1.000000\n", "311 1.0 0.500000\n", "312 3.0 -1.000000\n", "313 3.0 0.500000\n", "314 3.0 -1.000000\n", "315 3.0 -1.000000\n", "316 3.0 0.000000\n", "317 4.0 0.000000\n", "318 1.0 0.333333\n", "319 2.0 -1.000000\n", "320 2.0 -1.000000\n", "321 2.0 -1.000000\n", "322 2.0 -1.000000\n", "323 3.0 -1.000000\n", "324 2.0 -1.000000\n", "325 1.0 -1.000000\n", "326 4.0 -1.000000\n", "327 2.0 -1.000000\n", "328 3.0 -1.000000\n", "329 1.0 -1.000000\n", "330 3.0 -1.000000\n", "331 2.0 -1.000000\n", "332 2.0 -1.000000\n", "333 1.0 0.000000\n", "334 1.0 -1.000000\n", "\n", "[335 rows x 2 columns]" ] }, "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ "recalls_df" ] }, { "cell_type": "code", "execution_count": 30, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.625" ] }, "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ "1.0*df.sum()[\"tp\"]/(df.sum()[\"tp\"]+df.sum()[\"fp\"])\n" ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.08771929824561403" ] }, "execution_count": 31, "metadata": {}, "output_type": "execute_result" } ], "source": [ "1.0*df.sum()[\"tp\"]/(df.sum()[\"tp\"]+df.sum()[\"fn\"])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Marginals analysis" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Index([u'Unnamed: 0', u'label', u'marg', u'pred', u'split'], dtype='object')" ] }, "execution_count": 32, "metadata": {}, "output_type": "execute_result" } ], "source": [ "predictions_df.columns" ] }, { "cell_type": "code", "execution_count": 33, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsQAAAH9CAYAAAAdyGp7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3V9sXPWd///X/Dnzx3YGT5KJDQkkkKbGOHHMDqGUhBgDUUsoxigsS1qBYFd0g6pcRK1UJG56s1K1gAgrrdSL/ba0pd1CSgReO1SCNHGgRVZj5BjkhEKSBqz8m9gGd/5k/tjzu6CeH0P+ME5m5sw55/m4aTPnzJm353U+c95zOHM+rnw+nxcAAADgUG6zCwAAAADMREMMAAAAR6MhBgAAgKPREAMAAMDRaIgBAADgaDTEAAAAcDQaYgCookOHDumb3/ym/H6/nnnmGbPLAQBI8ppdAAA4yfz58/Vf//VfevXVV80uBQDwD5whBoAqWrRokdasWSPDMMwuBQDwDzTEAAAAcDQaYgAAADgaDTEAVNh///d/q6OjQx0dHTp+/LjZ5QAAvoQf1QFAhf3gBz/QD37wA7PLAABcgCufz+fNLgIAnOLkyZO66aabNDU1JbfbrYaGBo2OjioUCpldGgA4Fg0xAAAAHI1riAEAAOBoNMQAAABwNBpiAAAAOBoNMQAAAByNhhgAAACOZqn7EMdiMUUiEcViMWWzWbPLKRu/3690Om12GWVjGAY5WQA5WQM5WQM5WYMdczIzo6uuusqU160EzhDXALebGKyAnKyBnKyBnKyBnGofGZUH7yIAAAAcjYYYAAAAjkZDDAAAAEejIQYAAICj0RADAADA0WiIAQAA4GiWug8xyqu7u9vsEiqqt7fX7BIAAIAFWKoh9nq9SiaTMgxDXq+lSr8ot9utYDBodhm2U+731G45uVwuxpMFkJM1kJM12DEnu2VkFkvtDblcTuFwWIlEwjYzzEifN26pVMrsMmyn3O+p3XIyDEONjY2MpxpHTtZATtZgx5zMzCgcDpvyupXANcQAAABwNBpiAAAAOBoNMQAAAByNhhgAAACORkMMAAAAR6MhBgAAgKPREAMAAMDRSroP8eDgoIaHh3X69GmtXLlS999/vyRpcnJSzz//vAzDKKy7bt06dXZ2Svr8vsF9fX0aHR2VYRhau3atbr311sK6R44cUX9/vz777DMtWbJEPT09amxsLOffBwAAAFxUSQ3xvHnztH79eh0+fPi8N7J+8skn5fF4znl87969mpiY0LZt2xSPx/XCCy8oEoloxYoVSiQSeumll9Td3a2vf/3r2rNnj3bs2KHHH3/88v8qAAAAoEQlXTJxww03qLW1dc5TAw4PD2v9+vUKBoOKRCKKRqMaHh6WJB08eFCRSERtbW0yDEO33367Tp06pVgsNve/AgAAALhEZZm6efv27ZKk5cuXa8OGDaqvr1cqlVI8Hldzc3NhvaamJh06dEiSFIvFipb5fD6Fw2HFYjFFIhFNTU0pHo8Xvc709PTnRdtk/vFZHo+n6LITlEe531O75TQ7jhhPtY2crIGcrMGOOdktI7Nc1h5RV1enxx9/XM3NzUqlUurv79fOnTv18MMPK5PJSJL8fn9h/UAgoHQ6LUnKZDKqq6sr2t4Xlw8NDWlgYKBoeWdnp66++mpbzZ2NyolEImaXYAmMJ2sgJ2sgJ2sgJ3zZZTXEfr9fixcvliQ1NDRo48aNevbZZ5VOp+Xz+SRJ6XS68M0lnU4XGmSfz1dofmd9cXk0GlVLS0vR8tkzxJOTk8rlcpdTek3x+/3nvBe4fOW+/MZuOXm9XoXDYcZTjSMnayAna7BjTmZmZKcTT2X9bwYul0uSlM/nFQwG1dDQoFOnTqmhoUGSdPLkycKbF4lEdODAgcJzM5mMJiYmCstDoZBCoVDR9mcbnFwud94f91mV1+u11d9TK8r9nto1J8aTNZCTNZCTNdgpJ7tmVG0l/ahuenpa2WxW+Xxe+Xxe2WxW09PTGhsb05kzZzQzM6NkMqnXX39dy5YtUyAQkCStXr1a+/btUyqVUiwW07vvvquOjg5JUmtrq06fPq3R0VFls1kNDAyoqanJVt82AAAAUPtKOkO8b9++out5R0ZG1NnZqYULF2r37t1KJBLy+/267rrrtGnTpsJ6XV1d6uvr03PPPVe4D/GKFSskSfX19XrwwQe1a9cu7dy5U4sXL9YDDzxQ5j8PAAAAuLiSGuKuri51dXWdd9mqVasuvHGvVz09Perp6Tnv8uXLl2vr1q2llAAAAABUBFM3AwAAwNFoiAEAAOBoNMQAAABwNBpiAAAAOBoNMQAAAByNhhgAAACOVtaZ6irN6/UqmUzKMAx5vZYq/aLcbreCwaDZZdhOud9Tu+XkcrkYTxZATtZATtZgx5zslpFZLLU35HI5hcNhJRIJW01TGAwGlUqlzC7Ddsr9ntotJ8Mw1NjYyHiqceRkDeRkDXbMycyMwuGwKa9bCVwyAQAAAEejIQYAAICj0RADAADA0WiIAQAA4Gg0xAAAAHA0GmIAAAA4mqVuu1Zt3d3dZpcAAACACiupIR4cHNTw8LBOnz6tlStX6v7775ckffLJJ9qzZ4+OHz8ut9utZcuW6e6779a8efMkSXv27NFbb70lj8dT2NYTTzyh+fPnS5JOnDih3t5exWIxRSIRdXd368orryz33wgAAABcUEkN8bx587R+/XodPny46EbWZ8+eVTQa1YMPPii3261du3bp1Vdf1cMPP1xYp62tTZs2bTpnm7lcTr/73e90yy23aM2aNdq/f79+97vfaevWrbaZPQYAAAC1r6RriG+44Qa1traeMzXgihUr1NbWpkAgIJ/Pp5tvvlmffPJJSS/8t7/9TTMzM7rlllvk9Xp1yy23KJ/P6+jRo3P/KwAAAIBLVNZTsceOHVMkEil67K9//at++tOfat68ebr55pu1Zs0aSVIsFlNTU5NcLldh3aamJsViMa1YsUJTU1OKx+NF25qenv68aM4gowSGYZR1ex6Pp+zbNNPsOLLbeCInayAnayCn2me3jMxStj3i5MmTGhgY0ObNmwuPtbW1KRqNqqGhQWNjY3r55ZcVCAS0atUqZTIZ+f3+om0EAgGl02lJ0tDQkAYGBoqWd3Z26uqrr7bV3NmonC9/OcP5MZ6sgZysgZysgZzwZWVpiMfHx/Wb3/xGd999t5YuXVp4fNGiRYX/f8011+gb3/iGRkdHtWrVKvl8vkLzOyudThea5Gg0qpaWlqLls2eIJycnlcvlylE6bCwWi5V1e36//5x91sq8Xq/C4bDtxhM5WQM5WQM51T4zM7LTiafLbog//fRT/epXv9L69eu1evXqi67rcrmUz+clff4m/vnPf1Y+ny9cNnHq1KnCJRWhUEihUKjo+bMNTi6XK/pxH3A+5d5HvF6vLfc7u40ncrIGcrIGcqp9ds2o2kr6Ud309LSy2azy+bzy+byy2aymp6c1NTWlX/7yl0XXBn/RoUOHlEqllM/nNTY2psHBQV1//fWSpGXLlsntdmtwcFC5XE6Dg4OSpGuvvbaMfx4AAABwcSWdId63b1/R9bwjIyPq7OyUy+XS5OSk9u7dq7179xaWP/XUU5Kk999/X6+99ppyuZxCoZDWrl2rjo6Oz1/Y69VDDz2k3t5evfnmm1q4cKEeeughW13oDgAAgNpXUvfZ1dWlrq6u8y67/fbbL/i8Bx544KLbvfLKK/Xv//7vpZQAAAAAVERJl0wAAAAAdkVDDAAAAEejIQYAAICj8Qs2ADWpu7vb7BIqpre31+wSAABfwBliAAAAOBoNMQAAAByNhhgAAACOZqlriL1er5LJpAzDYAIPfKVgMFjW7bnd7rJv00wul4vxZJK57Ed2zYnxZA3kVPvslpFZLLU35HI5hcNhJRIJ5u3GV0qlUmXdXjAYLPs2zWQYhhobGxlPJpjLfmTXnBhP1kBOtc/MjMLhsCmvWwlcMgEAAABHoyEGAACAo9EQAwAAwNFoiAEAAOBoNMQAAABwNBpiAAAAOBoNMQAAABytpPsQDw4Oanh4WKdPn9bKlSt1//33F5YdOXJE/f39+uyzz7RkyRL19PSosbFR0uf3De7r69Po6KgMw9DatWt16623lvRcAAAAoBpKOkM8b948rV+/XjfeeGPR44lEQi+99JLuuOMO/fjHP9ZVV12lHTt2FJbv3btXExMT2rZtmx599FH96U9/0ocffljScwEAAIBqKOkM8Q033CBJOn78eNHMLgcPHlQkElFbW5sk6fbbb9d//ud/KhaLKRKJaHh4WD09PQoGgwoGg4pGoxoeHtaKFSu+8rlTU1OKx+NFdUxPT39etE2mW0RlGYZR1u15PJ6yb9NMs+OI8VR9c9mP7JoT48kayKn22S0js1zWHhGLxdTc3Fz4t8/nUzgcViwWU0NDg+LxeNHypqYmHTp06CufG4lENDQ0pIGBgaLX6+zs1NVXX22rqQJROZFIxOwSLIHxVH2Xsm+SkzWQkzWQE77sshriTCajurq6oscCgYDS6bQymYwkye/3n7Psq54rSdFoVC0tLUXLZ88QT05OKpfLXU7pcIBYLFbW7fn9/sL+aQder1fhcJjxZIK57Jt2zYnxZA3kVPvMzMhOJ54uqyH2+XznhJBOp+X3++Xz+Qr/nj2VP7vsq54rSaFQSKFQqGj57EEkl8sVXboBnE+59xGv12vL/Y7xVH2X8n7bLSfGkzWQU+2za0bVdlm3XYtEIjp16lTh35lMRhMTE4pEIgoGg2poaChafvLkycK3iYs9FwAAAKiWkhri6elpZbNZ5fN55fN5ZbNZTU9Pq7W1VadPn9bo6Kiy2awGBgbU1NRUaGpXr16tffv2KZVKKRaL6d1331VHR4ckfeVzAQAAgGoo6ZKJffv2Ff3AbWRkRJ2dnerq6tKDDz6oXbt2aefOnVq8eLEeeOCBwnpdXV3q6+vTc889V7gP8YoVKyRJ9fX1F30uAAAAUA0lNcRdXV3q6uo677Lly5dr69at59+416uenh719PTM+bkAAABANTB1MwAAABzNPnemBhymu7vb7BIAALAFzhADAADA0WiIAQAA4Gg0xAAAAHA0GmIAAAA4mqV+VOf1epVMJmUYhrxeS5UOEwSDwbJuz+12l32bcKa57Ecul8uWn3t2G0/kZA12zMluGZnFUntDLpdTOBxWIpFg3m58pVQqVdbtBYPBsm8TzjSX/cgwDDU2Ntruc89u44mcrMGOOZmZUTgcNuV1K4FLJgAAAOBoNMQAAABwNBpiAAAAOBoNMQAAAByNhhgAAACORkMMAAAAR6MhBgAAgKNd9n2I/+M//qPo37lcTmvWrNHGjRs1OTmp559/XoZhFJavW7dOnZ2dhXX7+vo0OjoqwzC0du1a3XrrrZdbEgAAAFCyy26In3rqqcL/T6fTeuaZZ3TDDTcUrfPkk0/K4/Gc89y9e/dqYmJC27ZtUzwe1wsvvKBIJKIVK1ZcblkAAABAScp6ycTBgwdVX1+vpUuXlrT+8PCw1q9fr2AwqEgkomg0quHh4XKWBAAAAFxUWaduHh4e1urVq+VyuYoe3759uyRp+fLl2rBhg+rr65VKpRSPx9Xc3FxYr6mpSYcOHZIkTU1NKR6PF21nenr686JtMv84KuuLl+qUg8fjKfs24Uxz2Y9mP+/s9rlnt/FETtZgx5zslpFZyrZHfPrppzp27Jjuu+++wmN1dXV6/PHH1dzcrFQqpf7+fu3cuVMPP/ywMpmMJMnv9xfWDwQCSqfTkqShoSENDAwUvUZnZ6euvvpqW82djcqJRCJmlwCc16Xsm3zuWQM5WQM54cvK1hAfOHBA11xzTdFO5vf7tXjxYklSQ0ODNm7cqGeffVbpdFo+n0/S59cdz36zSafThQY5Go2qpaWl6DVmzxBPTk4ql8uVq3TYVCwWK+v2/H5/4QsbcDnmsm96vV6Fw2Hbfe7ZbTyRkzXYMSczM7LTiaeyNsTr1q276Dqzl1Lk83kFg0E1NDTo1KlTamhokCSdPHmy8OaGQiGFQqGi588eRHK5nLLZbLlKh02Vex/xer3sdyiLS9mP7Pa5Z9fxRE7WYKec7JpRtZXlR3Uff/yx/v73v6utra3o8bGxMZ05c0YzMzNKJpN6/fXXtWzZMgUCAUnS6tWrtW/fPqVSKcViMb377rvq6OgoR0kAAABAScpyhvjAgQNqbW0tuh5Y+vzSht27dyuRSMjv9+u6667Tpk2bCsu7urrU19en5557rnAfYm65BgAAgGoqS0N87733nvfxVatWadWqVRd+ca9XPT096unpKUcZAAAAwJzZ574jAGAR3d3dZpdQUb29vWaXAABzUtaJOQAAAACroSEGAACAo9EQAwAAwNFoiAEAAOBoNMQAAABwNEvdZcLr9SqZTMowDHm9liodJggGg2XdntvtLvs2ATsqZZzYbTy5XC5bHp/IqfbZLSOzWGpvyOVyCofDSiQSTFOIr5RKpcq6vWAwWPZtAnZUyjix23gyDEONjY22Oz6RU+0zM6NwOGzK61YCl0wAAADA0WiIAQAA4Gg0xAAAAHA0GmIAAAA4Gg0xAAAAHI2GGAAAAI5GQwwAAABHK8t9iH/xi19obGxMbvfn/XUoFNLWrVslSSMjI9q9e7eSyaSuu+463Xfffaqrq5MkJZNJ9fb26vDhw6qrq9Odd96p9vb2cpQEAAAAlKRsE3Ns3LhR0Wi06LHTp0+rr69P3/3ud3XllVfq//7v/9Tf369//ud/liTt2rVLHo9HP/rRj3Ty5En99re/VXNzsxYtWlSusgAAAICLquglEyMjI/r617+uZcuWye/364477tDBgweVTqeVyWQ0Ojqqrq4u+f1+LV26VC0tLTpw4EAlSwIAAACKlO0M8e7du/Xmm29q4cKFuuOOO3TttdcqFovp6quvLqwzf/58eTwejY+Py+Vyye12a+HChYXlTU1NOnbsmCRpampK8Xi86DWmp6c/L9om84+jsgzDKOv2PB5P2bcJ2FEp48Ru42n2uGS34xM51T67ZWSWsuwRGzZsUCQSkcfj0fvvv6///d//1ZYtW5TJZOT3+4vWDQQCSqfTcrvdF1wmSUNDQxoYGCha3tnZqauvvtpWc2ejciKRiNklAI7k5LHH8ckayAlfVpaGeMmSJYX/39HRoffee08ffvihfD5focGdlU6n5ff75XK5LrhMkqLRqFpaWoqWz54hnpycVC6XK0fpsLFYLFbW7fn9/nP2WQDnKmXs2W08eb1ehcNh2x2fyKn2mZmRnb78VuS/GbhcLuXzeUUiEZ06darw+MTEhHK5nBYsWCCXy6WZmRmNj49rwYIFkqSTJ08W3txQKKRQKFS03dkP2Vwup2w2W4nSYSPl3ke8Xi/7HVCCUsaJXceT3Y5P5FT77JpRtV32j+pSqZQ++ugjZbNZTU9Pa2RkRMeOHdPXvvY1tbe364MPPtCxY8eUyWS0Z88etba2yu/3y+fzqbW1VXv27FEmk9HHH3+sDz74QKtXry7H3wUAAACU5LLPEM/MzOiPf/yjzpw5I5fLpYULF+qhhx4q/FjuO9/5jl555RWlUqnCfYhn3XPPPXrttdf09NNPKxgM6p577uGWawAAAKiqy26I6+vr9f3vf/+Cy9vb2y842UZdXZ02b958uSUAAAAAl4ypmwEAAOBoNMQAAABwNBpiAAAAOBoNMQAAAByNhhgAAACORkMMAAAAR6MhBgAAgKNVZOrmSvF6vUomkzIMQ16vpUqHCYLBYFm353a7y75NwI5KGSd2G08ul8uWxydyqn12y8gsltobcrmcwuGwEokE83bjK6VSqbJuLxgMln2bgB2VMk7sNp4Mw1BjY6Ptjk/kVPvMzCgcDpvyupXAJRMAAABwNBpiAAAAOBoNMQAAAByNhhgAAACORkMMAAAAR6MhBgAAgKNZ6rZrwFx0d3ebXQIAALCAy26Ic7mc+vv7deTIEaVSKYXDYd11111asWKFJicn9fzzz8swjML669atU2dnZ+G5fX19Gh0dlWEYWrt2rW699dbLLQkAAAAo2WU3xDMzMwqFQnr00Ud1xRVX6MMPP9SOHTv0xBNPFNZ58skn5fF4znnu3r17NTExoW3btikej+uFF15QJBLRihUrLrcsAAAAoCSX3RD7fD51dXUV/t3S0qLGxkadOHFCV1555UWfOzw8rJ6eHgWDQQWDQUWjUQ0PD2vFihWamppSPB4vWn96evrzom0y3SIA2NEX/6vghXg8npLWs4rZ45Ldjk/kVPvslpFZyr5HxONxjY+PKxKJFB7bvn27JGn58uXasGGD6uvrlUqlFI/H1dzcXFivqalJhw4dkiQNDQ1pYGCgaNudnZ26+uqrbTVVIADYzRc//52G45M1kBO+rKwN8fT0tF555RV1dHQoEokonU7r8ccfV3Nzs1KplPr7+7Vz5049/PDDymQykiS/3194fiAQUDqdliRFo1G1tLScs31JmpycVC6XK2fpAIAyicViX7mO3+8vfN7bgdfrVTgctt3xiZxqn5kZ2enLb9ka4pmZGe3cuVMej0cbN26U9HlIixcvliQ1NDRo48aNevbZZ5VOp+Xz+SRJ6XS6cKo/nU4XGuRQKKRQKFT0GrMfsrlcTtlstlylAwDKqJTPZ6/Xa8vPcbsdn8ip9tk1o2ory32I8/m8ent7lUgk9C//8i/n/QGdJLlcrsL6wWBQDQ0NOnXqVGH5yZMnbfVtAwAAALWvLA1xX1+fYrGYNm/eXHRh99jYmM6cOaOZmRklk0m9/vrrWrZsmQKBgCRp9erV2rdvn1KplGKxmN599111dHSUoyQAAACgJJd9ycSnn36qoaEheTwePfPMM4XH7733XrlcLu3evVuJREJ+v1/XXXedNm3aVFinq6tLfX19eu655wr3IeaWawBgbXafFKe3t9fsEgCU2WU3xI2NjfrJT35yweWrVq268It7verp6VFPT8/llgEAAABckrJcMgEAAABYFQ0xAAAAHI2GGAAAAI5GQwwAAABHoyEGAACAo9EQAwAAwNHKNnVzNXi9XiWTSRmGIa/XUqUDAGwiGAye85jL5bLl8cntdp/377UqO+Zkt4zMYqm9IZfLKRwOK5FIMG83AMAUqVTqnMcMw1BjY6Ptjk/BYPC8f69V2TEnMzMKh8OmvG4lcMkEAAAAHI2GGAAAAI5GQwwAAABHoyEGAACAo9EQAwAAwNFoiAEAAOBolrrtGgAAZuvu7ja7hIrq7e01uwSg6kxviJPJpHp7e3X48GHV1dXpzjvvVHt7u9llAQAAwCFMb4h37dolj8ejH/3oRzp58qR++9vfqrm5WYsWLTK7NAAAADiAqdcQZzIZjY6OqqurS36/X0uXLlVLS4sOHDhgZlkAAABwEFPPEI+Pj8vtdmvhwoWFx5qamnTs2DFNTU0pHo8XrT89PS1Jtpl/HACAWmP3a6Tt7PXXXze7BMsytbPMZDLy+/1FjwUCAaXTaQ0NDWlgYKBo2S233KKPPvpI0Wi0KvNn79+/v+KvYUdTU1MaGhpSNBpVKBQyuxxcADlZAzlZAzlZAznhQkxtiH0+n9LpdNFj6XRafr9f0WhULS0tRcsSiYR+85vfqKWlhR25hsXjcQ0MDJBTjSMnayAnayAnayAnXIipDfGCBQs0MzOj8fFxLViwQJJ08uRJRSIRhUKhc3bW48ePm1EmAAAAbMzUH9X5fD61trZqz549ymQy+vjjj/XBBx9o9erVZpYFAAAABzH912n33HOPXnvtNT399NMKBoO65557uOUaAAAAqsb0hriurk6bN28uad2GhgZ1dnaqoaGhwlXhcpCTNZCTNZCTNZCTNZATLsSVz+fzZhcBAAAAmMX0M8TnU+p0zvl8Xm+++abeffddSdI//dM/6a677pLL5ap2yY5TakZHjx7VwMCATpw4oUAgoG3btplQrXOVmtOf/vQnDQ8P67PPPlNdXZ3WrFmjtWvXmlCxM5Wa0zvvvKPBwUElk0n5fD6tXLlSGzZskMfjMaFq5yk1p1m5XE4/+9nPlE6n9cMf/rCKlTpbqTnt2bNHb731VtH4eeKJJzR//vxqlosaUZMNcanTOQ8NDenQoUPasmWLXC6XfvWrX6mxsVFr1qwxqXLnKDUjwzB04403auXKlXrrrbdMqta5Ss0pn8/r/vvvV1NTkyYnJ/XrX/9aoVBIq1atMqlyZyk1p5aWFnV0dCgYDCqZTOrll1/W4OCgbr31VpMqd5ZSc5r15z//WXV1defcXhSVNZec2tratGnTJhOqRK0x9S4T5zOX6ZyHh4f1zW9+U1dccYVCoZBuvfVWDQ8Pm1C1s8wloyVLlmj16tVVmUgFxeaS07p163TVVVfJ4/Fo4cKFamlp0SeffGJC1c4zl5zmz5+vYDBY+LfL5dLExEQ1y3WsueQkSZOTkxoZGdFtt91W5Uqdba45AbNq7gzxxaZz/rJYLKbm5uai9WKxWFXqdLK5ZATzXGpO+XxeH3/8saLRaKVLhOae08jIiPr6+pTJZFRXV6dvfetb1SrV0eaa065du3TnnXfK6625w6ytzTWnv/71r/rpT3+qefPm6eabb+a/MDtYzY3Ui03n/FXrBgIBZTIZ5fN5riOuoLlkBPNcak579+5VPp/XjTfeWMny8A9zzam9vV3t7e0aHx/XgQMHVF9fX40yHW8uOR08eFD5fF6tra06evRotUqE5pZTW1ubotGoGhoaNDY2ppdfflmBQIBLxRyq5i6ZuNh0zl+1bjqdls/noxmusLlkBPNcSk6Dg4M6cOCAvvvd73Jmq0oudTwtWLBAkUhE/f39lSwP/1BqTplMRm+88YbuvvvuapaHf5jLeFq0aJFCoZDcbreuueYafeMb39Do6Gi1SkWNqbmG+IvTOc+anc75yyKRiE6dOvWV66G85pIRzDPXnN599129/fbbeuSRR3TFFVdUq0zHu5zxNDMzo8nJyUqWh38oNafx8XF9+umn+vnPf66nn35aL730kuLxuJ5++mmyqoLLGU8ul0vcida5aq4hnst0zqtXr9Y777yjqakpTU1N6Z133lFHR4cJVTvLXDKamZnbhdNZAAAgAElEQVRRNpvVzMyMJCmbzSqXy1W7ZEeaS04jIyPavXu3HnnkEW45VGVzyWloaEjxeFySdPr0ab399tu69tprq12yI5Wa06JFi7Rt2zZt2bJFW7ZsUXd3t+rr67Vlyxa+aFbBXMbToUOHlEqllM/nNTY2psHBQV1//fUmVI1aUJMTcySTSb322ms6cuSIgsGg7rrrLrW3t+vYsWN68cUX9dRTT0n6/Mc/b7zxRtF9iDds2MAlE1VQakZHjx7VL3/5y6LnLl26VI899pgZZTtOqTlt375dU1NTRffjbG9v17333mtW6Y5Sak6vvvqqPvzww8IP6tra2tTV1SXDMEz+C5yh1Jy+6OjRo9q5cyf3Ia6iUnP6/e9/r8OHDyuXyykUCmnNmjW65ZZbTK4eZqnJhhgAAAColpq7ZAIAAACoJhpiAAAAOBoNMQAAAByNhhgAAACORkMMAAAAR6MhBgAAgKPREAMAAMDRaIgBAADgaDTEAAAAcDQaYgAAADgaDTEAAAAcjYYYAAAAjkZDDAAAAEejIQYAAICj0RADAADA0WiIAQAA4Gg0xAAAAHA0GmIAAAA4Gg0xAAAAHI2GGAAAAI5GQwwAAABHoyEGAACAo9EQAwAAwNFoiAEAAOBoNMQAAABwNBpiAAAAOBoNMQAAAByNhhgAAACORkMMAAAAR6MhBgAAgKPREAMAAMDRaIgBAADgaDTEAAAAcDQaYgAAADgaDTEAAAAcjYYYAAAAjkZDDAAAAEejIQYAAICj0RADAADA0WiIAQAA4Gg0xAAAAHA0GmIAAAA4Gg0xAAAAHI2GGAAAAI5GQwwAAABHoyEGAACAo9EQAwAAwNFoiAEAAOBoNMQAAABwNBpiAAAAOBoNMQAAAByNhhgAAACORkMMAAAAR6MhBgAAgKPREAMAAMDRaIgBAADgaDTEAFAl//qv/6pFixZp5cqVZpcCAPgCGmIAqJJHH31Uf/jDH8wuAwDwJTTEAFAl69ev1/z5880uAwDwJTTEAAAAcDQaYgAAADgaDTEAAAAcjYYYAAAAjkZDDABVsnnzZn3zm9/UBx98oCVLluj//b//Z3ZJAABJrnw+nze7CAAAAMAsnCEGAACAo9EQAwAAwNFoiAEAAOBoNMQAAABwNBpiAAAAOJrX7ALm4vjx42aXcFF+v1/pdNrsMi6ZYRiKRCKKxWLKZrNml3PJrJwDGdQGwzA0b948/f3vfycHE9lhPNghA8aC+Wp1LFx11VVml1A2lrrt2vj4uNzu2j2p7Xa7NTMzY3YZl8zlcsnn8ymTychCu8U5rJwDGdQGl8slj8ej6elpcjCRHcaDHTJgLJivVsdCOBw2u4SysdQZ4lr/dhcMBpVKpcwu45IZhqHGxkYlEoma+gY6V1bOgQxqg2EY8vl8Onv2LDmYyA7jwQ4ZMBbMV6tjwU4Nce2ebgUAAACqgIYYAAAAjkZDDAAAAEejIQYAAICj0RADAADA0Sx1lwkA/7/u7m6zS6io3t5es0sAADhExRviV155RUePHlUmk1FDQ4PWrl2raDSqyclJPf/88zIMo7DuunXr1NnZWemSAAAAgIKKN8S33Xab7rvvPnm9XsViMb3wwgu68sorFQwGJUlPPvmkPB5PpcsAAAAAzqvi1xAvWrRIXu/nfbfL5ZLL5dLExESlXxYAAAAoSVWuIe7r69Pw8LByuZyam5u1YsUKJZNJSdL27dslScuXL9eGDRtUX18vSZqamlI8Hi/aTiaTKSyvRR6Pp+gSEKuZ/eIy+79WZeUc7JJBOZiZodfrlcvlsnwOVh4Lkj3Ggx0yYCyYzw5joda58lWaFHtmZkaffPKJ/va3v2ndunXK5XI6c+aMmpublUql1N/fr0wmo4cffliStGfPHg0MDBRto7OzU11dXdUoF6h5N910k9klVNT+/fvNLgEA4BBV+6rhdru1dOlSjYyM6C9/+YtuueUWLV68WJLU0NCgjRs36tlnn1U6nZbf71c0GlVLS0vRNjKZjGKxWLVKnjO/3690Om12GZfM6/UqHA5rcnJSuVzO7HIumZVzsEsG5WDmWPd6vQoEAjp79qylc7DyWJDsMR7skAFjwXy1OhYikYjZJZRN1c+9z8zMaHJy8pzHXS6XJGn2hHUoFFIoFCpa5/jx48pms5Uv8hJ5vd6arq9UuVzO0n+HHXKwegblYPbfn8/nLZ+DHcaCZO3xYIcMGAu1w+o51LKK/qguHo/rvffeUzqd1szMjD766CO9//77uvbaazU2NqYzZ85oZmZGyWRSr7/+upYtW6ZAIFDJkgAAAIAiFT1D7HK5tH//fvX19Smfz6uxsVHf/va3df311+u9997T7t27lUgk5Pf7dd1112nTpk2VLAcAAAA4R0Ub4vr6ej322GPnXbZq1SqtWrWqki8PAAAAfKWK34cYAAAAqGU0xAAAAHA0GmIAAAA4Gg0xAAAAHI2GGAAAAI5mqUmx/X6/3O7a7eHdbreCwaDZZVwyl8ulZDIpwzAsPV+6lXOwSwblYGaGLpdL09PTls/BymNBssd4sEMGjAXz2WEs1DpLvau1Pu1iMBhUKpUyu4xLZhiGGhsblUgkLD0TjpVzsEsG5WBmhoZhyOfz6ezZs5bOwcpjQbLHeLBDBowF89XqWAiHw2aXUDa1e7oVAAAAqAIaYgAAADgaDTEAAAAcjYYYAAAAjkZDDAAAAEejIQYAAICjVfy2a6+88oqOHj2qTCajhoYGrV27VtFoVJJ05MgR9ff367PPPtOSJUvU09OjxsbGSpcEAAAAFFS8Ib7tttt03333yev1KhaL6YUXXtCVV16pK664Qi+99JK6u7v19a9/XXv27NGOHTv0+OOPV7okAAAAoKDil0wsWrSoMKuKy+WSy+XSxMSEDh48qEgkora2NhmGodtvv12nTp1SLBardEkAAABAQVVmquvr69Pw8LByuZyam5u1YsUK/fGPf1Rzc3NhHZ/Pp3A4rFgspkgkoqmpKcXj8aLtZDIZ1dfXV6PkS+LxeGQYhtllXLLZLy5WnxbSyjnYJYNyMDNDr9crl8tl+RysPBYke4wHO2TAWDCfHcZCravKO/ud73xHGzdu1CeffKK//e1v8nq9ymQyqqurK1ovEAgUpmceGhrSwMBA0fLOzk51dXVVo2RHs9NUjFZFBlIkEjG7BAWDQbNLgBgPtYCxUBsYC5VTta8abrdbS5cu1cjIiP7yl7/I5/MVmt9Z6XRafr9fkhSNRtXS0lK0PJPJ1PQlFX6//5y/yUq8Xq/C4bAmJyeVy+XMLueSWTkHu2RQDmaOda/Xq0AgoLNnz1o6ByuPBcke48EOGTAWzFerY6EWTlyUS9XPvc/MzGhyclKRSEQHDhwoPJ7JZDQxMVF4c0OhkEKhUNFzjx8/rmw2W9V658Lr9dZ0faXK5XKW/jvskIPVMygHs//+fD5v+RzsMBYka48HO2TAWKgdVs+hllX0R3XxeFzvvfee0um0ZmZm9NFHH+n999/Xtddeq9bWVp0+fVqjo6PKZrMaGBhQU1OTrb5tAAAAoPZV9Ayxy+XS/v371dfXp3w+r8bGRn3729/W9ddfL0l68MEHtWvXLu3cuVOLFy/WAw88UMlyAAAAgHNUtCGur6/XY489dsHly5cv19atWytZAgAAAHBRTN0MAAAAR6MhBgAAgKPREAMAAMDRaIgBAADgaDTEAAAAcDRLTYrt9/vldtduD+92uy09vaXL5VIymZRhGJaeL93KOdglg3IwM0OXy6Xp6WnL52DlsSDZYzzYIQPGgvnsMBZqnaXe1VqfdjEYDCqVSpldxiUzDEONjY1KJBKWngnHyjnYJYNyMDNDwzDk8/l09uxZS+dg5bEg2WM82CEDxoL5anUshMNhs0som9o93QoAAABUAQ0xAAAAHI2GGAAAAI5GQwwAAABHoyEGAACAo1nqLhMAgNrX3d1tdgkV1dvba3YJAMqsog1xLpdTf3+/jhw5olQqpXA4rLvuuksrVqzQ5OSknn/+eRmGUVh/3bp16uzsrGRJAAAAQJGKNsQzMzMKhUJ69NFHdcUVV+jDDz/Ujh079MQTTxTWefLJJ+XxeCpZBgAAAHBBFb2G2OfzqaurS+FwWG63Wy0tLWpsbNSJEycq+bIAAABAyap6DXE8Htf4+LgikUjhse3bt0uSli9frg0bNqi+vl6SNDU1pXg8XvT8TCZTWF6LPB5P0SUgVjM7HaTVp4W0cg52yaAczMzQ6/XK5XJZPgcrj4VaNpf31OoZMBZqA8eGynPl8/l8NV5oenpaL774oubPn697771X6XRaZ86cUXNzs1KplPr7+5XJZPTwww9Lkvbs2aOBgYGibXR2dqqrq6sa5QI176abbjK7hIrav3+/2SXgErFvArCaqnzVmJmZ0c6dO+XxeLRx40ZJkt/v1+LFiyVJDQ0N2rhxo5599lml02n5/X5Fo1G1tLQUbSeTySgWi1Wj5Evi9/uVTqfNLuOSeb1ehcNhTU5OKpfLmV3OJbNyDnbJoBzMHOter1eBQEBnz561dA5WHgu1bC77ptUzYCzUhlo9Nnzxv/hbXcUb4nw+r97eXiUSCX3ve9+74A/oXC5XYX1JCoVCCoVCRescP35c2Wy2sgVfBq/XW9P1lSqXy1n677BDDlbPoBzM/vvz+bzlc7DDWKhFc3lP7ZABY6F2WD2HWlbxiTn6+voUi8W0efPmout3xsbGdObMGc3MzCiZTOr111/XsmXLFAgEKl0SAAAAUFDRM8SffvqphoaG5PF49MwzzxQev/fee+VyubR7924lEgn5/X5dd9112rRpUyXLAWAhdp7cgYkdAKC2VLQhbmxs1E9+8pMLLl+1alUlXx4AAAD4ShW/ZAIAAACoZTTEAAAAcDQaYgAAADgaDTEAAAAcjYYYAAAAjmapSbH9fr/c7trt4d1ut4LBoNllXDKXy6VkMinDMCw9X7qVc7BLBri4au2fVh4LtWwu76nVM3C5XJqenrb8Z5IdcuDYUFmWeldrfdrFYDCoVCpldhmXzDAMNTY2KpFIWHomHCvnYJcMcHHV2j+tPBZq2VzeU6tnYBiGfD6fzp49a+nPJDvkUIvHhnA4bHYJZVO7p1sBAACAKqAhBgAAgKPREAMAAMDRaIgBAADgaDTEAAAAcDRL3WUCAOygu7vb7BJwGeyeX29vr9klAFVX0YY4l8upv79fR44cUSqVUjgc1l133aUVK1ZIko4cOaL+/n599tlnWrJkiXp6etTY2FjJkgAAAIAiFb1kYmZmRqFQSI8++qiefPJJ3XHHHdqxY4cmJyeVSCT00ksv6Y477tCPf/xjXXXVVdqxY0clywEAAADOUdGG2OfzqaurS+FwWG63Wy0tLWpsbNSJEyd08OBBRSIRtbW1yTAM3X777Tp16pRisVglSwIAAACKVPUa4ng8rvHxcUUiEe3fv1/Nzc2FZT6fT+FwWLFYTJFIRFNTU4rH40XPz2Qyqq+vr2bJc+LxeGQYhtllXLLZ6SCtPi2klXOwSwYArOuLn59er1cul8vyn0lWPi5IHBuqoWrv7PT0tF555RV1dHQoEokok8morq6uaJ1AIFCYnnloaEgDAwNFyzs7O9XV1VWtkh3LTlMxWhUZADBLJBI557FgMGhCJfgyjg2VU5WGeGZmRjt37pTH49HGjRslfX5GeLb5nZVOp+X3+yVJ0WhULS0tRcszmUxNX1Lh9/vP+ZusxOv1KhwOa3JyUrlczuxyLpmVc7BLBgCs64vHWa/Xq0AgoLNnz1r6M8nKxwWpdo8N5/vyZFUVb4jz+bx6e3uVSCT0ve99Tx6PR9Lnb+KBAwcK62UyGU1MTBTe3FAopFAoVLSt48ePK5vNVrrkS+b1emu6vlLlcjlL/x12yMHqGQCwri9/9uTzect/JtnhuCBxbKikik/M0dfXp1gsps2bNxddv9Pa2qrTp09rdHRU2WxWAwMDampqstW3DQAAANS+ip4h/vTTTzU0NCSPx6Nnnnmm8Pi9996r9vZ2Pfjgg9q1a5d27typxYsX64EHHqhkOQAAAMA5KtoQNzY26ic/+ckFly9fvlxbt26tZAkAAADARVX8kgkAAACgltEQAwAAwNFoiAEAAOBoNMQAAABwNBpiAAAAOJqlJsX2+/1yu2u3h3e73Zae3tLlcimZTMowDEvPl27lHOySAQDr+uLnp8vl0vT0tOU/k6x8XJA4NlSDpd7VWp92MRgMKpVKmV3GJTMMQ42NjUokEpaeCcfKOdglAwDW9cXPT8Mw5PP5dPbsWUt/Jln5uCDV7rEhHA6bXULZ1O7pVgAAAKAKaIgBAADgaDTEAAAAcDQaYgAAADgaDTEAAAAcjYYYAAAAjlbx264NDg5qeHhYp0+f1sqVK3X//fdLkiYnJ/X888/LMIzCuuvWrVNnZ2elSwIAAAAKKt4Qz5s3T+vXr9fhw4fPe++8J598Uh6Pp9JlAAAAAOdV8Yb4hhtukCQdP368pm4mDQAAAEg1MFPd9u3bJUnLly/Xhg0bVF9fL0mamppSPB4vWjeTyRSW1yKPx1N0CYjVzE4HafVpIa2cg10yAGBdX/z89Hq9crlclv9MsvJxQeLYUA2mvbN1dXV6/PHH1dzcrFQqpf7+fu3cuVMPP/ywJGloaEgDAwNFz+ns7FRXV5cZ5drSTTfdZHYJFbV//36zS7hkdpoOE4C1RCKRcx4LBoMmVIIv49hQOaY1xH6/X4sXL5YkNTQ0aOPGjXr22WeVTqfl9/sVjUbV0tJS9JxMJqNYLGZGuSXx+/1Kp9Nml4F/qOV95UK8Xq/C4bAmJyeVy+XMLgeAA33xs9Pr9SoQCOjs2bOW/kyy+vG5Vo8N5/vyZFU1c+7d5XJJkvL5vCQpFAopFAoVrVPr1yF7vd6ars9prJxFLpezdP0ArOvLnz35fN7yn0l2OT5bPYdaVvGGeHp6WjMzM8rn88rn88pms3K73Tpx4oQCgYDmz5+vs2fP6vXXX9eyZcsUCAQqXRIAAABQUPGGeN++fUXXAo+MjKizs1MLFy7U7t27lUgk5Pf7dd1112nTpk2VLgcAAFxEd3e32SVUVG9vr9kloAZVvCHu6uq64A/hVq1aVemXBwAAAC6KqZsBAADgaDTEAAAAcDQaYgAAADhazdx2rRbZ/YcFAAAA4AwxAAAAHM5SZ4j9fr/cbnp4lMaKU426XC4lk0kZhsGc9QBQARwbcD6WeletPO0iqi+VSpldwpwZhqHGxkYlEglmIwKACuDYUD7hcNjsEsqG060AAABwNBpiAAAAOBoNMQAAAByNhhgAAACORkMMAAAAR6MhBgAAgKNV/LZrg4ODGh4e1unTp7Vy5Urdf//9hWVHjhxRf3+/PvvsMy1ZskQ9PT1qbGysdEkAAABAQcXPEM+bN0/r16/XjTfeWPR4IpHQSy+9pDvuuEM//vGPddVVV2nHjh2VLgcAAAAoUvGG+IYbblBra+s5M8McPHhQkUhEbW1tMgxDt99+u06dOqVYLFbpkgAAAIAC02aqi8Viam5uLvzb5/MpHA4rFospEoloampK8Xi86DmZTEb19fXVLhUWZRiG2SXM2eyUnEzNCQCVwbEB52PaO5vJZFRXV1f0WCAQKEzPPDQ0pIGBgaLlnZ2d6urqqlqNsLZIJGJ2CZfMTtNhAkAt4diA8zGtIfb5fIXmd1Y6nZbf75ckRaNRtbS0FC3PZDJcUoGSWXFf8Xq9CofDmpycVC6XM7scALAdjg3lY+UvF19mWkMciUR04MCBwr8zmYwmJiYKb24oFFIoFCp6zvHjx5XNZqtaJ6zLyvtKLpezdP0AUKus/NnKsaFyKv6juunpaWWzWeXzeeXzeWWzWU1PT6u1tVWnT5/W6OiostmsBgYG1NTUZKtvGwAAAKh9FT9DvG/fvqJrgUdGRgrXAj/44IPatWuXdu7cqcWLF+uBBx6odDkAAABAkYo3xF1dXRf8Idzy5cu1devWSpcAAAAAXBBTNwMAAMDRaIgBAADgaDTEAAAAcDQaYgAAADgaDTEAAAAczVKTYvv9frnd9PAoTTAYNLuEOXO5XEomkzIMgznrAaACODbgfCz1rn55qmfgYlKplNklzJlhGGpsbFQikWA2IgCoAI4N5RMOh80uoWw43QoAAABHoyEGAACAo9EQAwAAwNFoiAEAAOBoNMQAAABwNBpiAAAAOJrpt137xS9+obGxscL9hUOhkLZu3WpyVQAAAHAK0xtiSdq4caOi0ajZZQAAAMCBuGQCAAAAjlYTZ4h3796tN998UwsXLtQdd9yha6+9VlNTU4rH40XrZTIZ1dfXm1QlrMYwDLNLmLPZKTmZmhMAKoNjA87Hlc/n82YWMDY2pkgkIo/Ho/fff1+7du3Sli1bdODAAQ0MDBSt29nZqa6urqrVdtNNN1XttVB++/fvN7uEimL/BAB8kd2Pe5Vk+leNJUuWFP5/R0eH3nvvPX344YeKRqNqaWkpWjeTySgWi1W7RFiUFfcVr9ercDisyclJ5XI5s8sBAFhItY97kUikqq9XSaY3xF/mcrmUz+cVCoUUCoWKlh0/flzZbNakymA1Vt5XcrmcpesHAFQfx41LZ+qP6lKplD766CNls1lNT09rZGREx44d09e+9jUzywIAAICDmHqGeGZmRn/84x915swZuVwuLVy4UA899JAWLlxoZlkAAABwEFMb4vr6en3/+983swQAAAA4HPchBgAAgKPREAMAAMDRaIgBAADgaDTEAAAAcDQaYgAAADhazU3McTF+v19uNz08ShMMBs0uYc5cLpeSyaQMw2DOegDAnFjxuFcrLHXETafTZpcAC0mlUmaXMGeGYaixsVGJRIIZhwAAc1Lt4144HK7q61USp1sBAADgaDTEAAAAcDQaYgAAADgaDTEAAAAcjYYYAAAAjkZDDAAAAEcz/bZryWRSvb29Onz4sOrq6nTnnXeqvb3d7LIAAADgEKY3xLt27ZLH49GPfvQjnTx5Ur/97W/V3NysRYsWmV0aAAAAHMDUSyYymYxGR0fV1dUlv9+vpUuXqqWlRQcOHDCzLAAAADiIqWeIx8fH5Xa7tXDhwsJjTU1NOnbsmKamphSPx4vWz2Qyqq+vr3aZsCjDMMwuYc5mp2tm2mYAwFxZ8bhXK0w96mYyGfn9/qLHAoGA0um0hoaGNDAwULSss7NTXV1dVatv//79VXutWjA1NaWhoSFFo1GFQiGzy3Gkqakp7dmzR9Fo9CunxHTa/llNjIXaQA7mI4PaQA6VZ2pD7PP5lE6nix5Lp9Py+/2KRqNqaWkpWtbQ0FDN8hwnHo9rYGBALS0tDDiTkEFtIIfaQA7mI4PaQA6VZ2pDvGDBAs3MzGh8fFwLFiyQJJ08eVKRSEShUIjQAQAAUHGm/qjO5/OptbVVe/bsUSaT0ccff6wPPvhAq1evNrMsAAAAOIjpv9y555579Nprr+npp59WMBjUPffcwy3XAAAAUDWmN8R1dXXavHmz2WVAn1+j3dnZybXaJiKD2kAOtYEczEcGtYEcKs+Vz+fzZhcBAAAAmMX0M8SorlKnyj569KgGBgZ04sQJBQIBbdu2zYRq7avUHP70pz9peHhYn332merq6rRmzRqtXbvWhIrtp9QM3nnnHQ0ODiqZTMrn82nlypXasGGDPB6PCVXbT6k5zMrlcvrZz36mdDqtH/7wh1Ws1L5KzWDPnj166623ivb9J554QvPnz69mubY1l7Fw/Phx/eEPf9CJEyfk8/l022236ZZbbqlyxfZCQ+wwpU6VbRiGbrzxRq1cuVJvvfWWSdXaV6k55PN53X///WpqatLk5KR+/etfKxQKadWqVSZVbh+lZtDS0qKOjg4Fg0Elk0m9/PLLGhwc1K233mpS5fZSag6z/vznP6uuru6cW3bi0s0lg7a2Nm3atMmEKu2v1BwSiYRefPFFffvb39YNN9yg6elpTU1NmVS1fZh6lwlU11ymyl6yZIlWr179lZNDYO7mksO6det01VVXyePxaOHChWppadEnn3xiQtX2MpcM5s+fr2AwWPi3y+XSxMRENcu1rbnkIEmTk5MaGRnRbbfdVuVK7WuuGaAy5pLDO++8o6997Wtqb2+X1+uV3+9XJBIxoWp74Qyxg1xsqmxUz6XmkM/n9fHHHysajVa6RNubawYjIyPq6+tTJpNRXV2dvvWtb1WrVFubaw67du3SnXfeydTmZTTXDP7617/qpz/9qebNm6ebb75Za9asqVaptjaXHMbGxtTU1KT/+Z//0cTEhJYsWaKNGzeqsbGxmiXbDp8qDnKxqbJRPZeaw969e5XP53XjjTdWsjxHmGsG7e3tam9v1/j4uA4cOKD6+vpqlGl7c8nh4MGDyufzam1t1dGjR6tVou3NJYO2tjZFo1E1NDRobGxML7/8sgKBAJdwlcFccpiamtKJEyf0yCOPaNGiRXrjjTf0yiuv6N/+7d+qVa4tccmEg1xsqmxUz6XkMDg4qAMHDui73/0uZ8fK4FLHwoIFCxSJRNTf31/J8hyj1BwymYzeeOMN3X333dUszxHmMhYWLVqkUCgkt9uta665Rt/4xjc0OjparVJtbS45GIah1tZWLV68WIZh6Pbbb9cnn3yis2fPVqtcW6IhdpAvTpU9a3aqbFTPXHN499139fbbb+uRRx7RFVdcUa0ybe1yxsLMzIwmJycrWZ5jlJrD+Pi4Pv30U/385z/X008/rZdeeknxeFxPP/00WVymyxkLLpdL3Lm1POaSQ1NTUzVLcwwaYgeZy1TZMzMzymazmpmZkSRls1nlcrlql2xLc8lhZGREu3fv1iOPPMKtjcpoLhkMDQ0pHo9Lkk6fPq23335b1157bbVLtqVSc1i0aJG2bdumLVu2aMuWLeru7lZ9fb22bNnCl8TLNJexcOjQIaVSKeXzeY2NjWlwcFDXX3+9CVXbz1xy6Ojo0BYVJtsAAAQLSURBVKFDh3TixAlNT09r3759uuaaaxQIBEyo3D6YmMNhksmkXnvtNR05ckTBYFB33XWX2tvbdezYMb344ot66qmnJH1+H+Jf/vKXRc9dunSpHnvsMTPKtp1Sc9i+fbumpqaK7vvZ3t6ue++916zSbaPUDF599VV9+OGHhR/UtbW1qaurS4ZhmPwX2EOpOXzR0aNHtXPnTu5DXCalZvD73/9ehw8fVi6XUygU0po1a7j3bRnNZSz85S9/0b59+5TNZnXNNdfonnvu4cvhZaIhBgD8f+3cP0hVfRzH8Y9kSF1SIkohIqHNQVAQgrTJ5QYhBg0S/SEniSaHJB2cWtqi1cEGN4dwTKm5MQqklqCIyiIuN5zS+0zPxXrseaDh3gd+r9f0u4dzDt+zvTn3xwEomi0TAAAUTRADAFA0QQwAQNEEMQAARRPEAAAUTRADAFA0QQwAQNEEMQAARRPEAAAUTRADAFA0QQwAQNEEMQAARRPEAAAUTRADAFA0QQwAQNEEMQAARRPEAAAUTRADAFA0QQzQIjs7O+0eAYB9CGKAX/T39+f+/fsZHBxMpVLJ9PR0Pn36lGq1miNHjmR8fDzfvn1Lkly+fDl9fX3p6enJ+fPn8+rVq+Z9bty4kZmZmVy4cCGVSiVPnz7N169fc/HixXR3d2dkZCQLCwsZHR1t16MCEEEMsK/V1dU8efIkr1+/ztraWqrVau7du5etra3s7u7mwYMHSZJqtZo3b97k8+fPGR4ezpUrV366z8rKSubn51Ov1zM6Oppbt26lUqnk48ePWV5ezvLycjseD4A9Ots9AMD/0e3bt9Pb25skGRsby4kTJzI0NJQkmZyczMbGRpLk5s2bzWsWFxdz9OjR1Gq19PT0JEkmJiZy7ty5JMnBgwezurqaly9f5vDhwxkYGMj169fz7NmzFj4ZAL/yhhhgH3/HcJIcOnToH7+/f/+enZ2dzM3N5cyZM+nu7k5/f3+S5MuXL81zT5061VxvbW3lx48fPx3buwagPQQxwB9aWVnJ48ePs76+nlqtlrdv3yZJGo1G85yOjo7m+vjx4+ns7Mz79++bx969e9eyeQHYnyAG+EP1ej1dXV05duxYtre3c/fu3X89/8CBA7l06VIWFxezvb2dzc3NPHr0qEXTAvA7ghjgD127di2nT5/OyZMnMzAwkLNnz/7nNQ8fPkytVktfX1+uXr2aqampdHV1tWBaAH6no7H3vz0AWurOnTvNL04A0B7eEAO00ObmZl68eJFGo5Hnz59naWkpk5OT7R4LoGg+uwbQQvV6PVNTU/nw4UN6e3szOzubiYmJdo8FUDRbJgAAKJotEwAAFE0QAwBQNEEMAEDRBDEAAEUTxAAAFE0QAwBQtL8ANpSjTUrosmYAAAAASUVORK5CYII=\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "" ] }, "execution_count": 33, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ggplot(data = predictions_df,aesthetics=aes(x = \"marg\"))+geom_histogram()+facet_wrap(\"label\",scales = \"free\")" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [], "source": [ "from sklearn.metrics import roc_curve,auc" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XeYU2X2wPHvAaWI2MAKIigoolJHig07iAXXAiii2FCxi23VtS3u2nsDwfZTwa7oorgqSlm6CFKkKwyKIIICMsDMnN8f58YJw0wmM0xyU87nefIkubnJPbmTycl93/eeV1QV55xzrjRVwg7AOedcavNE4ZxzLiZPFM4552LyROGccy4mTxTOOedi8kThnHMuJk8ULm4i0lNEPgs7jlQiImtFZN8QtttQRFREtkn2thNBRGaKyNEVeJ5/JpPAE0WaEpEfRGR98EW1TEReFpHtE7lNVX1dVU9M5DaiichhIvKliKwRkd9F5CMRaZas7ZcQz1cickn0MlXdXlUXJmh7+4vI2yLya/D+p4vIDSJSNRHbq6ggYTXemtdQ1YNU9asytrNFckz2ZzJbeaJIb6eq6vZAS6AV8PeQ46mQkn4Vi0gH4DPgQ2AvoBEwDRibiF/wqfbLXET2AyYAS4BDVHVH4GwgB6hdydsK7b2n2n53pVBVv6ThBfgBOD7q/oPAf6LuVwceBhYDvwDPAzWjHu8KfAv8ASwAOgfLdwQGAz8DS4H+QNXgsd7AmOD2c8DDxWL6ELghuL0X8C6wAlgEXBO13t3AO8BrwfYvKeH9jQaeLWH5J8Crwe2jgVzgNuDXYJ/0jGcfRD33FmAZ8H/AzsDHQcyrgtv1g/XvAwqAPGAt8HSwXIHGwe2XgWeA/wBrsC/6/aLiORGYA/wOPAt8XdJ7D9Z9LfrvWcLjDYNtXxC8v1+B26MebwuMA1YHf8ungWpRjytwJTAPWBQsewJLTH8AU4Ajo9avGuznBcF7mwLsDYwKXmtdsF+6B+ufgn2+VgP/A5oX++zeAkwHNgDbEPV5DmKfHMTxC/BosHxxsK21waUDUZ/JYJ2DgP8CvwXPvS3s/9VMuIQegF8q+Ifb/B+rPvAd8ETU448Bw4BdsF+gHwH/Dh5rG3xZnYAdVdYDmgaPvQ8MAGoBuwETgcuCx/76pwSOCr5UJLi/M7AeSxBVgi+SO4FqwL7AQqBTsO7dwCbg9GDdmsXe23bYl/IxJbzvC4Gfg9tHA/nAo1hS6Bh8YR0Qxz6IPPeB4Lk1gTrAmcH2awNvAx9Ebfsrin2xs2WiWBns322A14GhwWN1gy++M4LHrg32QWmJYhlwYYy/f8Ng2y8EsbfAvnQPDB5vA7QPttUQmA1cVyzu/wb7JpI8zwv2wTZAvyCGGsFjN2GfsQMACbZXp/g+CO63ApYD7bAEcwH2ea0e9dn9Fks0NaOWRT7P44Bewe3tgfbF3vM2UdvqTdFnsjaWFPsBNYL77cL+X82ES+gB+KWCfzj7x1qL/bpT4Atgp+Axwb4wo3/NdqDol+MA4LESXnP34Msm+sjjHGBkcDv6n1KwX3hHBfcvBb4MbrcDFhd77b8DLwW37wZGxXhv9YP31LSExzoDm4LbR2Nf9rWiHn8L+Ecc++BoYGPki7CUOFoCq6Luf0XZiWJQ1GNdgO+D2+cD46IeEyzRlpYoNhEc5ZXyeORLs37UsolAj1LWvw54v1jcx5bxGVsFtAhuzwG6lrJe8UTxHPDPYuvMATpGfXYvKuHzHEkUo4B7gLqlvOfSEsU5wNRE/t9l68XbB9Pb6ar6uYh0BN7AfrWuBnbFfhVPEZHIuoL9ugP7JTe8hNfbB9gW+DnqeVWwL7TNqKqKyFDsn3MUcC7WXBJ5nb1EZHXUU6pizUkRW7xmlFVAIbAn8H2xx/bEmln+WldV10Xd/xE7qilrHwCsUNW8vx4U2Q47CumMHSEB1BaRqqpaECPeaMuibv+J/SImiOmv9xzsv9wYr7MSe68V2p6I7I8daeVg+2Eb7Cgv2mZ/AxG5Ebg4iFWBHbDPFNhnZkEc8YD9/S8QkaujllULXrfEbRdzMXAv8L2ILALuUdWP49hueWJ05eCd2RlAVb/Gfs0+HCz6FWsGOkhVdwouO6p1fIP9k+5XwkstwY4o6kY9bwdVPaiUTQ8BzhKRfbCjiHejXmdR1GvspKq1VbVLdNgx3s86rPnh7BIe7oYdPUXsLCK1ou43AH6KYx+UFEM/rGmlnarugDWvgSWYmDHH4WfsSMle0LJX/dJX53OsGayinsOSbJPgvdxG0fuI+Ov9iMiRwM3Y/t1ZVXfCmicjzyntM1OSJcB9xf7+26nqkJK2XZyqzlPVc7CmzweAd4K/cVn7fwnWzOkqmSeKzPE4cIKItFDVQqzt+jER2Q1AROqJSKdg3cHAhSJynIhUCR5rqqo/YyONHhGRHYLH9guOWLagqlOxL+RBwAhVjRxBTATWiMgtIlJTRKqKyMEicmg53s+t2K/Sa0SktojsLCL9seaje4qte4+IVAu+7E4B3o5jH5SkNpZcVovILsBdxR7/hYp/Ef0HOERETg9G+lwJ7BFj/buAw0TkIRHZI4i/sYi8JiI7xbG92lifyFoRaQpcEcf6+VhH/jYicid2RBExCPiniDQR01xE6gSPFd8vLwCXi0i7YN1aInKyiMQ1WktEzhORXYO/YeQzVRjEVkjpf4OPgT1F5DoRqR58btrFs00XmyeKDKGqK4BXsQ5ksFEl84HxIvIH9gv1gGDdiVin8GPYr8avseYCsLb0asAsrAnoHWI3gbwBHB9cR2IpwL6wW2IjniLJZMdyvJ8xQCes8/dnrEmpFXCEqs6LWnVZEOdPWOfx5aoaaa4qdR+U4nGsY/hXYDzwabHHn8COoFaJyJPxvpfg/fyKHSE9iDUrNcNG9mwoZf0FWFJsCMwUkd+xI7bJWL9UWW7EmgPXYF/cb5ax/gjs/c7F9nUemzcPPYr1/3yGJaDB2L4C63N6RURWi0g3VZ2M9Vk9jf1t5mN9CfHqjL3ntdg+76Gq61X1T2z02dhgW+2jn6Sqa7ABGqdin4t5wDHl2K4rRWTEinNpJziT9zVVjdWEk5JEpAo2PLenqo4MOx7nYvEjCueSREQ6ichOIlKdoj6D8SGH5VyZEpYoRORFEVkuIjNKeVxE5EkRmR+UJmidqFicSxEdsFE5v2LNI6er6vpwQ3KubAlrehKRo7Bx/q+q6sElPN4FuBoba94OO1nMO56ccy7FJOyIQlVHYafRl6YrlkRUVccDO4lIPOPGnXPOJVGYJ9zVY/NRFbnBsp+LrygifYA+ALVq1WrTtGnTpATonHPlVVAAmzYlfjuLFsGff5a9XgN+ZCdWM538X1V114psKy3OzFbVgcBAgJycHJ08eXLIETnn3JYKC6FOHVi9uux1K0OXLjBgQAkPRLoURKj16nNUWbmcnR69+8eKbifMRLEUO+U+on6wzDnnEurjj+Gyy+zXf2VStSRx9tlw+umV+9olOeIIqF98cPjSpdD3CujeHXr2hNuCcy0fvbvC2wkzUQwDrgrqBbUDfg/ODHbOuUpx330wbdqWy2fOhJ9+gj59QIoXNtlKVavCdddBkyaV+7plUoVBg+DGG63t6+STK+2lE5YoRGQIVqGzblD87C6s4Byq+jxWlK4Ldtbmn9iZws45V2Fr1sCQIbBxo92/+27YYQfYffct1/3b3+D55ys/UYRiwQK49FIYORKOOQZeeAH2i7c0V9kSliiCol6xHo9MnOKcc3EpLIQ5cyA/v+THhw2DO+7YfNm998KVmf5N8913MGUKDBwIl1xS6dkvLTqznXMO7HvwirLKGwJz58LOO0OVKrDLLomPKxQzZsA338D551uHyMKF1pOeAJ4onHNp46OP7HroUNimlG+vPfYIoX8gmTZuhH/9yy677w7dukGNGglLEuCJwjmXglThhhvsR3K06dPtunv35MeUEiZMgIsvtt74886Dxx6zJJFgniiccylh2DCYFxSQ37gRHn8c9txz847ounXhnJi9nxls6VI48kjbIR9/XKmjmsriicI5VyE//wy5sSZzLaczztjyvIbHHsvio4eIuXNh//2hXj1480047jgbypVEniiccxXSujUsW1b2euXxz3/Ctdfa7SpVoFat2OtntNWr4eab7dyIr76Co46yMb0h8EThXAaaPt2G08dTC6ii8vLgrLOgd+/Keb2qVe27cLvtKuf10tqwYTa8a9kyuOkmOLQ8swhXPk8UziXQG2/AK68kf7u//AK//QYXXFDyyWaVoUoVuOiiDB9hFIZLLoHBg+GQQ+DDDyEnJ+yIPFG49LRxo51Vuyae2aND9MorVirikEOSu92aNaFzZ3jqKahdO7nbdhUQVcSPnBzYZx+45RaoVi3cuAKeKFzamTkTRo8uastOdT17wmuvhR2FS1lLlsDll0OPHtCrl91OMZ4oXFpZvBgOjpovcezY0Jtvy1TaiWEuyxUWWo3wW26x4V4hdVTHwz/CLmWdfTa8807Jj911F5x0ErRtmyFF3Vx2mTfP+iJGjYLjj7faJI0ahR1VqTxRuJTw3HNWliHapElw4IGWMKLVrAl9+yZ9KLlzlWfWLBua9uKLNmwsxX/tiEY6UdKEz3CXeX76yc4l2mknaNly88f69t0yUTiXlqZNg2+/taFoAKtWWeXCJBGRKapaoSFUfkThQrVpU1E10G7dSpnW0bl0tmED9O8P999vNUm6d7f6TElMElurStgBuOy0aZMNcX3lFTu3CGwop3MZZdw4aNXKEsW558LUqUkp4lfZ/IjCJd3w4XDqqTboI2Ly5JQZMu5c5Vi6FDp2tLrnw4fb6Is05YnCJcV//mM/qlStKkFhIdx2m9XyqV8f2rQJO0LnKsns2TYKo149eOstK+KX5mc9eqJwCbN+PTz8sJ09/fnnNlvjccdZp/Xxx1sBuCre+OkyxapV0K8fvPSSDXs98kibeS4DeKJwCTN2LNx5pzUpVa1qReo+/TTsqJxLgPfftyF6K1bA3/+e+meBlpMnClep1qwpKlszfrxdf/EFHHFEeDE5l1AXXWRHES1bWhtr69ZhR1TpPFG4SvPgg1aNoLgDD0x+LM4lVHQRv/btrYTujTfCttuGG1eCeKJwleaHH6xz+t57i5bttVdC53x3Lvl+/BEuu8yGu55/PvTpE3ZECeeJwlXIJ5/YJdro0TbpzA03hBOTcwlVWGi1Zm691Y4osqhkgCcKV26LF0OXLtZBXbze0lFHhROTcwk1Z44V8RszBk480UoINGwYdlRJ44nClYuqldoA+0E1ZEi48TiXFHPm2EQoL79szU0pXsSvsnmicOXSvz9MmGB9Dy+9FHY0ziXQ1KlWxO/CC+G002DhQjsJKAv56U4ubqNH23kRYOdDpGHJGufKlpdnZQMOPRTuvtvuQ9YmCfAjCleKefPgscds4q2IadPsun//5M8B7VxSjB0LF19sTU0XXgiPPOK/iPBE4QLz51uTUsSHH8Lbb8Puu2/eHHvYYTbow7mMs3SplQ+oVw9GjLBOawd4oshaq1fbWdQRF15oAzqi7byzTSrk9ZhcRps1C5o1swTx7ruWLLbfPuyoUooniiy0fLlVbN20afPlRx9tU/dG1K3rScJlsN9+s5N+XnkFvv7axnafemrYUaUkTxRZ5P77bfj3xo2WJPr0gbZtix4/8kirROBcxnv3XbjySli5Em6/ffN/BLcFTxRZZORIWLvWTparUQPuucfmVHEuq/TubUcRrVvb8L3iE7W7LXiiyBCTJxeNSipNbi40bmz/I85llegifocdZpUq+/WDbfwrMB4J3Usi0hl4AqgKDFLV+4s93gB4BdgpWOdWVR2eyJgyVadO1uRalgyZR8W5+C1aZO2s550HF1yQFUX8KlvCEoWIVAWeAU4AcoFJIjJMVWdFrXYH8JaqPicizYDhQMNExZSpxo2zJNGtm80oF4s3NbmsUVAAzzxjEwlVqQI9e4YdUdpK5BFFW2C+qi4EEJGhQFcgOlEoECkrtyPwUwLjyUj/+Ac8/bTdvugi2HvvcONxLiXMnm0nzo0bByedBM8/Dw0ahB1V2kpkoqgHLIm6nwu0K7bO3cBnInI1UAs4vqQXEpE+QB+ABv7HBuCNN+ykuJEjrWP65ptt+LdzDjuDdM4c+L//syOJLCviV9nCHiV/DvCyqtYHugD/JyJbxKSqA1U1R1Vzdt1116QHmUrWr4cXX7QRfR9/bJMCXXMNPPCAzU3tXNaaMsX+OcDOh1i0yPolPElstUQeUSwFohtC6gfLol0MdAZQ1XEiUgOoCyxPYFxpp7AQvv/emlxHjoRrr7XlF10EgweHG5tzoVu/3sZ6P/ywtb2ee64dZhefLMVVWCITxSSgiYg0whJED+DcYussBo4DXhaRA4EawIoExpSWnnsOrrpq82WTJ0OrVuHE41zKGDXKJhSaN8/6JB5+2Iv4JUDCEoWq5ovIVcAIbOjri6o6U0TuBSar6jCgH/CCiFyPdWz3Vo0MeM5uXbva/wDYDyaAt96ywRu77AJt2oQXm3MpYelSOO44O4r4/HO77RIioedRBOdEDC+27M6o27OAwxMZQzpZudL6G9ats/moDzgAjj3WHtt//6yaote50n33ndW5r1cP3n/fRnHUqhV2VBnNT0tMIe+9Z6OZmjSBgw6ySYL+9rewo3IuRfz6K1x/Pbz2WlERv1NOCTuqrOCJIoVMnGjXX38Ne+4ZbizOpQxVmxzlqqtg1Sq46y5oV3ykvUskTxQpJDIfhCcJ56JccIGdD5GTA1984dMrhsATRchWrYLmza1/Ii8P9t037IicSwHRRfw6drR/kuuu8yJ+IfG9HrJly6yq6ymnWEHLLl3Cjsi5kC1cCJdeaifLXXihDXt1oQr7zOysN3u2XZ93Hjz4oM0y51xWKiiAxx+3pqVJk3x6xRTiRxRJpgrffmtDYKGooN8++4QXk3OhmzXLSg1MmAAnn2xF/OrXDzsqF/BEkWTjxsHhxc4cOfhgaN8+nHicSwmLFsGCBTY+vEcPr8+UYjxRJNHw4fZjCexI4oAD7Hbk2rmsMmmSHV5feqn9YyxcCLVrhx2VK4EniiT45Rcr5Pfmm9Zh3bu3TbK17bZhR+ZcCP78084mfewxa3Pt1cvqM3mSSFneW5RAqlb1+MADrdLAvffaD6ibb/Yk4bLUV1/ZUNdHHrEjialTvYhfGvAjigSZNw8uu8zKgh95JAwcCE2bhh2VcyHKzYUTTrCjiC+/9Jm20ogfUVSyTZvg3/+2EX7ffAMDBtiPKE8SLmtNm2bX9evbtIzTp3uSSDOeKCrRxIlWZeC22+wEutmzrS/Ch4O7rLRihU0i1LKlFTADO6N0u+3CjcuVm3+FVYK1a626QPv2Vorjgw/gnXe8ZpPLUqowZAg0a2b/CPfcAx06hB2V2wpx9VGISDWggarOT3A8aeWTT6wP4s03YckSuOIKa3byGRhdVuvVC15/3Sq8Dh5sNfNdWiszUYjIycCjQDWgkYi0BO5S1ayeKWHVKjuKFoEWLewH1GGHhR2VcyEpLLR/BhHrf2jTxmbhqlo17MhcJYin6eleoB2wGkBVvwUaJzKoVJaXB2vWwK232v0LLrARfp4kXNaaP9+mIX3pJbt/8cU2wZAniYwRT6LYpKqriy3Lynmt586FHXe0pqWBA23ZM8+EG5NzocnPh4cftiF+U6dCtWphR+QSJJ4+itki0g2oIiKNgGuA8YkNKzUtWQIbN8KVV0KjRnZ07QM4XFaaMcNKgE+eDF27wrPPwl57hR2VS5B4EsVVwJ1AIfAeMAK4LZFBpapRo+z6vPO8iJ/LcosXw48/wtCh0K2bF/HLcPEkik6qegtwS2SBiJyBJY2ssHYtfP65jXICO5pwLutMmGAnz/XpYyM5Fi6E7bcPOyqXBPH0UdxRwrLbKzuQVPbss/C3v1mxyyOOgN13Dzsi55Jo3Tq44QY7F+LBB2HDBlvuSSJrlHpEISKdgM5APRF5NOqhHbBmqKwxdKhdT5sGDRuGGopzyfXll1a8b+FCO1Ho/vuhevWwo3JJFqvpaTkwA8gDZkYtXwPcmsigUsno0UUDOpo3Dzsa55IoNxc6dbK21q+/hqOOCjsiF5JSE4WqTgWmisjrqpqXxJhSwo8/wkMP2aAOsKl8ncsKU6dCq1ZWxO+jj6BjR6hZM+yoXIji6aOoJyJDRWS6iMyNXBIeWYhWroS77rJzJBYtspPp+vQJOyrnEuyXX6B7d2jduqiIX+fOniRcXKOeXgb6Aw8DJwEXkuEn3N1zD7zyCtSqBT/84P8nLsOpWm2ma6+1IX79+3upAbeZeI4otlPVEQCqukBV78ASRsZ66ikbFv7jj54kXBY491wr5HfAATYF4+23+xSMbjPxHFFsEJEqwAIRuRxYCmTc5Lb5+VauZskSu3/88VCnTrgxOZcw0UX8TjzRhr5eeaXXZ3IliueI4nqgFla643DgUuCiRAYVhh9+sDOv69SB3r1tSl/nMtLcuVbh9cUX7f6FF3qlVxdTmUcUqjohuLkG6AUgIvUSGVQYZsyw66uusoqwzmWc/Hx49FEbqVGjhrerurjFPKIQkUNF5HQRqRvcP0hEXgUmxHpeuvn5ZxvdBF7DyWWo6dPtw33LLXDSSTBrlvVNOBeHWGdm/xs4E5gG3CEiHwN9gQeAy5MTXuJNnWqjASN22im8WJxLmNxc64B7+20480wv4ufKJVbTU1eghaquF5FdgCXAIaq6MN4XF5HOwBNAVWCQqt5fwjrdgLuxIbfTVDVpP3O+/BLOOcdu33knHHmk13FyGeR//7MjicsvLyriV6tW2FG5NBQrUeSp6noAVf1NROaWM0lUBZ4BTgBygUkiMkxVZ0Wt0wT4O3C4qq4Skd0q9C4q4JNP4LbbYPlyuPpqm7HOm2xdRli71oa4PvUU7LefdVZXr+5JwlVYrESxr4hESokLNl/2X6XFVfWMMl67LTA/klxEZCh2lDIrap1LgWdUdVXwmsvLGX+5bNhgc1v/+Se88II103bqBI895gM+XIb47DMrI7B4sQ13/de/vIif22qxEsWZxe4/Xc7Xroc1V0XkYnNvR9sfQETGYs1Td6vqp8VfSET6AH0AGjRoUM4winzxhf24iujUCT7dYmvOpaklS+Dkk+0oYtQoq4nvXCWIVRTwiyRtvwlwNFAfGCUihxSfo1tVBwIDAXJycipcPuStt+x69GjYf3/YZZeKvpJzKWTKFJuXd++9Yfhw62yrUSPsqFwGieeEu4paCuwddb9+sCxaLjBMVTep6iJgLpY4Kt0vv1j9JoDDD4fddoNt4jkv3blUtWwZnH025OQUFfE74QRPEq7SJTJRTAKaiEgjEakG9ACGFVvnA+xoguBcjf2BuDvMy2PsWLs++2wfGejSnKr96mnWzMqA/+tfXsTPJVTcv6lFpLqqboh3fVXNF5GrgBFY/8OLqjpTRO4FJqvqsOCxE0VkFlAA3KSqK8v3FuIzZ45dX3ttIl7duSTq0cPaUQ8/HAYNgqZNw47IZbgyE4WItAUGAzsCDUSkBXCJql5d1nNVdTgwvNiyO6NuK3BDcEmoMWPs+qCDEr0l5xIguohfly7WD9G3L1RJZKOAcyaeT9mTwCnASgBVnQYck8igKtusWdbHB37mtUtD339v05AOHmz3L7jAipJ5knBJEs8nrYqq/lhsWUEigkmU1cEYqkcfDTcO58pl0ybrf2jRwn7tbL992BG5LBVPH8WSoPlJg7Otr8ZGJ6WF996DBx+0297s5NLGt9/aST/ffgtnnWVnWe+xR9hRuSwVT6K4Amt+agD8AnweLEsLt99ulWGPPRYOOSTsaJyL07Jldnn3XTijrCIIziVWPIkiX1V7JDySBNm0CapVs7OynUtpY8ZYEb++faFzZ1iwALbbLuyonIurj2KSiAwXkQtEJO2mQK1WzWZ6dC5lrVljndNHHgmPP25FycCThEsZZSYKVd0P6A+0Ab4TkQ9EJOWPMNauhXr1YPZsPwPbpbARI+Dgg+HZZ+0kn2++8SJ+LuXENb5OVf+nqtcArYE/gNcTGlUl+OEH+OknOOUUm9TLuZSzZIl9QLfbzpqdHn/cRza5lFRmohCR7UWkp4h8BEwEVgApXy9g1Sq7PuMMaNUq3Fic+4sqTJxot/fe2yZGmTrVS3C4lBbPEcUMoD3woKo2VtV+qpryc2b/+qtd77dfuHE495eff7ZpSNu1Kyrid/zxXsTPpbx4Wu/3VdXChEdSyV580a69lLgLnSq8/DLccAPk5cEDD1idJufSRKmJQkQeUdV+wLsissUcEHHMcBeqatWs6ddPsnOh69YN3nnHRjUNGmSToTiXRmIdUbwZXJd3ZrvQvf++nZHdsqWXFHchKSiwD1+VKnDqqXbG52WXeX0ml5ZK/dSqatDjxoGq+kX0BTgwOeFVzMcf2/Wll4Ybh8tSs2fb0UOkiN/558MVV3iScGkrnk/uRSUsu7iyA6lM338Pe+5pJ7g6lzSbNkH//nYoO2cO7Lhj2BE5Vyli9VF0x2alayQi70U9VBtYXfKzwqcK//ufDyRxSTZ1KvTubSU4uneHJ5+0+XadywCx+igmYnNQ1AeeiVq+BpiayKC2RkFQAN2bnVxS/fKLjcn+4APo2jXsaJyrVKUmClVdBCzCqsWmjXnz7HqvvcKNw2WBUaPgu+/gyiutiN/8+VCzZthROVfpSu2jEJGvg+tVIvJb1GWViPyWvBDLZ9Ysu27UKNw4XAb74w/rAOvY0ZqYIkX8PEm4DBWrMzsy3WldYNeoS+R+Svrvf+26RYtw43AZavhwOzlnwAA7gc6L+LksEGt4bORs7L2BqqpaAHQALgNqJSG2clu40P5/wc/IdgmwZIn1P+y4o42YeOQRqJWS/wrOVap4hsd+gE2Duh/wEtAEeCOhUVVQv352/dRTPuDEVRJVGD/ebu+9N3z2mR1FtGsXblzOJVE8iaJQVTcBZwBPqer1QL3EhlV++fk24ATg4pQ+y8OljZ9+gtNPhw4dior4HXOM1YdxLovEkyjyReRsoBcQnPPMtokLqWJWrrTrTp28T9FtJVWrydSsmR3yLkKNAAAbAElEQVRBPPywF/FzWS2e6rEXAX2xMuMLRaQRMCSxYZXfsmV2fdZZ4cbhMsBZZ1mxsI4dLWE0bhx2RM6FqsxEoaozROQaoLGINAXmq+p9iQ+tfCJlxffYI9w4XJqKLuJ3+uk20fqll3p9JueIb4a7I4H5wGDgRWCuiKTccXjk//mUU8KNw6WhGTOsaSlSxK9XL6/06lyUeP4THgO6qOrhqnoYcDLwRGLDqpgddgg7ApdWNm6Ee+6B1q1hwQLYeeewI3IuJcXTR1FNVWdF7qjqbBHxYR8uvU2ZYkX8ZsyAc8+Fxx+HXVP2PFLnQhVPovhGRJ4HXgvu9yQFiwLOnm1Vnp2Ly8qVsHo1fPSRt1c6V4Z4EsXlwDXAzcH90cBTCYuogqZNg/Xrw47CpbSRI62I3zXXWGf1vHlej965OMRMFCJyCLAf8L6qPpickCpm2TKbVMy5Lfz+O9x8MwwcCE2bWkd19eqeJJyLU6zqsbdh5Tt6Av8VkZJmuksJY8bY9b77hhuHS0EffWQnzg0aBDfeaH0TXsTPuXKJdUTRE2iuqutEZFdgODY8NuVMDXpMvHSH28ySJXDmmXYU8cEHcOihYUfkXFqKNTx2g6quA1DVFWWsG6r8fLtu3jzcOFwKiMyFC0VF/CZP9iTh3FaI9eW/r4i8F1zeB/aLuv9ejOf9RUQ6i8gcEZkvIrfGWO9MEVERySnvGwBYu9auq1atyLNdxsjNhdNOs5PnIkX8jj7ai/g5t5ViNT2dWez+0+V5YRGpis21fQKQC0wSkWHR52QE69UGrgUmlOf1o731ll3790GWKiyEF16Am26yw8tHH4Ujjgg7KucyRqw5s7/Yytdui9WFWgggIkOBrsCsYuv9E3gAuKmiG9p5Z9hzT08UWevMM60P4thjLWH4qAbnKlUi+x3qAUui7udSbB4LEWkN7K2q/4n1QiLSR0Qmi8jkFStWbPaYKoweDYccUklRu/SQn29HEmCJ4oUX4PPPPUk4lwChdVCLSBXgUaBfWeuq6kBVzVHVnF2LlVlYvNiuCwoSEKRLTdOn22RCL7xg9887Dy65xKq/OucqXdyJQkTKO/h8KTbfdkT9YFlEbeBg4CsR+QFoDwwrb4f266/bda9e5YzOpZ8NG+Cuu6BNG/jxR6/N5FySxFNmvK2IfAfMC+63EJF4SnhMApqISKOgiGAPYFjkQVX9XVXrqmpDVW0IjAdOU9XJ5XkDkURxZvGud5dZJk2yKq/33gvnnGPFvc44I+yonMsK8RxRPAmcAqwEUNVpwDFlPUlV84GrgBHAbOAtVZ0pIveKyGkVD7lIYSHMmgV168L221fGK7qUtWqVjYMePhxefRXq1Ak7IueyRjxFAauo6o+yeftvXD0CqjocO6M7etmdpax7dDyvGe233+y6Q4fyPtOlhS+/tCJ+115rRfzmzvXyG86FIJ4jiiUi0hZQEakqItcBcxMcV1xGj7brjh3DjcNVstWrbRrS446DAQOsbwI8STgXkngSxRXADUAD4Bes0/mKRAYVr1Gj7PqYMhvCXNr48EMr4vfii1bx1Yv4ORe6MpueVHU51hGdciKjIw88MNw4XCVZvBjOPtv+oMOGQU6FKro45ypZmYlCRF4AtPhyVe2TkIjitHo1rFtnA2Fq1gwzErdVVK1O/JFHQoMGdtJc+/Z+mr1zKSSepqfPgS+Cy1hgN2BDIoOKR+QE7c6dw43DbYXFi+Hkk+Goo4qK+B11lCcJ51JMPE1Pb0bfF5H/A8YkLKI4rVtn182ahRuHq4DCQnj+ebjlFjuiePJJL+LnXAqLZ3hscY2A3Ss7kPJauNCuvWpDGjrjDOu0PuEEm560YcOwI3LOxRBPH8UqivooqgC/AaXOLZEs2wSRN20abhwuTvn5UKWKXbp3h65doXdvz/TOpYGYiULsLLsWFNVoKlTVLTq2wzBnjl2nRjQupmnT4KKL7NyIyy+3EhzOubQRszM7SArDVbUguKTM13JkaH2DBuHG4WLIy4M77rBhrrm5sMceYUfknKuAeEY9fSsirRIeSTlFpkWuUSPcOFwpJk6EVq3gvvugZ08r4nf66WFH5ZyrgFKbnkRkm6CwXytsGtMFwDpAsION1kmKsUTLl9v1dtuFGYUr1R9/wPr18Omn0KlT2NE457ZCrD6KiUBroFIqvSZChw5QtWrYUbi/fPYZzJwJ118Pxx9vHUlefsO5tBer6UkAVHVBSZckxVeqkSNh06awo3CAlQC/8EI7chg82Iv4OZdhYh1R7CoiN5T2oKo+moB44uZzUKSI996DK6+0U+X//ne4805PEM5lmFiJoiqwPcGRRaopLISDDgo7iiy3eDH06AEHH2wTCrVKuTEPzrlKECtR/Kyq9yYtknLKz4dttw07iiykavXdO3a0sclffgnt2vkfw7kMVmYfRaratKno7GyXJD/+CCedBEcfXVTE74gjPEk4l+FiJYrjkhZFBfgRRRIVFsLTT1tb35gx8NRTVhbcOZcVSv1Nrqq/JTOQ8igosCMKHxqbJKefDh99ZKOaBgyAffYJOyLnXBKlZePNH3/Y9Z9/hhtHRotk4ipVrDbTWWdBr15exM+5LBRPCY+Us3GjXTduHG4cGeubb6BtW5szAixRnH++JwnnslRaJorIiXbeR1HJ1q+3cyHatoVly2DvvcOOyDmXAtKy6SlyROEzZlai8ePhggtg7lwrCf7ww7DzzmFH5ZxLAWmZKCZMsGsfHluJ1q2zQ7X//tfqNDnnXCAtv2qXLbPrNm3CjSPtffqpFfHr1w+OOw6+/94P05xzW0jLPor8fLuuVy/cONLWypXWzHTSSfDKK96W55yLKS0ThXdmV5AqvPMONGsGb7xhs89NmuQJwjkXU1o2PUWOKLyPopwWL4Zzz4XmzW3uiBYtwo7IOZcG0vaIQsTPzI6LqhXuAzuj+quvbISTJwnnXJzSMlG88YZ9/7kyLFoEJ55oHdWRIn6HHeaHYs65cknLRLF8uVWWcKUoKIAnnrB5IiZMgOee8yJ+zrkKS8uflmvXwi23hB1FCuvaFf7zH+jSxcpw+BnWzrmtkHaJIjKSs6Ag3DhSTnQRv169rD7Tued6fSbn3FZLaAOOiHQWkTkiMl9Ebi3h8RtEZJaITBeRL0SkzPrVkYqxBx5Y+fGmrcmTISfHmpgAuneHnj09STjnKkXCEoWIVAWeAU4CmgHniEizYqtNBXJUtTnwDvBgWa8b6cQ+4IDKjDZNrV9vbXDt2sGKFT5PhHMuIRJ5RNEWmK+qC1V1IzAU6Bq9gqqOVNXIrBLjgfplvWik6Snr69WNG2dDXB980Ir4zZoFp5wSdlTOuQyUyD6KesCSqPu5QLsY618MfFLSAyLSB+gDsP32dlBSt26lxJi+1q+3KUo//9yGvzrnXIKkRGe2iJwH5AAdS3pcVQcCAwH22CNHN22C3XZLYoCpYvhwK+J3001w7LEwe7bXMXHOJVwim56WAtHjMusHyzYjIscDtwOnqeqGeF44674bf/0VzjsPTj4ZXn+9qP0t63aEcy4MiUwUk4AmItJIRKoBPYBh0SuISCtgAJYklsfzomvXFn1PZjxVGDrUhni99RbcdRdMnOhF/JxzSZWwpidVzReRq4ARQFXgRVWdKSL3ApNVdRjwELA98LbYUM7FqnparNfNy8uicygWL7Zy4C1awODBcMghYUfknMtComlWNEkkR/feezKLF4cdSYKowhdfFM0yN348HHqoV0B0zm0VEZmiqjkVeW5aVkw6//ywI0iQBQtsBNMJJxQV8Wvf3pOEcy5UaZkoMu57s6AAHn3UmpamTIEBA7yIn3MuZaTE8NjyyrjKFKeeCp98YifMPfcc1C/zvEPnnEuatEwUGVFifONGmxeiShXo3dsK+fXokYFZ0DmX7tLyKzftv0snToQ2beDZZ+1+t25W7TXt35hzLhOlZaJI2yOKP/+Efv2gQwdYtQr22y/siJxzrkxp2fSUlj+8x4yxcyIWLoTLLoMHHoAddww7KuecK5MnimSJTCw0ciQcfXTY0TjnXNzSMlEUFoYdQZw++sgK9918MxxzjJUC3yYtd7lzLoulZWt/06ZhR1CGFStsGtLTToMhQ4qKU3mScM6lobRMFClbFFAV3njDivi98w7cey9MmOBF/JxzaS0tf+Km7PloixfDhRdCq1ZWxO+gg8KOyDnntlpaHlGk1DQMhYUwYoTd3mcfGD0axo71JOGcyxieKLbGvHk201znzjBqlC1r2zYDi1E557JZWiaK0PuE8/PhoYegeXP49ltrZvIifs65DBX2V26FhH5Eccop1tzUtauV4dhrr5ADci41bdq0idzcXPLy8sIOJWvUqFGD+vXrs20lflGmZaIIpYTHhg2WoapUgUsugYsugrPPTtOz/5xLjtzcXGrXrk3Dhg0R/19JOFVl5cqV5Obm0qhRo0p73bRsetp55yRvcPx4aN0annnG7p91lhXy8w++czHl5eVRp04dTxJJIiLUqVOn0o/g0jJRJK2veN06uP56OOwwWLMGmjRJ0oadyxyeJJIrEfs7LZuekpIoRo+2In6LFkHfvvDvf8MOOyRhw845l1rS8ogiKX0U+fnWJ/H119bk5EnCubT1wQcfICJ8//33fy376quvOOWUUzZbr3fv3rzzzjuAdcTfeuutNGnShNatW9OhQwc++eSTrY7l3//+N40bN+aAAw5gROQcrGK++OILWrduTcuWLTniiCOYP38+AIsXL+aYY46hVatWNG/enOHDh291PPHwRBHtgw/syAGsiN/MmXDUUQnamHMuWYYMGcIRRxzBkCFD4n7OP/7xD37++WdmzJjBN998wwcffMCaNWu2Ko5Zs2YxdOhQZs6cyaeffkrfvn0pKCjYYr0rrriC119/nW+//ZZzzz2X/v37A9C/f3+6devG1KlTGTp0KH379t2qeOKVlk1PlZ4ofvkFrr4a3n7bOq379bP6TKGfsOFc5rjuOjvtqDK1bAmPPx57nbVr1zJmzBhGjhzJqaeeyj333FPm6/7555+88MILLFq0iOrVqwOw++67061bt62K98MPP6RHjx5Ur16dRo0a0bhxYyZOnEiHDh02W09E+OOPPwD4/fff2SsYgl/a8kRLy2/CSuujUIXXXrNP8Nq1cN99cNNNKXCihnOusnz44Yd07tyZ/fffnzp16jBlyhTatGkT8znz58+nQYMG7BBHk/P111/PyJEjt1jeo0cPbr311s2WLV26lPbt2/91v379+ixdunSL5w4aNIguXbpQs2ZNdthhB8aPHw/A3XffzYknnshTTz3FunXr+Pzzz8uMrzKkZaKotCOKxYvtnIicHDu7OuXrlzuXvsr65Z8oQ4YM4dprrwXsy3vIkCG0adOm1NFB5R019Nhjj211jCW95vDhw2nXrh0PPfQQN9xwA4MGDWLIkCH07t2bfv36MW7cOHr16sWMGTOokuCO2+xLFJEifiedZEX8xo61aq9en8m5jPPbb7/x5Zdf8t133yEiFBQUICI89NBD1KlTh1WrVm2xft26dWncuDGLFy/mjz/+KPOoojxHFPXq1WPJkiV/3c/NzaVevXqbrbNixQqmTZtGu3btAOjevTudO3cGYPDgwXz66acAdOjQgby8PH799Vd22223OPdIBalqWl2gjRYUaMXMmaN65JGqoPrVVxV8EedcvGbNmhXq9gcMGKB9+vTZbNlRRx2lX3/9tebl5WnDhg3/ivGHH37QBg0a6OrVq1VV9aabbtLevXvrhg0bVFV1+fLl+tZbb21VPDNmzNDmzZtrXl6eLly4UBs1aqT5+fmbrbNp0yatU6eOzpkzR1VVBw0apGeccYaqqnbu3FlfeuklVbV9u+eee2phYeEW2ylpvwOTtaLfuxV9YlgXaFPiHyCmTZtU779ftXp11Z12Un3pJdUSdq5zrnKFnSiOPvpo/eSTTzZb9sQTT+jll1+uqqpjxozRdu3aaYsWLTQnJ0c/++yzv9bbsGGD3nTTTbrffvvpQQcdpG3bttVPP/10q2Pq37+/7rvvvrr//vvr8OHD/1p+0kkn6dKlS1VV9b333tODDz5Ymzdvrh07dtQFCxaoqurMmTP1sMMO0+bNm2uLFi10xIgRJW6jshOF2PPTh0iOqk4u35M6dYLPPoMzzrBzIvbYIzHBOec2M3v2bA488MCww8g6Je13EZmiqjkVeb207KOIS16ejV6qWhX69LHLmWeGHZVzzqWdtDvhrkaNOFYaO9YGWEeK+J15picJ55yroLRLFDGtXQvXXGOTCOXlgR/yOhe6dGveTneJ2N+Zkyi+/hoOPhiefhquugpmzIATTgg7KueyWo0aNVi5cqUniyTRYD6KGnE1vcQvs/oottvOqr4efnjYkTjnsDOPc3NzWbFiRdihZI3IDHeVKe1GPdWsmaPr1wejnt57D77/Hm67ze4XFPiJc845V4KtGfWU0KYnEeksInNEZL6I3FrC49VF5M3g8Qki0jCuF162zGaZO/NMeP992LjRlnuScM65SpewRCEiVYFngJOAZsA5ItKs2GoXA6tUtTHwGPBAWa+7U8FK66T++GMrCf6//1mlV+eccwmRyCOKtsB8VV2oqhuBoUDXYut0BV4Jbr8DHCdlVOSql/+jdVpPmwa33uqVXp1zLsES2ZldD1gSdT8XaFfaOqqaLyK/A3WAX6NXEpE+QJ/g7gYZM2aGV3oFoC7F9lUW831RxPdFEd8XRQ6o6BPTYtSTqg4EBgKIyOSKdshkGt8XRXxfFPF9UcT3RRERKWftoyKJbHpaCuwddb9+sKzEdURkG2BHYGUCY3LOOVdOiUwUk4AmItJIRKoBPYBhxdYZBlwQ3D4L+FLTbbyuc85luIQ1PQV9DlcBI4CqwIuqOlNE7sXK3Q4DBgP/JyLzgd+wZFKWgYmKOQ35viji+6KI74sivi+KVHhfpN0Jd84555Irc2o9OeecSwhPFM4552JK2USRsPIfaSiOfXGDiMwSkeki8oWI7BNGnMlQ1r6IWu9MEVERydihkfHsCxHpFnw2ZorIG8mOMVni+B9pICIjRWRq8H/SJYw4E01EXhSR5SIyo5THRUSeDPbTdBFpHdcLV3QO1UResM7vBcC+QDVgGtCs2Dp9geeD2z2AN8OOO8R9cQywXXD7imzeF8F6tYFRwHggJ+y4Q/xcNAGmAjsH93cLO+4Q98VA4IrgdjPgh7DjTtC+OApoDcwo5fEuwCeAAO2BCfG8bqoeUSSk/EeaKnNfqOpIVf0zuDseO2clE8XzuQD4J1Y3LC+ZwSVZPPviUuAZVV0FoKrLkxxjssSzLxTYIbi9I/BTEuNLGlUdhY0gLU1X4FU144GdRGTPsl43VRNFSeU/6pW2jqrmA5HyH5kmnn0R7WLsF0MmKnNfBIfSe6vqf5IZWAji+VzsD+wvImNFZLyIdE5adMkVz764GzhPRHKB4cDVyQkt5ZT3+wRIkxIeLj4ich6QA3QMO5YwiEgV4FGgd8ihpIptsOano7GjzFEicoiqrg41qnCcA7ysqo+ISAfs/K2DVbUw7MDSQaoeUXj5jyLx7AtE5HjgduA0Vd2QpNiSrax9URs4GPhKRH7A2mCHZWiHdjyfi1xgmKpuUtVFwFwscWSaePbFxcBbAKo6DqiBFQzMNnF9nxSXqonCy38UKXNfiEgrYACWJDK1HRrK2Beq+ruq1lXVhqraEOuvOU1VK1wMLYXF8z/yAXY0gYjUxZqiFiYzyCSJZ18sBo4DEJEDsUSRjfOzDgPOD0Y/tQd+V9Wfy3pSSjY9aeLKf6SdOPfFQ8D2wNtBf/5iVT0ttKATJM59kRXi3BcjgBNFZBZQANykqhl31B3nvugHvCAi12Md270z8YeliAzBfhzUDfpj7gK2BVDV57H+mS7AfOBP4MK4XjcD95VzzrlKlKpNT84551KEJwrnnHMxeaJwzjkXkycK55xzMXmicM45F5MnCpdyRKRARL6NujSMsW7D0ipllnObXwXVR6cFJS8OqMBrXC4i5we3e4vIXlGPDRKRZpUc5yQRaRnHc64Tke22dtsue3micKlovaq2jLr8kKTt9lTVFlixyYfK+2RVfV5VXw3u9gb2inrsElWdVSlRFsX5LPHFeR3gicJVmCcKlxaCI4fRIvJNcDmshHUOEpGJwVHIdBFpEiw/L2r5ABGpWsbmRgGNg+ceF8xh8F1Q6796sPx+KZoD5OFg2d0icqOInIXV3Ho92GbN4EggJzjq+OvLPTjyeLqCcY4jqqCbiDwnIpPF5p64J1h2DZawRorIyGDZiSIyLtiPb4vI9mVsx2U5TxQuFdWManZ6P1i2HDhBVVsD3YEnS3je5cATqtoS+6LODco1dAcOD5YXAD3L2P6pwHciUgN4GeiuqodglQyuEJE6wN+Ag1S1OdA/+smq+g4wGfvl31JV10c9/G7w3IjuwNAKxtkZK9MRcbuq5gDNgY4i0lxVn8RKah+jqscEpTzuAI4P9uVk4IYytuOyXEqW8HBZb33wZRltW+DpoE2+AKtbVNw44HYRqQ+8p6rzROQ4oA0wKShvUhNLOiV5XUTWAz9gZagPABap6tzg8VeAK4GnsbkuBovIx8DH8b4xVV0hIguDOjvzgKbA2OB1yxNnNaxsS/R+6iYifbD/6z2xCXqmF3tu+2D52GA71bD95lypPFG4dHE98AvQAjsS3mJSIlV9Q0QmACcDw0XkMmwmr1dU9e9xbKNndAFBEdmlpJWC2kJtsSJzZwFXAceW470MBboB3wPvq6qKfWvHHScwBeufeAo4Q0QaATcCh6rqKhF5GSt8V5wA/1XVc8oRr8ty3vTk0sWOwM/B/AG9sOJvmxGRfYGFQXPLh1gTzBfAWSKyW7DOLhL/nOJzgIYi0ji43wv4OmjT31FVh2MJrEUJz12DlT0vyfvYTGPnYEmD8sYZFLT7B9BeRJpis7etA34Xkd2Bk0qJZTxweOQ9iUgtESnp6My5v3iicOniWeACEZmGNdesK2GdbsAMEfkWm5fi1WCk0R3AZyIyHfgv1ixTJlXNw6prvi0i3wGFwPPYl+7HweuNoeQ2/peB5yOd2cVedxUwG9hHVScGy8odZ9D38QhWFXYaNj/298AbWHNWxEDgUxEZqaorsBFZQ4LtjMP2p3Ol8uqxzjnnYvIjCuecczF5onDOOReTJwrnnHMxeaJwzjkXkycK55xzMXmicM45F5MnCuecczH9PzdBfG3ApUJqAAAAAElFTkSuQmCC\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "fpr, tpr, threshold = roc_curve(predictions_df[\"label\"],predictions_df[\"marg\"])\n", "roc_auc = auc(fpr, tpr)\n", "\n", "# method I: plt\n", "import matplotlib.pyplot as plt\n", "plt.title('Receiver Operating Characteristic')\n", "plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)\n", "plt.legend(loc = 'lower right')\n", "plt.plot([0, 1], [0, 1],'r--')\n", "plt.xlim([0, 1])\n", "plt.ylim([0, 1])\n", "plt.ylabel('True Positive Rate')\n", "plt.xlabel('False Positive Rate')\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [], "source": [ "%matplotlib notebook" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "application/javascript": [ "/* Put everything inside the global mpl namespace */\n", "window.mpl = {};\n", "\n", "\n", "mpl.get_websocket_type = function() {\n", " if (typeof(WebSocket) !== 'undefined') {\n", " return WebSocket;\n", " } else if (typeof(MozWebSocket) !== 'undefined') {\n", " return MozWebSocket;\n", " } else {\n", " alert('Your browser does not have WebSocket support.' +\n", " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", " 'Firefox 4 and 5 are also supported but you ' +\n", " 'have to enable WebSockets in about:config.');\n", " };\n", "}\n", "\n", "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n", " this.id = figure_id;\n", "\n", " this.ws = websocket;\n", "\n", " this.supports_binary = (this.ws.binaryType != undefined);\n", "\n", " if (!this.supports_binary) {\n", " var warnings = document.getElementById(\"mpl-warnings\");\n", " if (warnings) {\n", " warnings.style.display = 'block';\n", " warnings.textContent = (\n", " \"This browser does not support binary websocket messages. \" +\n", " \"Performance may be slow.\");\n", " }\n", " }\n", "\n", " this.imageObj = new Image();\n", "\n", " this.context = undefined;\n", " this.message = undefined;\n", " this.canvas = undefined;\n", " this.rubberband_canvas = undefined;\n", " this.rubberband_context = undefined;\n", " this.format_dropdown = undefined;\n", "\n", " this.image_mode = 'full';\n", "\n", " this.root = $('
');\n", " this._root_extra_style(this.root)\n", " this.root.attr('style', 'display: inline-block');\n", "\n", " $(parent_element).append(this.root);\n", "\n", " this._init_header(this);\n", " this._init_canvas(this);\n", " this._init_toolbar(this);\n", "\n", " var fig = this;\n", "\n", " this.waiting = false;\n", "\n", " this.ws.onopen = function () {\n", " fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n", " fig.send_message(\"send_image_mode\", {});\n", " if (mpl.ratio != 1) {\n", " fig.send_message(\"set_dpi_ratio\", {'dpi_ratio': mpl.ratio});\n", " }\n", " fig.send_message(\"refresh\", {});\n", " }\n", "\n", " this.imageObj.onload = function() {\n", " if (fig.image_mode == 'full') {\n", " // Full images could contain transparency (where diff images\n", " // almost always do), so we need to clear the canvas so that\n", " // there is no ghosting.\n", " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", " }\n", " fig.context.drawImage(fig.imageObj, 0, 0);\n", " };\n", "\n", " this.imageObj.onunload = function() {\n", " fig.ws.close();\n", " }\n", "\n", " this.ws.onmessage = this._make_on_message_function(this);\n", "\n", " this.ondownload = ondownload;\n", "}\n", "\n", "mpl.figure.prototype._init_header = function() {\n", " var titlebar = $(\n", " '
');\n", " var titletext = $(\n", " '
');\n", " titlebar.append(titletext)\n", " this.root.append(titlebar);\n", " this.header = titletext[0];\n", "}\n", "\n", "\n", "\n", "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n", "\n", "}\n", "\n", "\n", "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n", "\n", "}\n", "\n", "mpl.figure.prototype._init_canvas = function() {\n", " var fig = this;\n", "\n", " var canvas_div = $('
');\n", "\n", " canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n", "\n", " function canvas_keyboard_event(event) {\n", " return fig.key_event(event, event['data']);\n", " }\n", "\n", " canvas_div.keydown('key_press', canvas_keyboard_event);\n", " canvas_div.keyup('key_release', canvas_keyboard_event);\n", " this.canvas_div = canvas_div\n", " this._canvas_extra_style(canvas_div)\n", " this.root.append(canvas_div);\n", "\n", " var canvas = $('');\n", " canvas.addClass('mpl-canvas');\n", " canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n", "\n", " this.canvas = canvas[0];\n", " this.context = canvas[0].getContext(\"2d\");\n", "\n", " var backingStore = this.context.backingStorePixelRatio ||\n", "\tthis.context.webkitBackingStorePixelRatio ||\n", "\tthis.context.mozBackingStorePixelRatio ||\n", "\tthis.context.msBackingStorePixelRatio ||\n", "\tthis.context.oBackingStorePixelRatio ||\n", "\tthis.context.backingStorePixelRatio || 1;\n", "\n", " mpl.ratio = (window.devicePixelRatio || 1) / backingStore;\n", "\n", " var rubberband = $('');\n", " rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n", "\n", " var pass_mouse_events = true;\n", "\n", " canvas_div.resizable({\n", " start: function(event, ui) {\n", " pass_mouse_events = false;\n", " },\n", " resize: function(event, ui) {\n", " fig.request_resize(ui.size.width, ui.size.height);\n", " },\n", " stop: function(event, ui) {\n", " pass_mouse_events = true;\n", " fig.request_resize(ui.size.width, ui.size.height);\n", " },\n", " });\n", "\n", " function mouse_event_fn(event) {\n", " if (pass_mouse_events)\n", " return fig.mouse_event(event, event['data']);\n", " }\n", "\n", " rubberband.mousedown('button_press', mouse_event_fn);\n", " rubberband.mouseup('button_release', mouse_event_fn);\n", " // Throttle sequential mouse events to 1 every 20ms.\n", " rubberband.mousemove('motion_notify', mouse_event_fn);\n", "\n", " rubberband.mouseenter('figure_enter', mouse_event_fn);\n", " rubberband.mouseleave('figure_leave', mouse_event_fn);\n", "\n", " canvas_div.on(\"wheel\", function (event) {\n", " event = event.originalEvent;\n", " event['data'] = 'scroll'\n", " if (event.deltaY < 0) {\n", " event.step = 1;\n", " } else {\n", " event.step = -1;\n", " }\n", " mouse_event_fn(event);\n", " });\n", "\n", " canvas_div.append(canvas);\n", " canvas_div.append(rubberband);\n", "\n", " this.rubberband = rubberband;\n", " this.rubberband_canvas = rubberband[0];\n", " this.rubberband_context = rubberband[0].getContext(\"2d\");\n", " this.rubberband_context.strokeStyle = \"#000000\";\n", "\n", " this._resize_canvas = function(width, height) {\n", " // Keep the size of the canvas, canvas container, and rubber band\n", " // canvas in synch.\n", " canvas_div.css('width', width)\n", " canvas_div.css('height', height)\n", "\n", " canvas.attr('width', width * mpl.ratio);\n", " canvas.attr('height', height * mpl.ratio);\n", " canvas.attr('style', 'width: ' + width + 'px; height: ' + height + 'px;');\n", "\n", " rubberband.attr('width', width);\n", " rubberband.attr('height', height);\n", " }\n", "\n", " // Set the figure to an initial 600x600px, this will subsequently be updated\n", " // upon first draw.\n", " this._resize_canvas(600, 600);\n", "\n", " // Disable right mouse context menu.\n", " $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n", " return false;\n", " });\n", "\n", " function set_focus () {\n", " canvas.focus();\n", " canvas_div.focus();\n", " }\n", "\n", " window.setTimeout(set_focus, 100);\n", "}\n", "\n", "mpl.figure.prototype._init_toolbar = function() {\n", " var fig = this;\n", "\n", " var nav_element = $('
')\n", " nav_element.attr('style', 'width: 100%');\n", " this.root.append(nav_element);\n", "\n", " // Define a callback function for later on.\n", " function toolbar_event(event) {\n", " return fig.toolbar_button_onclick(event['data']);\n", " }\n", " function toolbar_mouse_event(event) {\n", " return fig.toolbar_button_onmouseover(event['data']);\n", " }\n", "\n", " for(var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " // put a spacer in here.\n", " continue;\n", " }\n", " var button = $('