{"nbformat":4,"nbformat_minor":0,"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.1"},"colab":{"name":"linear_regression_numpy.ipynb","provenance":[]}},"cells":[{"cell_type":"markdown","metadata":{"id":"Pfgo0zOrouke","colab_type":"text"},"source":["# <center>CS568:Deep Learning</center>  <center>Spring 2020</center> "]},{"cell_type":"markdown","metadata":{"id":"lQglfW2Noukj","colab_type":"text"},"source":["## Import libraries"]},{"cell_type":"code","metadata":{"id":"5F1nObTqoukl","colab_type":"code","colab":{}},"source":["import numpy as np\n","import matplotlib.pyplot as plt"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"KyxcNflpoukq","colab_type":"text"},"source":["## Load toy dataset"]},{"cell_type":"code","metadata":{"id":"6RuSpn64oukr","colab_type":"code","colab":{},"outputId":"59c96df3-15fb-4889-ca6d-ddda5ee22fa0"},"source":["data_x = np.linspace(1, 10, 200)[:, np.newaxis] # generate 100 numbers between 1 and 10.\n","print(data_x.shape)\n","data_y = np.sin(data_x) +  0.1 * np.power(data_x, 2) + 0.5 * np.random.randn(200, 1)\n","print(data_y.shape)\n","data_x /= np.max(data_x) # normalize data_x values between 0 and 1\n","data_y /= np.max(data_y)\n","#print(data_x)\n","## add bias input \n","data_x = np.hstack((np.full_like(data_x, 0.1), data_x))\n","print(data_x.shape)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["(200, 1)\n","(200, 1)\n","(200, 2)\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"raZxYbkloukv","colab_type":"text"},"source":["## Split data into train and test sets"]},{"cell_type":"code","metadata":{"id":"u66uOehXoukw","colab_type":"code","colab":{},"outputId":"9e3f881a-fe28-487f-f3d8-7ed469b7fb60"},"source":["order = np.random.permutation(len(data_x)) # randomly permute data_x values\n","\n","# split data into train and test sets\n","split = 40\n","\n","train_x = data_x[order[split:]]\n","train_y = data_y[order[split:]]\n","\n","print(\"train_X, train_Y:\",train_x.shape, train_y.shape)\n","\n","test_x = data_x[order[:split]]\n","test_y = data_y[order[:split]]\n","\n","print(\"test_X, test_Y:\",test_x.shape, test_y.shape)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["train_X, train_Y: (160, 2) (160, 1)\n","test_X, test_Y: (40, 2) (40, 1)\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"0AxJMgXFoukz","colab_type":"text"},"source":["## Fit model "]},{"cell_type":"code","metadata":{"id":"bRvpFJcwouk1","colab_type":"code","colab":{}},"source":["def calculate_loss(weights):\n","    pred = np.dot(weights, train_x.T)\n","    loss = (1/2)*np.sum((pred - train_y.T) ** 2)\n","    delta = pred - train_y.T\n","    return loss, delta\n","\n","def get_gradient(delta):\n","    temp = np.dot(delta, train_x)\n","    return temp\n","\n","def plot_results(X, y, c, label):\n","    plt.plot(data_x[:,1], data_x.dot(weights.T), c='g', label='predicted model')\n","    plt.scatter(X[:,1], y, c=c, label=label)\n","    plt.grid()\n","    plt.legend()\n","    plt.xlabel('X')\n","    plt.ylabel('Y')\n","    plt.show()\n","\n","W = np.random.randn()\n","b =np.random.randn()\n","weights = np.array([W, b]).reshape((1, 2))\n","lr = 0.01\n","tolerance = 1e-5\n","                   \n","# Perform Gradient Descent\n","Epochs = 1\n","while True:\n","    loss, delta = calculate_loss(weights)\n","    gradient = get_gradient(delta)\n","    new_w = weights - lr * gradient                  \n","    \n","    # Stopping Condition\n","    if np.sum(abs(new_w - weights)) < tolerance:\n","        print(\"Converged.\")\n","        break\n","    \n","    # Print loss every 100 iterations\n","    if Epochs % 100 == 0:\n","        print(\"Epoch: %d - Loss: %.4f\" %(Epochs, loss))\n","    \n","    Epochs += 1\n","    weights = new_w\n","    \n","\n","print(\"weights =\",weights)                \n","plot_results(train_x, train_y, 'b', 'training points')"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"zAc1vGYZouk5","colab_type":"text"},"source":["## Plot results"]},{"cell_type":"code","metadata":{"id":"_N5gRDLgouk6","colab_type":"code","colab":{},"outputId":"780145d8-b54b-41a4-ca3d-5b54860944b6"},"source":["plot_results(test_x, test_y, 'r','testing points')"],"execution_count":0,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XdcVfX/wPHXBxcq5M4yFazcW5xpKZm7vuY3/Tkw03Kl\nVlrOLEd+Kc1ZbrK0cpYjtcyRgatS1By5UdHQci9EReDz++PCBZFxxXvuueP9fDx4wDn3wz1vPuJ5\ncz5Taa0RQgghALzMDkAIIYTzkKQghBDCSpKCEEIIK0kKQgghrCQpCCGEsJKkIIQQwkqSghBCCCtJ\nCkIIIawkKQghhLDKbnYAD6pw4cLa39/f7DAeys2bN8mbN6/ZYTgFqYtkUhfJpC6S2asudu3adVFr\nXSSzci6XFPz9/dm5c6fZYTyUsLAwGjVqZHYYTkHqIpnURTKpi2T2qgul1ClbyknzkRBCCCtJCkII\nIawkKQghhLByuT6FtNy9e5eoqChu375tdig2yZcvH4cOHTI7DKeQXl14e3tTvHhxcuTIYUJUQngu\nt0gKUVFR+Pr64u/vj1LK7HAydePGDXx9fc0OwymkVRdaay5dukRUVBSlSpUyKTIhPJNbNB/dvn2b\nQoUKuURCEJlTSlGoUCGXefITwp24RVIAJCG4Gfn3FMIchiUFpdRXSqnzSqm/0nldKaU+V0pFKKX2\nKaVqGBWLEEII2xj5pDAPaJ7B6y2A0okfPYGZBsbicnx8fAA4e/Ysbdu2zbDslClTiImJeaD3DwsL\n48UXX8xyfLaIjIykUqVKGZY5depUpmWE8EgLFoC/P+zaZfm8YIFDLmtYUtBabwYuZ1CkNfCNtvgD\nyK+UetyoeJxBfHz8A39PsWLFWLp0aYZlspIUhBBObMEC6NkTTiVOQj51ynLsgMRgZp/CE8DfKY6j\nEs+5nMjISMqVK0dQUBDly5enbdu21pu0v78/Q4YMoUaNGnz//fccP36cNm3aEBAQwLPPPsvhw4cB\nOHnyJPXq1aNy5cp88MEH97x30l/S8fHxDBw4kEqVKlGlShWmTp3K559/ztmzZwkMDCQwMBCA9evX\nU69ePWrUqEG7du2Ijo4GYO3atZQrV44aNWqwfPnyNH+WefPm8fLLL9OkSRP8/f2ZNm0akyZNonr1\n6tStW5fLly15fs+ePdStW5cqVarQpk0brly5AsCuXbuoWrUqVatWZfr06db3jY+PZ9CgQdSqVYsq\nVaowe/Zse/4TCOFehg+H1H/oxcRYzhvMJYakKqV6YmliomjRooSFhd3zer58+bhx4wYAQ0KHsP/C\nfrtev3KRyowLHJfu69HR0Rw5coSpU6cya9Ys+vTpw+TJk3n77bfRWuPj48OmTZsAeOmll5g4cSJl\nypQhPDycXr168eOPP9K3b1+6du1Kp06dCAkJASzDNaOjo0lISODGjRvMmTOHiIgItmzZQvbs2bl8\n+TIFCxZk4sSJrF69mkKFChEZGcno0aNZsWIFefPmZfLkyXzyySf079+f7t27s3r1ap566im6du1K\nXFyctd6S3L59m3379rF161bu3LlDtWrVGD16NJs3b2bo0KGEhITQt29fOnfuzPjx42nQoAH/+9//\nGD58OOPGjeO1115jwoQJ1K9fnw8++MAa+9y5c/H29ubXX3/lzp07NG3alGeeeQattbVMardv377v\n39qdRUdHe9TPmxGPr4u33rJ+GV28OGETJiS/ZnC9mJkUzgAlUhwXTzx3H611CBACULNmTZ16cahD\nhw5Zx7rnzJmTbNmy2TXQnDlzZjivwMfHhxIlStCkSRMAunXrxueff46vry9KKbp06YKvry/R0dFs\n376dbt264eVleUi7c+cOvr6+bN++nZUrV5IjRw569OjByJEj8fX1xcfHBy8vL3x9fdm6dSt9+/al\nQIECANaYlFL4+Pjg6+vLpk2bOHLkCM2bW7pzYmNjqVevHmfOnOHJJ5+kevXqAHTt2pWQkJD7fi5v\nb28aN25MsWLFAEvCbdeuHb6+vgQEBLBv3z4SEhK4fv06LVq0AKBnz560a9eO+Ph4rl+/br32G2+8\nwcaNG/H19WXz5s3s27eP1atXA3Dt2jX++ecfihUrZv35UvP29rbG6wlkEbhkHl8XXbtam47CJkyg\n0cCBlvN+fhAZaeilzUwKq4B+SqnFQB3gmtb6n4d90ynNpzx0YFmReghlyuOkZW8TEhLInz8/27Zt\nS/MmaI9hmFprmjRpwqJFi+45v2fPHpvfI1euXNavvby8rMdeXl7ExcVlOa6pU6fSrFmze87/9Vea\ng9OE8GzBwZY+hJRNSHnyWM4bzMghqYuA34GySqkopdQbSqneSqneiUXWACeACOALoI9RsTjC6dOn\n+f333wFYuHAhDRo0uK/MI488QqlSpVixYgVguVHu3bsXgPr167N48WIAFqTTmdSkSRNmz55tvTEn\nte/7+vpam1/q1q3Ltm3biIiIACxrsR89epRy5coRGRnJ8ePHAe5LGg8iX758FChQgC1btgDw7bff\n0rBhQ/Lnz0/+/PnZunXrfT9Hs2bNmDlzJnfv3gXg6NGj3Lx5M8sxCOHWgoIgJMTyZACWzyEhlvMG\nM3L0UUet9eNa6xxa6+Ja6y+11rO01rMSX9da675a66e01pW11i69SULZsmWZPn065cuX58qVK7z5\n5ptplluwYAHffPMNVatWpWLFiqxcuRKAzz77jOnTp1O5cmXOnEmzFY3u3btTsmRJqlSpQtWqVVm4\ncCFgab5p3rw5gYGBFClShHnz5tGxY0eqVKlCvXr1OHz4MN7e3oSEhNCqVStq1KjBo48++lA/79df\nf82gQYOoUqUKe/bsYcSIEQDMnTuXvn37Uq1aNbTW98ReoUIFatSoQaVKlejVq1eWnzqE8AhBQZam\nooAAy2cHJAQAlfI/riuoWbOmTr3JzqFDhyhfvrxJEVlGCL344os2N4XI2kfJMqoLs/9dHc3j29FT\nkLpIZsdNdnZprWtmVs5tlrkQQgjx8CQp2IG/v790mArhrpJmFnt5OXRmsVlcYp6CEEKYImlmcdIo\noKSZxeCwNn5HkycFIYRIj4kzi80iSUEIIdJz+vSDnXcDkhSEECI9JUs+2Hk3IEnBDq5evcqMGTOy\n/P2pVzlt2bIlV69etUdoD8yWa8+bN4+zZ886KCIhTBQcbJlJnJLRM4tTd2xfzmixafuTpGAH9k4K\na9asIX/+/PYI7YHZcm1JCsJjpJxZrJTxM4tTLpmtteXzqVMOHfHkmUnBzkPMhg4dyvHjx6lWrRqD\nBg0CYPz48dZlokeOHAlYlpxo1aoVzzzzDJUqVWLJkiVpLn3t7+/PxYsXiYyMpHz58vTo0YOKFSvS\ntGlTbt26BUB4eDhVqlSxXjOtjWrCwsJ47rnnaNWqFWXLlqV3794kJCQAlmUuKleuTKVKlRgyZIj1\nezK79tKlS9m5cydBQUFUq1aNW7duMXToUCpUqECVKlUYmLRwlxDuImlmcUKC8TOL0+rYTkhwbMe2\n1tqlPgICAnRqBw8evO9cuubP1zpPHq0tedjykSeP5XwWnTx5UlesWNF6vG7dOt2jRw+dkJCg4+Pj\ndatWrfSmTZv00qVLdffu3fX169e11lpfvXpVa621n5+fvnDhgvX7k45Pnjyps2XLpv/880+ttdbt\n2rXT3377rdZa64oVK+rffvtNa631kCFD7rl+ktDQUJ0rVy59/PhxHRcXp1944QX9/fff6zNnzugS\nJUro8+fP67t37+rAwEC9YsUKm6/dsGFDHR4errXW+uLFi7pMmTI6ISFBa631lStXHqjukuoiLQ/0\n7+oGQkNDzQ7BaXhsXSh1770JdOiECZbzDwnYqW24x3rek4IDhpitX7+e9evXU716dWrUqMHhw4c5\nduwYlStXZsOGDYwYMYItW7aQL1++TN+rVKlSVKtWDYCAgAAiIyO5evUqN27coF69egB06tQp3e+v\nXbs2Tz75JNmyZaNjx45s3bqV8PBwGjVqRJEiRciePTtBQUFs3rzZpmunli9fPry9vXnjjTdYvnw5\neVK3vwohbOcEHduelxQcMMRMa82wYcPYs2cPe/bsISIigjfeeIMyZcqwe/duKlSowAcffMBHH32U\n6XulXMY6W7ZsD7yIXEZLetvj2tmzZ2fHjh20bduWH3/80bqXghAiC9Lq2PbycsiS2dbLOexKzsKA\nTJxy6WqwLBP91VdfWbfBPHPmDOfPn+fs2bPkyZOHDh06MGjQIHbv3p3m92cmf/781o15AOuS22nZ\nsWMHJ0+eJCEhgSVLltCgQQNq167Npk2buHjxIvHx8SxatIiGDRtm6eeNjo7m2rVrtGzZksmTJ1uX\nAhdCZEFaHdt+fg6dPe15y1wYsHlFoUKFqF+/PpUqVaJFixaMHz+eQ4cOWZt3fHx8mD9/PhEREdaO\n6Fy5cjFz5kwgeenrYsWKERoaatM1v/zyS3r06IGXlxcNGzZMtymqVq1a9OvXj4iICAIDA2nTpg1e\nXl6MHTuWwMBAtNa0atWK1q1b2/zzdu3ald69e5M7d25+/vlnWrduze3bt9FaM2nSJJvfRwiPs2CB\npan69GnLH6LBwfff8IOC7j3n6G1Jbel4cKaPh+5o1trSqeznZ+m88fN7qE7mrMioc9VWN27csH79\nySef6Lfffvu+MqGhobpVq1YPfS0jSUdzMo/tXE2DW9ZFFge52KsusLGj2fOeFOD+TOyCfvrpJz75\n5BPi4uLw8/Nj3rx5ZockhMhIRoNc0rkf/f7375y5lfamW0bxzKTgBtq3b0/79u0zLNOoUSPZqEQI\nZ/EAg1xOXzvN0F+GsuivRTQv2pwgHPdHrNt0NGsX20FOZEz+PYVDOWLPBBsGudyMvcnI0JGUm1aO\nFYdX8OFzH/J26bftH0sG3CIpeHt7c+nSJbmRuAmtNZcuXcLb29vsUIQnSGtpiZ497Z8YMlhHKUEn\nMH/ffMpOK8tHmz+idbnWHOl3hI8CPyJ3ttz2jSMTbtF8VLx4caKiorhw4YLZodjk9u3bcsNLlF5d\neHt7U7x4cRMiEh4nC239WZL0XqlGH/3R8Cn6f/kM289sp2axmixpu4T6Jevb77oPyC2SQo4cOShV\nqpTZYdgsLCyM6tWrmx2GU5C6EKZz5J4JKQa5/H3tb4ZuHMrCLxfyuM/jzGs9j1ervoqXMrcBxy2a\nj4QQHshe/QAOXloi5m4Mo8JGUXZaWZYdXMbwZ4dz9K2jvFbtNdMTArjJk4IQwsPYc+9kAya0piVB\nJ7Bo/yKGbhxK1PUo2ldsz7gXxuGX38+u13lY5qclIYR4UPZc2NIBeyZsj9rOM18+Q+cVnSmatyhb\num1hcdvFTpcQQJ4UhBCuyN79AAZNaI26HsWwjcOYv28+j/k8xtzWc+lStYtTNBOlR5KCEML1lCxp\naTJK67wTiLkbw4TfJjBu2zjiE+J5v8H7DG0wFN9cvmaHlilD05VSqrlS6ohSKkIpNTSN1/MppVYr\npfYqpQ4opboZGY8Qwk2YsXeyDbTWLNy/kLLTyjIybCStSrficL/DBDcOdomEAAY+KSilsgHTgSZA\nFBCulFqltT6Yolhf4KDW+iWlVBHgiFJqgdY61qi4hBBuIJ0x/2auabbjzA76r+3P71G/U+PxGiz8\n70Ke9XvWtHiyysgnhdpAhNb6ROJNfjGQen1mDfgqy84vPsBl4MF2kRFCuL+0hp86cu/kDJy5foYu\nK7pQZ04dTl49yVf/+YrwHuEumRAAlFFLQyil2gLNtdbdE49fBeporfulKOMLrALKAb5Ae631T2m8\nV0+gJ0DRokUDMtpUxhVER0fj4+NjdhhOQeoimdRFsnvq4vJlS/9BQkJyAS8vyyihggXNCRC4HX+b\n76K+Y9HpRcTreNoVb0dQySDyZLfvlrT2+r0IDAzcpbWumWlBW9bXzsoH0BaYk+L4VWBaGmUmAwp4\nGjgJPJLR+6a1n4Krccu14rNI6iKZ1EWye+rCz+++zew1WM6bICEhQS/av0iXmFRCMwrd9ru2+sTl\nE4Zdz9H7KRjZfHQGKJHiuHjiuZS6AcsTY45ITArlDIxJCOFqHLkMBWQ4Uzr8TDgN5jag47KOFMpT\niLDXwvi+3feUKuA6y+xkxsghqeFAaaVUKSzJoAPQKVWZ00BjYItSqihQFjhhYExCCFfjyOGn6cyU\nPht3hWH5wvlm7zcUzVuUOS/NoWu1rmTzymb/GExmWFLQWscppfoB64BswFda6wNKqd6Jr88CxgDz\nlFL7sTQhDdFaXzQqJiGEC3LQMhTAfTOlb2WHiTVj+OTY28R552BI/SG8/+z7PJLrEftf20kYOnlN\na70GWJPq3KwUX58FmhoZgxDCxTly+Glik5QGvqsIg5vA6fzwykHNp7MO8WSBJ+1/TSfjvHOthRAi\nScrhp8HBlgRhxC5pJUuysxg8+zp0aAcFbkPoPFi63c8jEgLIMhdCCFdiz9VRUzl74yzD3y7JvBun\neDQavlgF3f6EbLnzQIi5M6UdSZ4UhBCuw5bVUR9wn4Vbd28RvDmYMlPLsDBmO4Pzv8ixH0rQ/U9F\ntpL2XzHV2cmTghDCdWQ2PPUBniS01nx/8HsGbxjMqWunaFOuDeObjOepgk/BOwbF7wLkSUEI4Toy\n2yXNxn0Wdp3dxXPznqP90vbk987Pr11+ZXn75ZaE4OEkKQghXEdmq6Nm8iTxz41/eH3l69T6ohZH\nLh4h5MUQdvXcRWCpQAODdi3SfCSEcB2ZDU9NZ6Lb7VIlmLzlEz7e+jF34u4w8JmBDH92OPm88zkw\neNcgSUEI4Voy2iUt1UQ3DSyrlpNBbW8R+ev7vFzuZcY3Gc/TBZ92XLwuRpqPhBDGe8ARQVmWYr/l\n3Y9Do965aPdyLL4FH2Njl42saL9CEkIm5ElBCGEsA+cWpOXf1o0Z7vMrc/fMpVAeX2YFfkb3Gt3d\ncp0iI0hSEEIYK6MRQXZMCrfjbjPljykEbwnmTtwd3q33Lh8+96H0GzwgSQpCCGMZvPS11prlh5Yz\naMMgTl49SeuyrRnfZDylC5W2y/t7GulTEEIYK7O5BSkl9T3s2mVT38Of//xJ4NeBtP2+LXlz5uWX\nV3/hhw4/SEJ4CJIUhBDGymxuQZKkvoekIaVJfQ9pJIZz0efosaoHASEBHLhwgJmtZvJnrz9p/GRj\ng34IzyFJQQhhrBQjglDK8jmt9YRsmI18J+4O47aOo/TU0szbO48BdQdw7K1j9K7Zm+xe0hpuD1KL\nQgjjZTS3IEkGfQ9aa1YcXsGgDYM4ceUEL5V5iQlNJ1CmUBn7x+rhJCkIIZxDOrOR91R7jAHfPE9Y\nZBgVi1Rkfef1NHmqiQkBegZpPhJCOIdUfQ/n8kLPNtmo0fpf9p/bz4yWM9jTe48kBIPJk4IQwjkk\nNi/d+fB9FseGsvAdxa2c0L9ufz587kMK5C5gcoCeQZKCEMIpaK1ZGZCXgf1zcPzKT7xY6UUmNJlA\n2cJlzQ7No0jzkRDCdPvO7aPxN41ps6QNubLn4tPKn7K642pJCCaQpCCEMM35m+fptboX1WdXZ++5\nvUxrMY29vfdSq2Ats0PzWNJ8JIRwuNj4WD7f/jljNo8h5m4Mb9V+i5ENR0q/gROQpCCEcBitNauO\nrGLghoFEXI6gZemWTGw6kXKFy5kdmkgkSUEI4RD7z+1nwLoBbDy5kfKFy/Nz0M80f7q52WGJVCQp\nCCEMdeHmBUaEjiBkdwj5vfMztcVUegX0Ike2HGaHJtJgaFJQSjUHPgOyAXO01mPTKNMImALkAC5q\nrRsaGZMQwjFi42OZun0qH23+iJuxN+lXqx8jG42kYO6CZocmMmBYUlBKZQOmA02AKCBcKbVKa30w\nRZn8wAygudb6tFLqUaPiEUI4htaa1UdX897694i4HEGLp1swselEyhcpb3ZowgZGDkmtDURorU9o\nrWOBxUDrVGU6Acu11qcBtNbnDYxHCJEWO+6fvP/cfprOb0rrxa3J7pWdNZ3WsCZojSQEF6K01sa8\nsVJtsTwBdE88fhWoo7Xul6JMUrNRRcAX+Exr/U0a79UT6AlQtGjRgMWLFxsSs6NER0fj4+NjdhhO\nQeoimSl1cfmyZRG6hITkc15eluWtC9rezHM19ipzT83lx7M/kjd7Xrr6d+U/j/8ny8tZy+9FMnvV\nRWBg4C6tdc3Mypnd0ZwdCAAaA7mB35VSf2itj6YspLUOAUIAatasqRs1auToOO0qLCwMV/8Z7EXq\nIpkpdeHvn+bKpPj5QWRkpt8eGx/L9B3TGf3HaKJjo+lbuy8jG46kUJ5CDxWW/F4kc3RdGJkUzgAl\nUhwXTzyXUhRwSWt9E7iplNoMVAWOIoQwXhb3T9Za89Oxn3hv/XscvXSUZk81Y1KzSVQoUsGAIIUj\nGdmnEA6UVkqVUkrlBDoAq1KVWQk0UEplV0rlAeoAhwyMSQiR0oPsn5zowPkDNJvfjJcWvYRC8VOn\nn1jbea0kBDdhWFLQWscB/YB1WG7032mtDyileiuleieWOQSsBfYBO7AMW/3LqJiEEKnYun8ycDHm\nIn1/6kuVWVUIPxvOlGZT2P/mflqWbumgYIUjGNqnoLVeA6xJdW5WquPxwHgj4xBCpCNpi8zhwy1N\nRiVLWhJCiq0zY+NjmRE+g9GbRnPjzg3erPkmoxuNfuh+A+GcZJVUITxdUJClUzkhwfI5MSForfnp\n6E9UnlmZAesGUPuJ2uztvZdpLafdnxDsOKxVmMvs0UdCCCd08MJBBqwbwPrj6ylTqAw/dvyRlqVb\nopS6v/CCBdCzJ8TEWI5PnbIcwz1PHMI1yJOCEMLqUswl3lrzFlVmVmHHmR1MbjaZ/W/up1WZVmkn\nBLA0PSUlhCQxMZbzwuXIk4IQgrvxd5kRPoNRm0Zx/c51egf0ZnTgaArnKZz5N2dxWKtwTvKkIIQ7\nS6+tP8X5NQ2KUnmcH/3X9adWsVrs7b2X6a2m25YQIEvDWoXzkqQghLtKaus/dQq0Tm7r79MHevbk\nYMwpWnTStGpynoRz/7L68fdY13kdlR6t9GDXeYBhrcL5SVIQwl2l09Z/6dvZvN0whipvwu8lYNJa\n+Gu65sWPl6bfb5CRoCAICbEsjaGU5XNIiHQyuyjpUxDCXaVq07/rBTNrwahGCVzLBb12wehQKBKT\ndvkHEhQkScBNSFIQwl2VLGld7G7t0/BuMzhUBBqfgMlroXLqheqlD0AgzUdCuK/gYA4X96ZlELTo\nbHlSWLk8Jxvy9qZytPQBiLRJUhDC3SxYwOUyJXhnQWcqvX6bbSUVE9bDgR9L8p/BX6FmzJQ+AJEu\naT4Swo3cnf81s2f1YGSbu1z1hh67Yczv3hT57AtLgeHD4dVX01zjSAiQpCCE21gXsY4Bu3pwqMld\nnj8Bk9dBlXMAt+Cdd+DWLVmKQmRKmo+EcHGHLx6m1cJWNF/QnNiEu/ywCH75JikhJLp0SZaiEDaR\npCCEi7py6wr91/an8szKbD29lfFNxnNgdUlaHwGbZxvIUhQiFWk+EsLFxCXEMXvnbEaEjeDq7at0\nr96dMc+P4dG8j8KYx+9dsRQsI4ty57Y8LaQmw1BFKpIUhHAh64+vZ8C6ARy8cJBA/0AmN5tM1ceq\nJhdIb9McSDtZyDBUkUq6SUEptQboo7WOdFw4Qoi0HLl4hIEbBvLj0R95ssCTLP+/5bxc7uW0l6XI\naHZxBjusCQEZ9ynMBdYrpYYrpXI4KiAhRLIrt67w7rp3qTSzEpsiN/HpC59ysM9B2pRvk5wQbN31\nLJ0d1oRIKd0nBa3190qpn4EPgZ1KqW+BhBSvT3JAfEJ4pLiEOEJ2hTAidASXb12me43ujAkcQ1Gf\novcWlF3PhJ1l1qcQC9wEcgG+pEgKQghj/HLiFwasG8Bf5/+ioV9DpjSfQrXHqqVdOKNdzyQpiCzI\nqE+hOTAJWAXU0FrHpFdWCPHwomKi+M+i/7D66GpK5S/Fsv9bRptybTJezlp2PRN2ltGTwnCgndb6\ngKOCEcITXb19lTGbxvD5zs/JnSM3YxuP5Z267+Cd3Tvzb06xEup954XIgoz6FJ51ZCBCeJq4hDjm\n7J7Dh6EfcinmEi0ea8GXQV/ymM9jtr9JcLAMNRV2JfMUhDDBxhMb6b+uP3+d/4vn/J5jSrMpXDty\n7cESAqQ/L0H6E0QWSVIQwoGOXTrGwA0DWXVkFf75/Vnabin/Lf9flFKEHQnL2pvKrmfCjgxd+0gp\n1VwpdUQpFaGUGppBuVpKqTilVFsj4xHCLNduX2Pg+oFUnFGRX0/+yieNP+FQ30O8UuGVrO2LLIRB\nDHtSUEplA6YDTYAoIFwptUprfTCNcuOA9UbFIoRZ4hPirf0GF2Mu0q1aN/73/P943Pdxs0MTIk1G\nNh/VBiK01icAlFKLgdbAwVTl3gKWAbUMjEUIh/v15K8MWDeAfef28WzJZ5nSfAo1Hq9hdlhCZMjI\npPAE8HeK4yigTsoCSqkngDZAIJIUhJuIuBzBoA2D+OHwD/jl8+O7tt/RtkJbaSYSLsHsjuYpwBCt\ndUJG/2GUUj2BngBFixYlLCzMMdEZJDo62uV/Bntxp7qIjotm/qn5LDuzjOwqO91Ldadd8XbkvJCT\nTZs2Zf79blQXD0vqIpnD60JrbcgHUA9Yl+J4GDAsVZmTQGTiRzRwHng5o/cNCAjQri40NNTsEJyG\nO9RFXHycnr1zti7yaRGtRind7Ydu+uz1sw/8Pu5QF/YidZHMXnUB7NQ23LuNHH0UDpRWSpVSSuUE\nOmBZMiNlQiqltfbXWvsDS7Es1f2DgTEJYVehJ0OpEVKDXj/2omzhsoT3COer1l8Z15Fs64qoQmSR\nYc1HWus4pVQ/YB2QDfhKa31AKdU78fVZRl1bCKMdv3ycQRsGseLwCvzy+bGk7RLaVWhnbL+BrIgq\nHMDQPgWt9RpgTapzaSYDrXVXI2MRwh6u37lO8OZgpmyfQg6vHPwv8H+8W+9dcufIbfzFZUVU4QBm\ndzQL4RLiE+KZu2cuw38dzvmb53mt6mt83PhjivkWc1wQsiKqcABDZzQL4Q42RW6i5hc16bG6B6UL\nlia8RzjXlVIFAAAUx0lEQVTzXp5n/4Rw+XLG/QXprXwqK6IKO5KkIEQ6Tlw5wSvfvUKjrxtx+dZl\nFr+ymC3dtlCzWE37X2zBAksfwalToHVyf0HKxBAcbFkBNSVZEVXYmSQFIVK5fuc6Q38ZSvnp5Vkb\nsZYxgWM43Pcw7Su1N64jefhwy97JKSX1FyQJCoKQEPDzA6Usn0NCpD9B2JX0KQiRKD4hnnl75jH8\n1+Gcu3mOLlW78PHzH/PEI08Yf3Fb+wtkRVRhMEkKQgCbT22m/9r+/PnvnzxT4hlWd1xNrSccuPKK\n9BcIJyFJQXi0k1dOMviXwSw9uJQSj5Rg4X8X0qFSB8evUxQcDOfP33tO+guECaRPQXikG3du8P7G\n9yk/vTxrjq3ho0YfcbjfYTpW7mjOwnVBQZY+AukvECaTJwXhURJ0AvP2zOP9je9z7uY5Xq3yKh83\n/pjijxQ3OzQoWBAiI82OQng4SQrCY2w5tYX+6/qz+5/d1C1el5UdVlKneJ3Mv1EIDyJJQbi9yKuR\nDN4wmO8Pfk/xR4qz4L8L6FjJpGYiIZyc9CkIt3Xjzg2GbxxOuWnl+PHoj4xqOIoj/Y7QqXIncxKC\nrHAqXIA8KQi3k6AT+GbvNwzbOIx/o/8lqHIQY18Ya26/gaxwKlyEPCkIt7L19FZqf1Gbbiu74ZfP\nj9/f+J35/51vfkdyRiucCuFE5ElBuIVTV08x+JfBfHfgO57wfYL5bebTsXJHvJST/N0jK5wKFyFJ\nQbi06Nhoxm4dy4TfJuClvBjZcCSDnhlE3px5zQ7tXiVLWpqM0jovhBORpCBcUoJO4Nu93zJs4zD+\nif6HTpU7MbbxWErkK2F2aGkLDr63TwFkxrJwSpIUhMvZdnob/df1Z+fZndR+ojbL/m8Z9UrUMzus\njCV1Jg8fbmkyKlnSkhCkk1k4GSdpcBUic6evnabjso40mNuAszfO8s3L3/D7G787b0JIPQQVLDOW\nExIsnyUhCCckTwrC6UXHRvPptk8Z/9t4AEY8N4LB9Qc7X79BSjIEVbgoSQrCaSXoBObvm8+wjcM4\ne+MsHSt1ZOwLYymZzwU6ZzMagipJQTgxaT4STum3v3+j7py6vPbDazzh+wTbXt/GwlcWOi4hPOzs\nYxmCKlyUPCkIp3L62mmG/jKURX8t4nGfx/n65a/pXKWzY+cb2KPpR4agChclTwrCKdyMvcnI0JGU\nm1aOFYdX8MGzH3D0raN0qdrF8RPQ7DH7ODjYMuQ0JRmCKlyAPCkIUyX1Gwz9ZShnbpyhfcX2jHth\nHH75/cwLyh5NPzIEVbgoSQrCNH9E/UG/P/tx6MYhAh4PYEnbJdQvWd/ssOzX9BMUJElAuBxDn8uV\nUs2VUkeUUhFKqaFpvB6klNqnlNqvlPpNKVXVyHiEc/j72t8ELQ+i3pf1OH/nPPNaz2NHjx3OkRBA\nmn6ERzPsSUEplQ2YDjQBooBwpdQqrfXBFMVOAg211leUUi2AEEC2wnJTMXdj+HTbp3y67VMSdALD\nnx1OfV2fFtVamB3avaTpR3gwI5uPagMRWusTAEqpxUBrwJoUtNa/pSj/B+AEG+UKe0vQCSzav4ih\nG4cSdT2K/6v4f4x7YRz++f0JCwszO7y0SdOP8FBGNh89Afyd4jgq8Vx63gB+NjAeYYLtUdt55stn\n6LyiM0XzFmVLty0sabsE//z+jg3EiF3PZCc14YaU1tqYN1aqLdBca9098fhVoI7Wul8aZQOBGUAD\nrfWlNF7vCfQEKFq0aMDixYsNidlRoqOj8fHxMTsMQ124c4EvTnzBhvMbKJizIN1LdadZ0Wb3DS91\nSF1cvmzpOE5ISD7n5QV+flCwoNO8pyf8XthK6iKZveoiMDBwl9a6ZqYFtdaGfAD1gHUpjocBw9Io\nVwU4DpSx5X0DAgK0qwsNDTU7BMPcjL2pR4WO0rn/l1vnGpNLv//L+/r67evplndIXfj5aQ33f/j5\nOdV7uvPvxYOSukhmr7oAdmob7rFGNh+FA6WVUqWUUjmBDsCqlAWUUiWB5cCrWuujBsbiGUxsztBa\ns3D/QspOK8uoTaN4scyLHOp7iODGwfjm8nVYHGkyYskJWcZCuCnDkoLWOg7oB6wDDgHfaa0PKKV6\nK6V6JxYbARQCZiil9iildhoVj1vI6KaftDTDqVOWv1mTlmZwQGLYcWYH9b+qT9DyIIrkKcKmrpv4\nrt13lCpQyvBr2yS9+QUPs+SEEe8phBMwdJ6C1nqN1rqM1voprXVw4rlZWutZiV9311oX0FpXS/zI\nvL3LU2V20zdhY/gz18/QZUUX6sypw4krJ/jyP18S3iOc5/yeM+yaWWLrvIMHedKSuQzCTcnaR64i\ns5u+A5szYu7GMGbTGMpMK8OSA0sYWn8ox946xuvVXyebVza7X++hBQVBSIilE1gpy+eQkHuHnD7o\nk5Yt7ymEC5JlLlxFZjd9B6zKqbVmyYElDN4wmL+v/80r5V/h0yaf8mSBJ+12DcNkNu8gK/sfyFwG\n4YbkScFVZNaGbXBzRviZcBrMbUDHZR0plKcQYa+FsfT/lrpGQrCFdBwLAUhScB2Z3fQNas44c/0M\nr/3wGrXn1CbicgRzXprDzh47aejf8KHe1+lIx7EQgDQfuQ5b1uOxY3PGrbu3mPj7RD7Z+glxCXEM\nqT+E9599n0dyPWKX93c6wcH3bqwD0nEsPJIkBVfigDZsrTXfHfiOwb8M5vS10/y3/H/59IVPearg\nU4Ze13SyCJ4QgCc3H8m6NffZeXYnz859lg7LOlDAuwChr4Wy7P+WOS4hmP1vEhQEkZGWpSsiIyUh\nCI/kmUnBnhO9zL6R2cHZG2fptrIbtb6oxdFLRwl5MYRdPXfRyL+R44IwcfKdECKZZyYFe030cvEb\n2a27twjeHEyZqWVYuH8hg58ZzLG3jtEjoEfW5xtkNUmaMPlOCHE/z0wK9hp+6KI3sqR+g/LTy/NB\n6Ac0faopB/scZFyTceTzzpf1N7Y1SaZMHPv3W45lSKgQTsEzk4K9hh+64I1s19ldPDfvOdovbU8+\n73xs7LKR5e2X26ffwJYkmTpxxMZajtNbblqGhArhUJ6ZFOw10csVxrYn/lX+zyOK14N8qPVFLY5c\nPMLsF2ezu+duni/1vP2uZUuSTC9xgKwlJIQT8MykYK+JXs6+KNqCBdx+sweflDhFmX4w/6mbvLcj\nG8eKBtMzoKf91ymyJUmmlzguX5a1hIRwAp6ZFMA+ww+deFE0rTVLQ/pT/vVbvP8CND4JB6fD+DVx\n5BthUNKyJUlmlDhkSKgQpvPcpGAvTngj2/3Pbhp93Yh2z1/ENxZ++Rp+WAxPX04sYFSfhy1J0tmf\nroTwcJIU3Mi/0f/yxso3qBlSk4MXDjLrt4Lsnm15SriH1sbNqcgsSaZOHDlzpv105QbzP4RwRZIU\n3MDtuNuM3TqW0lNL8+2+b3m33rsce+sYvbp8TnbvPGl/k5lzKlImjsqV004IqYe2du4MhQtLchDC\nYJIUzJTeX8M2/pWstWbZwWVUmF6BYRuH8Xyp5znQ5wATmk4gv3f+e/8qT4uzzqlIa4QSwKVLLjU5\nUAhXJEnBLOlN9OrTx6YJYH/+8yeBXwfS9vu25M2Zl/Wd17Oyw0pKFyp973WS/ipXKu04nHFORUYx\nOWsiE8JNSFIwS3rj9UNCMpwAdi76HD1W9SAgJIC/zv/FjJYz+LPXnzR5qknG13OFORVJMovJGROZ\nEG5CkoJZ0ruxxcenefrOmVOM2zqO0lNLM2/vPAbUHUDE2xG8WetNsnvZsAK6K436SSvWlJwxkQnh\nJiQpmCW9G1u2eyeUaWB5eajwdnaGbhxKI/9GHOhzgInNJlr6DWzlxHMq7pMUa6FC97/mrIlMCDch\nScEs6f3l3rOn9fyex+D51+CV9pC78GOs67yOVR1XUaZQmaxd0wnnVKQrKAguXoT5810jkQnhJmTn\nNbNksNPXuTqV+HDdEOaUiabgHS+mF3mVnr3n2NZM5G4csNucECKZB95lnEiqG96duDt8vm08Y86O\n4Vb527xTqz8jGo6gQO4CJgYphPAkkhScgNaalUdWMnD9QI5fOU6r0q2Y2HQiZQuXNTs0IYSHkaRg\nsn3n9tF/bX9CI0OpUKQCa4PW0uzpZmaHJYTwUIZ2NCulmiuljiilIpRSQ9N4XSmlPk98fZ9SqoaR\n8TiT8zfP02t1L6rPrs7ec3uZ1mIae3vvlYQghDCVYU8KSqlswHSgCRAFhCulVmmtD6Yo1gIonfhR\nB5iZ+Nlt3Ym7w5K/l7Dwj4XE3I3hrdpvMaLhCArmTmfnMSGEcCAjm49qAxFa6xMASqnFQGsgZVJo\nDXyjtdbAH0qp/Eqpx7XW/xgYlym01qw6sor31r/H8SvHaVm6JRObTqRc4XJmhyaEEFZGJoUngL9T\nHEdx/1NAWmWeAO5JCkqpnkBPgKJFixIWFmbvWA11IvoE049PZ/fV3fjl8WPU06NoWKwh//71L//y\nr9nhmSo6Otrl/j2NInWRTOoimaPrwiU6mrXWIUAIQM2aNXWjRo3MDchGF25eYEToCEJ2h5AvVz4+\nb/45vWv2ZtuWbbjKz2C0sLAwqYtEUhfJpC6SOboujEwKZ4ASKY6LJ5570DIuJzY+lqnbp/LR5o+4\nGXuTfrX6MbLRSOk3EEI4PSOTQjhQWilVCsuNvgPQKVWZVUC/xP6GOsA1V+5P0Fqz+uhq3lv/HhGX\nI2j+dHMmNZ1E+SLlzQ5NCCFsYlhS0FrHKaX6AeuAbMBXWusDSqneia/PAtYALYEIIAboZlQ8Rtt/\nbj/vrn+XX078QrnC5VjTaQ0tSrcwOywhhHgghvYpaK3XYLnxpzw3K8XXGuhrZAxGu3DzAiPDRjJ7\n12zy5crHZ80/482ab5IjWw6zQxNCiAfmEh3Nzig2PpZpO6bx0aaPiI6Npk/NPoxqNIpCedJY7lkI\nIVyEJIUHpLXmp2M/8d769zh66SjNnmrGpGaTqFCkgtmhCSHEQ5Ok8AAOnD/AgHUD2HBiA2ULleWn\nTj/R4ukWqPT2PxZCCBcjScEGF2MuMjJ0JLN2zeKRXI8wpdkU+tTqI/0GQgi3I0khA7HxscwIn8Ho\nTaO5cecGb9Z8k1GNRlE4T2GzQxNCCENIUkiD1po1x9bw7vp3OXrpKE2ebMLkZpOp+GhFs0MTQghD\nSVJI5eCFgwxYN4D1x9dTplAZVndcTavSraTfQAjhESQpJLoUc4lRYaOYuXMmPjl9mNR0En1r9yVn\ntpxmhyaEEA7j8UnhbvxdZoTPYNSmUVy/c53eAb0ZHTha+g2EEB7Jo5PCmmNreHfduxy5dIQXnnyB\nyc0mU+nRSmaHJYQQpvHIpHDwwkHeW/8eayPWUrpgaVZ1WMWLZV6UfgMhhMfzqKRwKeYSozeNZkb4\nDHxy+jCx6UT61e4n/QZCCJHIY5LCmmNr6Ly8M9fuXKNnjZ58FPgRRfIWMTssIYRwKh6TFMoUKkOd\n4nUY98I4qhStYnY4QgjhlDwmKTxd8Gl+DvrZ7DCEEMKpeZkdgBBCCOchSUEIIYSVJAUhhBBWkhSE\nEEJYSVIQQghhJUlBCCGElSQFIYQQVpIUhBBCWCmttdkxPBCl1AXglNlxPKTCwEWzg3ASUhfJpC6S\nSV0ks1dd+GmtM13bx+WSgjtQSu3UWtc0Ow5nIHWRTOoimdRFMkfXhTQfCSGEsJKkIIQQwkqSgjlC\nzA7AiUhdJJO6SCZ1kcyhdSF9CkIIIazkSUEIIYSVJAUDKaWaK6WOKKUilFJD03g9SCm1Tym1Xyn1\nm1KqqhlxOkJmdZGiXC2lVJxSqq0j43MUW+pBKdVIKbVHKXVAKbXJ0TE6ig3/P/IppVYrpfYm1kU3\nM+J0BKXUV0qp80qpv9J5XSmlPk+sq31KqRqGBaO1lg8DPoBswHHgSSAnsBeokKrMM0CBxK9bANvN\njtusukhR7ldgDdDW7LhN+p3IDxwESiYeP2p23CbWxfvAuMSviwCXgZxmx25QfTwH1AD+Suf1lsDP\ngALqGnmvkCcF49QGIrTWJ7TWscBioHXKAlrr37TWVxIP/wCKOzhGR8m0LhK9BSwDzjsyOAeypR46\nAcu11qcBtNaeXBca8FVKKcAHS1KIc2yYjqG13ozl50tPa+AbbfEHkF8p9bgRsUhSMM4TwN8pjqMS\nz6XnDSx/CbijTOtCKfUE0AaY6cC4HM2W34kyQAGlVJhSapdSqovDonMsW+piGlAeOAvsB97RWic4\nJjyn86D3kyzzmD2anZlSKhBLUmhgdiwmmgIM0VonWP4w9FjZgQCgMZAb+F0p9YfW+qi5YZmiGbAH\neB54CtiglNqitb5ubljuTZKCcc4AJVIcF088dw+lVBVgDtBCa33JQbE5mi11URNYnJgQCgMtlVJx\nWusfHBOiQ9hSD1HAJa31TeCmUmozUBVwt6RgS110A8ZqS6N6hFLqJFAO2OGYEJ2KTfcTe5DmI+OE\nA6WVUqWUUjmBDsCqlAWUUiWB5cCrbv6XYKZ1obUupbX211r7A0uBPm6WEMCGegBWAg2UUtmVUnmA\nOsAhB8fpCLbUxWksT0wopYoCZYETDo3SeawCuiSOQqoLXNNa/2PEheRJwSBa6zilVD9gHZaRFl9p\nrQ8opXonvj4LGAEUAmYk/oUcp91wETAb68Lt2VIPWutDSqm1wD4gAZijtU5zmKIrs/F3YgwwTym1\nH8uomyFaa7dcOVUptQhoBBRWSkUBI4EcYK2LNVhGIEUAMVieooyJJXG4kxBCCCHNR0IIIZJJUhBC\nCGElSUEIIYSVJAUhhBBWkhSEEEJYSVIQ4iEopUoopU4qpQomHhdIPPY3NzIhskaSghAPQWv9N5b1\nmsYmnhoLhGitI00LSoiHIPMUhHhISqkcwC7gK6AHUE1rfdfcqITIGpnRLMRD0lrfVUoNAtYCTSUh\nCFcmzUdC2EcL4B+gktmBCPEwJCkI8ZCUUtWAJlh2xBpg1OYnQjiCJAUhHkLirmAzgf6Ju6WNByaY\nG5UQWSdJQYiH0wM4rbXekHg8AyivlGpoYkxCZJmMPhJCCGElTwpCCCGsJCkIIYSwkqQghBDCSpKC\nEEIIK0kKQgghrCQpCCGEsJKkIIQQwkqSghBCCKv/BzrsI5dBM8yPAAAAAElFTkSuQmCC\n","text/plain":["<matplotlib.figure.Figure at 0x8e773b61d0>"]},"metadata":{"tags":[]}}]}]}