{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CNN-ek - befejező notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0.) Inicializáló rész"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Az előző notebookban (vagy mnég azelőtt) definiált segédfüggvények átemelése, melyet most is változtatás nélkül fogunk használni."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from keras import backend as K\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.per_process_gpu_memory_fraction=0.095\n",
    "session = tf.Session(config=config) # Letrehozunk egy TF session-t a megfelelo mem. korlattal\n",
    "K.set_session(session) # atadjuk azt a Kerasnak"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "from keras.models import Model\n",
    "from keras.layers import Input, Dense, Flatten, Conv2D, MaxPooling2D \n",
    "from keras.layers import Concatenate, Lambda, Average, Add, Dropout\n",
    "from keras.optimizers import SGD, Adam\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "from keras.utils import to_categorical\n",
    "from matplotlib import pyplot as plt\n",
    "from random import sample as RS\n",
    "from keras.metrics import top_k_categorical_accuracy\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "def inTop3(x,y):\n",
    "    return(top_k_categorical_accuracy(x,y,k=3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.datasets import cifar10\n",
    "\n",
    "def mintak_cifar10():\n",
    "    (x_tan,y_tan),(x_tst,y_tst)=cifar10.load_data()\n",
    "    x_tan=x_tan.astype('float32')/255.0\n",
    "    x_tst=x_tst.astype('float32')/255.0\n",
    "    classes=['airplane', 'auto', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']\n",
    "    idx=RS(set(range(y_tan.size)),5)\n",
    "    print('Mintakepek es a cimkeik: '+str([x[0] for x in y_tan[idx]]))\n",
    "    print('Cimkehez tartozo osztalyok: '+str([classes[x[0]] for x in y_tan[idx]]))\n",
    "    \n",
    "    (f, sf)=plt.subplots(1, 5)\n",
    "    f.set_size_inches(16, 6)\n",
    "    \n",
    "    for i in range(5):\n",
    "        img = x_tan[idx[i]]\n",
    "        sf[i].imshow(img)\n",
    "    plt.show()\n",
    "    \n",
    "    y_tan=to_categorical(y_tan).astype('float32')\n",
    "    y_tst=to_categorical(y_tst).astype('float32')\n",
    "    return(x_tan,y_tan,x_tst,y_tst,classes)\n",
    "\n",
    "def tanitas_earlystop(model, x_tan, y_tan, epn, tol):\n",
    "    \n",
    "    model_checkpoint = ModelCheckpoint('CNN.hdf5', monitor='val_loss', verbose=1, save_best_only=True)\n",
    "    earlystop=EarlyStopping(monitor='val_loss', patience=tol)\n",
    "    hst=model.fit(x=x_tan, y=y_tan, verbose=1, batch_size=32, epochs=epn, callbacks=[earlystop, model_checkpoint], validation_split=0.1)\n",
    "    model.load_weights('CNN.hdf5')\n",
    "    \n",
    "    plt.title('Tanito gorbek')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.ylabel('Loss')\n",
    "    \n",
    "    epn=len(hst.history['loss'])\n",
    "    \n",
    "    epi=np.linspace(1,epn,epn)\n",
    "    plt.plot(epi, hst.history['loss'], 'b-', epi, hst.history['val_loss'], 'r-')\n",
    "    plt.legend(['tanito', 'validacios'], loc='upper left')\n",
    "    plt.show()\n",
    "    return model\n",
    "\n",
    "def teszt(halo, x_test, y_test):\n",
    "    \n",
    "    tmp=halo.evaluate(x_test,y_test)\n",
    "        \n",
    "    y_pred=np.argmax(halo.predict(x_test),axis=1)\n",
    "    print('Confusion Matrix:')\n",
    "    print(confusion_matrix(np.argmax(y_test, axis=1), y_pred))\n",
    "    print()\n",
    "    print('Loss:'+str(tmp[0])+'; Acc:'+str(tmp[1])+'; Top3Acc:'+str(tmp[2]))\n",
    "    print()\n",
    "    return({'loss':tmp[0], 'acc':tmp[1], 'top3acc':tmp[2]})\n",
    "\n",
    "def halo_mentes(model,nev):\n",
    "    model_json=model.to_json()\n",
    "    with open(nev+'.json','w') as json_file:\n",
    "        json_file.write(model_json)\n",
    "    model.save_weights(nev+'.hdf5')\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Olvassuk be az adathalmazt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mintakepek es a cimkeik: [8, 2, 9, 7, 1]\n",
      "Cimkehez tartozo osztalyok: ['ship', 'bird', 'truck', 'horse', 'auto']\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6IAAAC8CAYAAABizBPxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzsvWmQZcl133fyLm+tV3tXdfU+S3fPDDFADzDYCEAGCIE7uEi0JMohyRGKAD+YEWKYERaD4QgzFP7AsCXRZtBWEBIpgBJNSzJJEyJpUyBAAsRCDGYwg8HM9PT09L7Uvr39vrukP0zTnsz/GXR1dc2r6uf/L6Kj+57Oe1/ezJMn8753zz+NtVYIIYQQQgghhJBhEex3BQghhBBCCCGE/P8LPogSQgghhBBCCBkqfBAlhBBCCCGEEDJU+CBKCCGEEEIIIWSo8EGUEEIIIYQQQshQ4YMoIYQQQgghhJChwgdRQgghhBBCCCFDhQ+ihBBCCCGEEEKGyn09iBpjftAYc8EY87ox5hf2qlKEHBTo42TUoY+TUYb+TUYd+jh5kDHW2t2daEwoIq+JyCdE5KaIfFNEftpa+8pbnROVK7ZUH/OvpFzcO1aqaJTzTIA2q5yMxZTzlHZR28oo9VBu6e6f+BYnKp9ZKDa/PXbar9pHmh3cwG795q3O1WyB11Fan2v921lfX7PWHtp1Bf/q83bh43EltuV6xbEVRaFc/K4GMUbrZ82G3yf5zRkofRpGIdgKi3VV66/Vw/jH+JlBoHz3tUNf8kvtYJjduTx+ZmHzHXzCzggMXj/c4X0W3l1oMUwxyfri9r74eLXSsBNjs/41dvJBaNpZsbe43t2vpsfYHV5/L1Hd6u6+tsvTdn495Vo7jetaMdUmfuzQYj+et7Z5fd9ieDkOba0SOzYtbvm2HY2Dtyi303OxDjuLsUWB8c4occuvh16t+7jPHZwXRRHYAmWuSpIET/Z8SZ+7kDRNwdbt9sFWqD7uGne6NOoP8n3z8ZmZGXv8+AnHpvnSTnpMW49aZR2hX921BqGyltlVrd763LfzWju9Xp5lyonKOiJEv99JTbSxp8X2TKuHcv0wcMek+qylXP/555/fkY/jiN857xOR1621l0VEjDH/u4j8uIi8pfOX6mNy9hM/5hqVxpfAbXyboVOHIVa9VKuALc8HYCvH7mdqzj9IsYO0YBUoQTOMlEW/dxxpC6cI65ErkS/p4T1FnqNkSl3VB3qlrqVSDDZ/bktTrIPN7/5w8sa5WLcsw4myUi05x2GIFyuU8772W5+5hp+6K+7Zx8v1ipz7oXOOrdPvQTnrDWSrBJzI4L1VFVto0O/T1O2wsTL26fhMA2ydtAO2bg9txuL4iDz/jeMSlKlUysq1lMWC8qBYeGNBC9JFrvhIgfXoKfdUiOuX6pc0aJJyhNefGh8HW5ag3w+8WBfVMJ5UYvzUz/7Sf9wXH58Ym5W//+P/xLEFAfZD6MVUbeLyy7xh0xbWWA9cMO9sQW6C+3iS2yX6Q9vdF8i+v4uIqENFXQhqDyP2ux6L6AskvRy2d6bM06l1Y1+e47hOlRj+r/7Df7VvMbxWieVj5447trEGxspy2Y1l2gOUFqNKJYwXcYzx2Z+vc+VhsqasecbG0NZqN8FWqWA5/x60Mao+lCtfhmprtNhrD20czMzOgK0xOQm2S5cvgc0fbJ02xnktXt28uQy25198FWzdPtY38+YqrZ8Uk7xyfWvffPz48RPyhS/+mWMrlXBuxv7Hcd9P8QuBXoIP8bESp+LQ9fsq/FD1Fg+i2kSsxVm1nFtwp18C6TOHtua9+/WaGxtgi5R1RH0C1xH+DxTaa63al+NphuuPjc11sBVKjJ5sTDvHsbKezAq8fr02tiMfv59Xc4+KyI03Hd+8Y3MwxnzKGPOsMebZTHFOQg4w9+7jfeULAEIOLnf18Tf7d6/fGmrlCLlP7jmGJ6n2tgQhB5Z79vH19bWhVY6Qu/G2ixVZaz9trX3aWvt0VMZv4Ah50HF8vKJ8s03IA8yb/btawV+GCHnQebOPl2PtdThCHmze7OMzM7N3P4GQIXE/D6K3ROTN768cu2MjZFSgj5NRhz5ORhn6Nxl16OPkgeZ+ckS/KSKnjTEPyRtO/3dE5O/e7SRj3HfsdyKEYnN8Bz0MlPfZbRVsiZKPWORuHoyW26jWS0v2yZQ8pKqSyFv4963kWeTYHUmi5BukSs5syS0X7iCH787V8Fo5tpnN3XIl5TV4NetJMcZKHphRcmTC2PMDwbYo1d7WXyDv2ceDMJBaw8tzUN6n93NETaDkFwXYh6UAG7S3gu25emnLOV4PcQw99DSOoVjJL5quYN5GmrXBlgy8XFhNeEDJnxir1sEWKmPNz8nS8qriGO9pYwPzo+IQfTyK/OvjLyNaflRFya2ZnsCcpqpSt0rVbds0xXzi0tv7A809+bgxRuLQzyPTckRdW6CMeU3QKVRilOJGmCOqisloSb7DzxHV8OcDEcz11ERACnWu0u5JyZ/2c0SVa2VWy7vG17HDSAnsWk5u5vpKruSYhUpd95B7juFhYGSq4cbBxjiuLSreG15BjDE8U3IgNQ0GLW/UX4NouiLlCrZdRckzL1cxHtWqmH9WLrv3Wa3ifauCc0rltF4tVdzrxcpbcpmSUKmN5RPHT4LN159otdB3qxWcb6zFPnn51ctg6yZdsEFo07Q43t53D+/Zx1dWV+V/+l8+7dje9773QrnZWU+YLkQ/vXwd0wBffOGbYKsr+hazM66Ozfd+7Puxskp7qiI+aj6o5ktuZ0TqtXamVaDhx3Etd/yzn/51sM3NL4Dt4z/8SbClmbuWC5UYEyhzx+Kt62D7g9//P8DWaeOYed8H/ppzXKlhXFjfwnzTnbLrB1FrbWaM+VkR+RN5I+b8prX25V3XhJADBn2cjDr0cTLK0L/JqEMfJw869/OLqFhr/1hE/niP6kLIgYM+TkYd+jgZZejfZNShj5MHmbddrIgQQgghhBBCCHkzfBAlhBBCCCGEEDJU7uvV3HvFWCvhwBPVUZKC04Gb+F5WlHHKkbI5bqAkJqvbabjXKyl1mNA2sFZEBkQR0AnK2KyBv7Nujp85UDb21jamrVVR9KRe84RWIkUwCSwiomzurCfWe5voKoWsck95jonU2qbmWYpiB/3MFW6xWuK5lmi+z+ReW0WK3xSe+EIYoAiA9jXRIMW9eKsxbgA+Hbv+e+3mi1BmcxtFfMoh+lY5R5u2+bKflK+JrWQ5tkVvoIgCKaIy1hMn6ifob9MVTKI/eXQKP7OHG53nnihaWynTKxQ/VQZWTxGCKVXx3nPvekkP+6Q6XsMP2CeMMVLy+lkTY/DFJFRhImXsKppdqkgEiBWp4hKKKNrO9i7fUzSNO6uIjvniNFYRIdI2aFe1iiw2mi9WlBlFYEYZd9vruPl6eRyFbjTBHT9i58q8kR8QAam/Io5jOTw/79hKZUUc0RPtKisCHkYR39LaKY6UDeK9eXIwGCh1QIeolJW5RGnjqiISV6u680a9jrFHHWuaYIrB8b26se0cbywtQ5lDc4fANq4IMFWVtV3V0z4KDJapKWJFm+Movjc3j/WojeP8W3j9VGhrngzb/ztXt8A2LNIsk9UNV1zGPxYRGZuY8E7E+0gSFEK8dQtFe8eVNWnmCW9ubGGbaNFBW0Nq4oWhonTX6/rrSi0W46fmioinL44lItJsunO4Vq/XXr8ItuVV3Nu1NIU+OMjcz8xTjAtdRXCovb4CtiuXFEGuDo6FNHAHVqWOApb3s/syfxElhBBCCCGEEDJU+CBKCCGEEEIIIWSo8EGUEEIIIYQQQshQGWqOaKVckbOnzzo2o+YoerkXdcx5qOA+yFJRcklT5d38PHevr22GXo6U/CUlR1HL0QiVfJKStxFwrNx3orxvXqpgFxlRNhP3E5GUTaG1HNR+irZCOTfz3o9PlHyVTMkfUHM/+5hTkCSYe9Hve+/yK9kCByu76I38n+Mn3Y22E+8+RETSgXu/oZLL0uorfajkZs6MzYItWXPzFGptLNMYmwZbVEV/LgnmPlXLmEOde+Oj18Ncg1jJnx7k6EtlJY+q7+W9aTlTnZVNsNVjJXdL2Vy7n7g5oWmm+GSOfRnHON61vNflDaxvbtzPSPrbUGZCyXvdLwJjpBy7fRMpsRJyRJVcHc2m54MqNi//M1AK+WXe8lr6TuiKbXdoOaJa/rT1ckQLJXdJ27VdzUEtsJx3eYm1HPRAqxfa6mXMS8fRIlLErs9nSo5odl/ZRXuPCUIp1d3cOD8fVAT9vlTGfMqSkq+p5Yhq+dK5N19HIcaPQslZL5cwXpTKOL9UKhjDK2X33GoVc8HUHFElH7Tdx/XMn3zp887xSxevQZn3P30ObD/8n70HbDUlJ9ePKUoqoTTGJsFWr+G8UalgXM+Nkv85cO+zUNZUuRKL9hNrC0m9+e7GNcwX9MtUAvSjrS3MIY817RRljbO4tOgcf/OZr0GZqSnsL23un5zE9Uyzi2vNzW13jp2bm4MyJSWvc2sbfSRV8jN9rBKg+wOsV0lZA3f6aMu9OWCgTBN5jLEoi/ChyUTKs0oFL3jkkdPO8ez8gnItJTdd/mfFhvAXUUIIIYQQQgghQ4UPooQQQgghhBBChgofRAkhhBBCCCGEDBU+iBJCCCGEEEIIGSpDFSsKolCqU+7m8lbZodsXMAoUYZ9yBZOhp8cxKbhI8fqbbTepdj3B5/HWABPtQ6vYlI2i8zaWi70E41KMTV8oIkS9dUxW7mkJzJl7bmTw+rkiHDRQkq2tIsrk27QEbOW0t9gYeGfiFHHoJlJHiqBDkuE97SelUkmOHzvh2rDa0tz0No9WRLVsgAJDHSWhPR5gkn5rym1js4HJ94mSMD9exo3qa2XcADxSEtN7fVfYIIzRn6t1pTGsJiaEY9L3uboiDpIrQhFpgeNR83FTdj9TCU0SaaI7yqb12gbyVhH5KDzxhyDEexKLggL7hTFGyp7wSagIcfhjNVQ2Mw9UMSHNpokaeWJFimicpg+iCRhpQkeaMNpuKVThIE1wziujihUppymxuMjvvkm7VeZVTS2qOokxoaSIjoEakojkxo0TgeBY1PpkP8myXJbXXUETbVN6X2CoUsJ4WlUEbyJV+Ahtvo/3FNG7cgnPq1Yxnm5t4gb3FWUNVSq5n9HtdqGMLgKG7bO8gWJ1L5y/5ByvNfH6l69dB9vW1sNg63Y7YPNHQreDbdZuoW19E4VoNA0ztZ98kTHlRHPAZBV73a688MLzjm17ewvK3V456hzPzeKaxCjruZomHKS0cX/L9ZFWG/vURIrQlrL+6CgimIUS42pVdz0z6Crr6QivpYlXlcqKampwdzHUXFlcDJT1eWUMBcUyb21f0eZHxU9NhrFXrCaaivVIvXNrEyggVQj2007hL6KEEEIIIYQQQoYKH0QJIYQQQgghhAwVPogSQgghhBBCCBkq95Ujaoy5KiItEclFJLPWPr0XlSLkoEAfJ6MOfZyMOvRxMsrQv8mDzF6IFX3MWru2k4JhGMmEl+RaKGI5qSeikqWYTGxSFEaoBJhMPD6OQitp5v4Q/PKqkpWeYtMszOIPyKVYEYpI8J4K756KkiJioCRDz4yh8EBcRlup7J5bDBTRlh6241gdk63HqmhLUjfBv5/hPWoaRJpYQJJg0nQpxuTz0Ev6DxVRke12Ez9079mxj1trJcvdZO+aktDeGHcFQIwiOFIOsU3SFpabjI+DbbXsikDkOYpCTE3P4bWmUCzHakJK+TbYwsD1r6kJTF6vVtGWpXj9KMRykScYogm+2AIT7TUtobxQxocnUFZRBFnSHEUMUkUgSXFViUKsb+gJDRSKKMrUGIrFvA3syMcDY0AgJVbES3whl1ATE1JEHFQxIaWcLzCknwcmtV80gSRNAGi3aIJ8moib/5lWEytSqqWKFYWKzRcrUuoQ5kq9ajiH+oJVIiKBIpCUGtemvYJlFDGnt4kd+fggHcjtmzcdW6mEsTj2RFSsYByolPE8da6LUDQk8gJXkqCQy8zsDNjGxvH6N27eAJumxpOmbgzUxp4/tkVEykpsW2uhrdl05+vpGtb18ROKEEqK935rZRFs1hNy6Q9wnaK19eraOtgSZR3nC2mKiATeZ1qDc1AQDEVUccfrlNyKtLw1dLWB88zcqbPO8SDCNWpVWfhNzMyDLYlQhCrvufFmcm4ByoTaWkBZa2YD9LdGRVnf1tx7SFq4htTWrWkFfTUYmwKbjd14aZXxUijiXkkXP1MTlAtid/0faGJ4yngslH6ymsCTKGJF3vOLFe2edv+CLV/NJYQQQgghhBAyVO73QdSKyH8yxjxnjPmUVsAY8yljzLPGmGe7w/n1ipC95N58vIW/PBJywPmuPv5m/2538ZdwQh4AduzjA+UNB0IOOPe0Tsm1rTwI2Sfu99XcD1trbxlj5kTk88aYV621X35zAWvtp0Xk0yIih08+fLA2UyLk7tyTjy88dIQ+Th40vquPv9m/Ty6coX+TB5Ed+/hko0IfJw8a97ROqdTH6OPkwHBfv4haa2/d+XtFRH5fRN63F5Ui5KBAHyejDn2cjDr0cTLK0L/Jg8yufxE1xtRFJLDWtu78+/tF5J98t3Matap89Olzji1UEvw3V5edY02zoT9og629vQS21WVMaB/kbgJz2lSS7w0KAh0aw+aan8EE78hgQnfsiaNUJyagTKV69/NERDJFsaLkJWVH2vddiphEYwzFaSolTKRu91yxgHaiiBClmOTc3MLXsZM+igDU6w2wlSNPFEURctGERn4NLLtjNz5urQXRm82+koSeu23Q7GM75XianDvxfrDVAxSs2Nx0X6GcO4xJ9VPTY2ALIvSRgSL4UInxO6x61RM1scq4KqHwyUDw+kGoCYO518uVJP1ASaKPlWsFAQoP2MItp4nMtAfYKb44lcgbgic+JcV/e303jlXrirhJhOftFffq48YYKXv9EKmCJoF3jG2pjV3/vDfKaf3nHyvXPyBiRYqmljqn+Z/o+6NeSvdT3eZfXxHP0oS3FGEiowi+aO0deB8aKN99m+ztFSu6Vx+PwlCmJv04hXWsVj3REOX+y4rAiSZ8FGmCJp6TmABfp9REo7Trz82hMJ02ZvqeeIkmCKdRL+PNb3Q2wdb1BBPnZ3AddHQe66qJPj106iTYrCds01YEYOp1RXBH6ael27fw+kp7ZJ7olxZPAqV/94pdrVMCA+I7VllwHKm799ZWRKMCZY2al/B+mxVlrZ+5656wtwVl5sZxjV2XnfllrsTL0Lh1q0+hP0zUUDBrdRkFrXpdbI/qvOvTmRYXlZA3UMREq5ronPcYUijrIBsoj3bKnGaUdrRKjK4f8gQxlXVcpE1qO+R+Xs2dF5HfvzPoIhH536y1//d9XI+QgwZ9nIw69HEy6tDHyShD/yYPNLt+ELXWXhaRd+1hXQg5UNDHyahDHyejDn2cjDL0b/Kgw+1bCCGEEEIIIYQMFT6IEkIIIYQQQggZKve7fcs9UYojeWh+2rHZAhPwF8ZdEYBcEdnJCxQISZKjYCtSTNBtDVzbQ0/g9eslTJBuNDCZe1pJpC7HKAAUGPczI8H6BwYTvMMIk+jTHJO+I0+goKwkKwea0IWSDC052qbG3QTsnrIPVZ6hQEs+r4hfBJrQBdryzOsXpaqFPVh7vllrJU/dtslzbCvrtXEQYj+bKvbhyZNnwFbqY2L9uaddUaNrq4pYQoZ7Qo6VMQk9NxWwab7ki2T0eugP5QpeK4rwPjXBB1/sR5OTCRRRkVwR0bLK2YEnCpRnKATSKKOImTGaWJGiNKUJJxSuOEik+HO99vaJFd0rgTFS8fohUoSIfPEVTcjFKIoNoVIwUGKDX0y7li5MpAg2vM1iRVYRk9BEYPzPLNQ6KDG8QFumhEUop4l4acIXigqR0USlNFEO77tuTaRD65P9JI4jOXLUFcwJFbEZP27VajjvawJ7geLjkSL+lHpxa3sLY2elirZyCWNUfR7FAKNYWSNA3ZR4mmG8K8dY7sbaRbCJuPUtV1FErz6GtkLxcU2UycSu8EyhiHY16kr7VNHWqKOYX7u9s7juo/X5flIUhbT73p7nihDRex8+5Bwn/jki0lXEJ6vKuvhigG33R899wzm2yygQdXYOr5X3sa5pE69//gIKmFan3GeQifkFKLO+chtsh5R5YiPBe5876tY3KuM4a0Q4hm4nuB7L28tgK9Xd9d7AKMKLCc4v60rbFrkiqqjEIl80qZxsQJnOxgrYdsrBGh2EEEIIIYQQQkYePogSQgghhBBCCBkqfBAlhBBCCCGEEDJUhpojKiJSeIkkfhqgiIh4uVpBhIWCQMmbqqDNFkrepZefV55UctkizPcYKO9dd1r4rnqraIKt13VtWxdegjLTs/NgO3HufViPHN8vTzvu9TdXVqFMkuCGuWEV22y8hvkkfg5jL8W8x16vBbZcyWe1Ss5XMsDrZX4epZKENFDuaT8JTCBVLw8yCPAd/jhyczGLDPMbghg3Wp6dnAFbruR/+vlgizfw/f0Pnj4BtoVDs2Dr9jFM9HNlY+vQ7etKjH0aBsqAV3JEixTLFX33epmSp2wttkW5jGO5UPJVul6Oht9HIiITSp9Uy8oG5kr86LQxzoi3cXang7kXoUnAtl+YwEi55MaMMFByX72cukgZu6GWK67NSFruoZ8jquQeipoPquRKK+W0XEnfI41yT1rupyg5tFbJXfNTQrU81SBX8nEn0FZW+iTz8oG2N7FMLjimCiXP0cZ47zZVxoFnMkYZ11rf7SNhGMp4w83zCiMt19OteK2K8ULLbdRy4rX80szLxQwDjD1+Hd6wYV1rNcx31OrR9+ZTrWuCEK+fJOg3165jfltWuOeurOJa6eo1zM8rFjCHs9nBfMXQy4/t9DB2ticmwFYYZa6dOw62INoC2+rqonNsLa7PDhqBiDS8oNlQ5jbj5YTOKvmOaRljwVxFac9xbPeNkyed45MVHEPR+hrYvnX+BbBVe1i3sIP9v3HlpnN8ofk1LLOGn3nuUVwvFRM4bhfOun5z8hhq1/x5A/15ex3H0JMzyhpk2s2hTmK8VpagD978jrJm7+I6rjSBn3l6xh23h+vYrjcWMQd1p/AXUUIIIYQQQgghQ4UPooQQQgghhBBChgofRAkhhBBCCCGEDBU+iBJCCCGEEEIIGSpDFSvqJYm8/Polx1YUys7bXop8mqOwSH+AtkwRxkkUgSFf2aJSwsTqifFJsBlFeKBSwiaMYiy3vuwmtF/60z+FMk9+4ENgWzt5Gmx5pGze7ql3xGOYcDw+h0I3YQXrX4+VTbI9saK+Ila00UQRg1YPBQU0AYFA2Zh7aswVTdKEE6pK++8rRiTwNv+1io+HvoCJskl4yaJfFsoGyjcuXwfbxfPPOMfT4yiY9K7HngBbO0HxCE2EJI5x0/F84NatUmA/Fy0Ul0rQRURy9KU0ca+3vY2J9toG5vUptFUa+B3coVnX36o19C0zwPZRtBokVMQvIsVXS56owyQOA6lE2Hf7RWBEyrEr+GMUsZ+diRWhLVCEfbSB748ff8y91XlWUTWq1HCOqIZKm4du/21von9bu7Pvdn0hGhGR0PNdLd41xhW/PaWI5HTw+knqCc5t48BTtP1AYFBEJFfaO1CEwjJ/zowwFl6+8k380H3EWpHMi3mayFXoiQJlKZbpKII6xuDcGUcoJlR47Znn2OZavMsyLKeJCWWKUuS1azec47ExrNeEIvbTVGLxjZsokJcX7hhaW0OBw5devgC244fPga0U4xgtvPGt6H1JV+mnucMoTLTVxbYNq7iGarXbznGnhUI3Wj32k1iMzHs+nTRxTf3v/t0fOMeH53DeVzSvZGa1DbYPLJwE2/vn3XZfXkc/eunyJbCtGowj08E42MrjKOQz6wkMTSqCl8HRBbAdnzoEtrSBsXew7o756+uvQplDXaz/j5w9CzbzTYyNPXHXDPGZh6DMkccfBduJWazrFxWB0UgRUjo67Z777rPYl+cengObyH+v2BD+IkoIIYQQQgghZKjwQZQQQgghhBBCyFDhgyghhBBCCCGEkKFy1wdRY8xvGmNWjDEvvck2bYz5vDHm4p2/8cVxQh4Q6ONk1KGPk1GHPk5GGfo3GVV2ovTyGRH5NRH5rTfZfkFEvmCt/WVjzC/cOf7Hd7tQEBgpeeI4aYYJ4bkn7pIrAhCJkpDviweIiFSreP3US9KPyoo4TxWvNd1AAaPxOp4blbC+kx1XyGVyHBOfT1RRBCAYnwZbx6LwQF649zQ9jYnbUYzd3R1gcrikmEjd2Vx16xVgwn+jgsIGaaqIWtRRyCVVhKZKXhZ80sW6pjmKcuyCz8ge+XgUGJmquD5RKP1VePdbL2PfTE7gnFIq0J/Xbl8G25XXnnWOj5zERPL+FoqttFoortHDXH4xRQNsZePaggyT41vrm2DLujheSjFev7Plnms6SlxIUCSht7aN9eii6NDCKXf8veMcClg8cgqT9AOD7WiU8WEsigCYwB0fRjDGTCvCabvgM7IHPm5EpOSJpZlAEfUI3b4JFZEgTZgo0q6liBpBuwWaWJEikBTj9Y8cRz8NrCJ8N3BjbL+Nn6mJEGm2Sgnnl8L7TCvKHDeG9TIhioz0Uhy0hXVjzMwRnLuKHAVs4tlZsK13sFxrG+sWWHdsrKxchTKLi98C2y75jOyBj/d6fXnpFVdgRJs7Y88WKIJGWYJtovXr+DjO1/4Y6vYwZs3M4BzR7+OaYX0NBXQ0oaP19XXneHISrz+r+EO/p6wZuuiDeeGOhVAZt602zkHZAOfQy5deB1tUdeeNEw8/DmUmD58A29g43udMF308XVwF29ikK2C0sX4byqgibPfOZ2SP1imlKJTj0+58l/exv/qeKFm3h2KJbUWQ8pVvnwdbZRn998xpV2in1lHEGLsYP09X0W8erWAcbNRqYKsdcsfa2MQZKJMUGJ+XO+jjN5R54vam247Z1StYhwRjxekT7wDb5sYi2NbOv+Qcb3zxz6HM2FEco2sFzoe2h+Pq+hqu0b796kXnuBYpIrDaHLxD7vqLqLX2yyKy4Zl/XEQ+e+ffnxWRn9iGJ0VeAAAgAElEQVR1DQjZZ+jjZNShj5NRhz5ORhn6NxlVdpsjOm+t/atH9SURmX+rgsaYTxljnjXGPLu9ubXLjyNk6OzKxztN5RdmQg4mO/JxJ4a3GcPJA8U9+3iS7slbNoQMg12tU5IB/vJIyH5x32JF1lororxr8v/9/6ettU9ba5+emNqTV8wIGSr34uP1cXzNj5CDznfzcSeGjzGGkweTnfp4WXkNl5CDzr2sU8qlg7M3NSG7fRBdNsYsiIjc+Rt3LibkwYY+TkYd+jgZdejjZJShf5MHnt1+9fc5EfkHIvLLd/7+g52cVIpjOXnksGPbbuOrjNst9/UvazBxOJyaAVspxnIlg0m1gbiJ41mE3w4VighOV0nK9sUYRERC5RvVb7/mJmp3LkIR2b71dbBNnXwKbNMPoxhBt+e2WTLAulYU0Y+BIlhQNfilWvOZLznHpYmjUMaefBTrleErIHEJE7XHYmzvcuDa6hMo9lKyeK09Ylc+XitX5D1nzjq2PEcfaW65ifWF4DiIyyj+lCWa8BG28cmjriBDcxPHxr/99S+BrTGNIkGlCEVNZqoobDDliXkVgon8A+Wtt9DgPW1vXwPb2qoruJEpY7QoNBEzFMcyKd7Td/7SFRV49XkUXPjYx54E27ufPg22w0fwl0NNsCLP3XuvKH0em7ftF5p79nFjREqRJxSkiLSEkWsLQqVfNAGPQHEQ5fZDTzApUMoEivCRJiQnJZwjBimOqcwTaJhcQL/q9/A+BwMcBzOzin/Efrti/Yscx4oo46BWxz5Jtt05ojrAMVyaQnGPOMQ2KzbRT/sdFAprbrvCLZcuozDRQBEe3EPu2cezPJe1TVewI1b8plxxY2qkCJRpIjtWud1cFMEzT2yr2UTRtZIiqjg5gzF8q7UOtkwRiux4a5xyFeNkp4dzVa+P9+4LKIqIGPGEXBTNkzDG9U2sxPDbi0tgO3HaFYGcmkXBuVoDhYmsEmTGGso66zJ+Zqniiv6Ux1B4qtP1Uzv3jF2tUyYmG/KjP/ZxxxYp8bjkxXFTYD/fWEYhrJu3l8H2zAYKPVUuuePjqQls8+ok+nPUw8+cu43zdd7HVJJwzP2MsH4YygxqGJ+3QkU4NMZnh7Rwx3J2CRf7k3Pol8u3MH4GJbx3OeyeW+niXLX9GvppXxlsRyvoq2mEMeWb33zBNSjCaUWoDOYdspPtW35HRL4uImeNMTeNMf9Q3nD6TxhjLorIX79zTMgDCX2cjDr0cTLq0MfJKEP/JqPKXb9qt9b+9Fv818ffwk7IAwV9nIw69HEy6tDHyShD/yajyn2LFRFCCCGEEEIIIfcCH0QJIYQQQgghhAyVoeqUWytSpG5Cq5bMnyVuQntFEcPIFGGAgaKEEpcx8TaKA+8YrxXFKNpgFFGIKMJn+TTDz/zSTTdB/lAXk5XHbn0FbLe/cRlsHz08C7bZmiseUYRYh6qikjAQLJcnKHRUWnWT7csRJniXp1DAYkwRHwkV8ZVQ6eMic5O+bYi+UlZEDPYXK+IJihQJJnaXxS2TBShoYhTdloEiYDJ7/BGwfcBzy+0unvf8C8+BbXURRfc0gZT+OPpgZ9xNfG+M4xja3kbRhooiiFEpKwJDnuCNL6QgItKYQJGBXoKiLH1lT2M/pPR7OLb/z9//Jti+8nUUSXj/h58A2/ecexhscdUTKVHEwyrBwZHaN8ZI7MW8QInFvlBQEOF9aWJC9XmMIfUp9AVfV0PRYZNAEdkxoVLXGONiJLgNU8mLn5HVxBmUeUMZs2kffTJfcwU4FM046a7g+GlvoS9bg/Gk+borxpVvKkI6cziu68dR3KU0h/49UOLc15/5E7eufRQZWe+0wLafFIWVdtftW0VLTwaZa1Q0PiRJsR/yHG192wObP133eliJ/Da2Z1v5zH5fEXZL0NbruH6phFOJYhyPaQv9OcuwHuKNmUIZQ+tbKNpy8SaKLU0uYDx95AlX3HFsCtcpgSJOqa0djWC5WhXFY44ceqdznCZY/+deuAG2/aSwVjqp22etbezDbt8Vwuk00U+3eiiW06+jCE5PEeL7RtONZ8vK+sAoS7xqinU9nGPAnIowjgc33f6pDFBMKFNEG800+tJEgWvZ5dtuX4cZrqerC3P4mc1NsNk+ipgNPGHLiSNHoExjBreTrW9i2yZraHt0QRkzY27f9ZR6hcp8u1P4iyghhBBCCCGEkKHCB1FCCCGEEEIIIUOFD6KEEEIIIYQQQoYKH0QJIYQQQgghhAyVoYoV5XkhW203cbfV7kC5d5x93Dk+NjsDZdodTFa+uoiJ+6kiFBF4Qg42UxJvY1QeGAgm1k83MAE7irC+eeyeu3n4HJRZLG2D7eXvXAJbGmGbffwT73WOj53CBOZcSaKvKl9FXFnEBObf/eorznHcQPGAj584AbYTJzEpu7AoDNA1irCIcftgoIg1DAQFN/aToiik03N93FisY6ni3VuCifZpH8+bHkexlUfPPAa2rW23f67dfhXKlEv4mc0mjoUkQYGCVgfP7XtCJBvbmqAYXiuOqmiLUcCo7cWKhjL2BoqwUqGovuQFCiwkA/f6gSKg1VOEGS6+tgi26zdwfHzrW7fA9tRHTrp1jVCwwI9X+0lgRCqe4E8RYvsG4o5VG6LwRyh4XmjQ/zo3boKtNjPtHI8r4g+K7pMUik3THDLad7RewUxTE1KwiohG9xr6zPYr33aOyzUUCTIJjpUiQDGMtI+iScWWe/MTYzhPJdEE2MK+Iha4tQq2r/3Fb6PtxW84x1YRZcsKTfRp/8iyQjY23DgVaD4euDariHUUiiCXJl4lW7ieCbxyVhEbXGviWuDKrSWwVSs4/myCsazwhI6aHRyPvT72YYLFpNtFv/TvIFDm/dsrGDv/4jmcv37yP/8JsFXG3TiQK0o3aV+bu3Bt0W7jXFWp4Ly0tOTWt1bDcdVPDpaPtzt9+dozbpuuKiJRA6+ri0IRvYqxD2t1jFPJDAoH+eJEwRL2fXUMx0vcwr5ZUQQZ6489CbZDc66QUn0T193NFsa33mAZ6yaK+J3n99E0rlOkhQJzEiiiTw1FbCl1P7Nssf17ijhgRZlvJ5R6bL+MfdCYfrdznOXoz0abXHcIfxElhBBCCCGEEDJU+CBKCCGEEEIIIWSo8EGUEEIIIYQQQshQGWqOaGEL6XgbeScJ5nStLrvvKOc9TEAol/Dd/wVlE9ogxHyGwcB9hzsd4HvYqfKM3lTy24ySt1EKsFyx8oxzfL52CspsxR8E2zufwveu3/3Bo2A7ccrdhLZex/M2CmzrzR7W/0++/jzY/vSau+l4EGFOi33lNtZ1/izYIqVPJmJsszx3P6ObortOBQcr98JaK0nmtnOeKRtFF65PJyn2zUDJR1weXAZbI8Rcstu3rznHq8t4XtLD/KLxOo6r2RPob5JjrsjamptDsbWF+QcmQH/r5Mp4ydF/ez23XJ5j+4iSkxWW0G+yTNmh3ssBb7VbUKLZxHvKlByNbg99/C+9PGsRka3EzQl95wcPQZmgjLlW+4URkZLxxlxJyT/z8uX8HHkRkUjJFe9dxTycL/yH3wXbx/7+TznHY4eVdlP6JVDyWLTMFi2u+6bCYn6QcpoUEfpfqOgeTD7ixkoTYJ7n8lXMM64I5mNJimM7arjlNjP070AZU/XqNNiaG1iPzhXM45v0xvvsBF5rSwnh51cwL3h4GCm83C9b4Hi2fg6nch+5kh9sFa0JJX1LrHfBkrLmKZVQyyLNcb5pKfmOJSW/zXg+vbGFfrS5dR6vFWHuZJ4p+g3e9bU87kwZo+stJac/x/Hd6brzaq+LuiGJsp5sNjFP8NVXXwZbmil5fG33ejPTuJ6sN3COFsE8xGGRZrksrrk5oVGEvlQru7F9oGgrdFJFV6KrlAvR36aqrt9spnjeKyt4/VNjuNY/VsFxdfkK+uqFmpuz+a75BSgjiTI3tbX8WGyzxryXp1zFOL6l6B40Exy3408/BbY89cpt4BhNlbHXX8e5dSLAcq0NHDPdm259G4+gBo1V1jw7hb+IEkIIIYQQQggZKnwQJYQQQgghhBAyVPggSgghhBBCCCFkqNz1QdQY85vGmBVjzEtvsv2SMeaWMeaFO39++O2tJiFvH/RxMurQx8koQ/8mow59nIwqOxEr+oyI/JqI/JZn/xVr7T+9lw8Lw1CmJicd29rqBpR78eWLzvHEIRR2KCsbNJ9ZQCGEM8dxM9lmy032bRa4iXem6AmE6gb3mGC8chuFYTYvfs45vrGCAjDFGIoR/OLH/guwvef9j4Jte9ndxLrYwuT7rI+2a+uYrPziFRSi+PD3f59zfOad3wNlLkw9BrZ//SqqMIzF6Hbvnsb+nKp6oj8ptv/kFJ63Cz4je+TjeVFIu+8KQxSKWJH4Iha+AIyIhCW0Xbr6LNgWL22C7bXzF5zjTg+FSUJls+9aFUUANlZXwDY7Owe2wwvHnOPXX78AZbodTKzXNmovCm0Defd7MytKcryiftFTRJ8yJZl/YsLd6NooQli9Hm76HcTog72+tlk8Xu/i+UvO8bGH8FoTh3cvAvAmPiN74OP9fkcunHd9MK6gWE7jqOsLrTWMKYkinnDh1lWwXW6jf3/yETd+GkXtpVCCeNLGfgkUgatcEXbz/dQW+D1uVRGmEEWgqzQ5DrbCm196bS1uoHBV73YTbJ11nFcbU8ed4/r4JJTpKwJGA8WX62Xs80gZx83EnXMem38Yyqxs74kY12dkj2K4FSu59frf4L3BbKRs8l5X/OH0o6fBNjeP8XRtzRVtvHb1GpRptnBszMxiv54+jWuGh06iLQxd8ZWlpSUoc/XqVbDduo7iK8bsZIN7Lc7jPF8o67FOC+eSYwuuaGMoKKLU7uK40kQtr167CLabSnx64rF3OsdXrmNc6/QU4aZ75zOyRz5ujEjkTf+BRUGr8bob4zZb2HYdZa25qayLV9tY7h0zbiVmxzGuvHhdEVU7hGuX+RDrHzexHlearujQocPHoMyjk/NgK6+huFS7uQ62bS+GRjUcj5ooXKSIO+avvA42mXafV6xyrWAb40J7HcVEtV8iowLFoXob7vUiRfBrIlTmvh1y119ErbVfFhGc1QgZEejjZNShj5NRhv5NRh36OBlV7idH9GeNMS/eeV1A06YWERFjzKeMMc8aY57d3uQYIg8U9+zjrSZ+S0vIAeauPv5m/2528RdhQg4w9xzDc+VtDEIOMPfs44OBsvUZIfvEbh9E/4WIPCIi50RkUUT+2VsVtNZ+2lr7tLX26YkpfHWWkAPKrny8MY57iBFyQNmRj7/Zv8drmOpAyAFlVzE8VF7VJuSAsisfLyl7PxOyX+zqQdRau2ytze0bOy7/SxF5395Wi5D9hT5ORh36OBll6N9k1KGPk1FgJ2JFgDFmwVq7eOfwJ0Xkpe9W/v/FWrGpmyi8tILJvuMT7tsF2xkUkeWLKAhUKK+NHTv8JNh6mZsM380wUb1QmiYI8VukcoTP8ksdvN7chJtc/Z4cE+GnDqOA0cLDh8D2+m1MOr7wjRed41nllzk7i/VfrWJS9hM/8rfB9jeffsQ5blu878svoOjETIpJzccnUECg6GOieavrnmv7KGIwiN+eb6936+N5YWWz4ybIB4qIR566r8b0E2ynOAaTpBVM3N8oUAhmuesKDDXXMBG+XsEk+u2mInKliC1tJ1iuVHZ/LWspr/+sb2HCfzlG4YG5mcNgGx93hZSUZlXFw0Tx1VYLfdW/XlFg4Dk8Nwu2dhd9t1DUNTTxo7aX9N9rYex47B2nwCbyFcV2b+zGx8uVWB553BVW+dPf/2Mo1zrvik4MWui3D0/jW2TXz2MVTh4/A7axyI0Fmi9oAjMm1GwYQ6IAY6X1X9lUQk8RoM8EKfZptrIGts4Fdxy3V3GsxF0UJspLWJFKGdu26Lr1zwNMIzAhjsVeE+OuNHBMrSniNNa45bJYEaJRxL72gl2vU0Sk8L6jV8WwvOMYLCIf/fBHwPbh7/sE2Ko1Zb725ojLly9BmWtXUMzkyXfimufRs2fBNjaGIo3+OOr3MZ4uLi2C7XP/678C2+2tq2Bb9tpxdYDXH2hxUhFf+c63vw22H/qE27b1CXyDo2dxYjUh+rMmAtNpY3rZ0tIV5zhUxJZ63T0R5AJ26+OhEWl4a6eZGYwZc4fcNUJzG9fYa0qc2u4rgjfKHPBi2xV2erKCIm5T0yhWaqbx7cpsFQWzoghj76Nl1++r2q/DTfS3XPGHQlm35YnbRllXeQ26hGvgWh3XY3YD433ad30pDxV/UwTmFF1SyQfol4lyT6tev48vYwyYmkfRp51y1wdRY8zviMhHRWTWGHNTRP47EfmoMeacvCF5dlVEfmbXNSBkn6GPk1GHPk5GGfo3GXXo42RUueuDqLX2pxXzb7wNdSFkX6CPk1GHPk5GGfo3GXXo42RUuR/VXEIIIYQQQggh5J7hgyghhBBCCCGEkKGyK7Gi3WKMSOiJRdQbDSg3NuXaMkVMoqhiYvXNNj5Xf/sKiiFt91yRhrCOyb7lQBOYQVttAuvROPw42M79oCsANDWNohD1+TmwLZeqYItrmOi88PQHnWOboGjGqkURgOcuK4nOKQqy/OXLbvJ2T0lyXsjQnc6Use/WzqPowqXbmJSdJO65gy4KG9S/FxO895M0z2XZE/wJCvTLSuT2v6KXJb0C2zhXBHQef/odYHvo0Xc6x1deeRXKXH4dxR6WbuJ4CXIUtWgly2AbDJac42oVhSIaDRwvZ08/CrbbN1GQq+cJWjWU2HHlGibRdzsoJlSrojjIUtstp+3iUKviGOr0FNEXRYCpUMZf1nNFBdrLKAYxER2cLYHiOJT5BVdQYryKzru06fqHKWMcyyfxvq5voF+dPP0EVsSbE4wmlKPMbtEk9ouiQ6PGeusVDJTrZ4UiTLSMwkTXv/5VsIWxJ+wjqC5RMRhLUsG42FfEQgbtq74ByozPncDrZzj+w9oC2K73MDbVxBUCCRXhwaooqmz7ihFjPf9S1LBs7vb1wgIK/733ve8HW6wIsrR62F9VT5BrZhaFC5944nvAtnAERQ+THOufpIr4indPRhFonFtAUZJjFYzFkwGuUw6X3L5+NkUfXzE4hgJlPXbzOorTbG66onz1cfTdzOL1Oy0U3+ts4lwYFthPa55YkSjzfZErSjH7SL1Wkw885YpajTUwNvqCieYoxtlrN1GsaLOFAooVZb5utdz2fHkTRYJqh3EdkShri6RSA5sm7jbWdX3EbGN8FmV9WyjjxaY4pxeeTxd9XB+US+gPUUUR8Yxw3vSXwe0KruHXlfVNdgoFy66+dgFsueD4aA7c+jY0Qa5s93vT8hdRQgghhBBCCCFDhQ+ihBBCCCGEEEKGCh9ECSGEEEIIIYQMFT6IEkIIIYQQQggZKkMVKwrCSMYmZhzbyZOoXjA54yblryp53hMhJo1PBZhke7uDtuubbgLwkSVM0i4pYg+aoEz/IiYiL1pM6L5cf7dzPG6w6R/zM8NF5KXbmDSd9DFxv7Xt1rfTwvpvd7GtjWDS8alJrFt3wxW26G1gYvLV60tgW7yOiewrN1EYoJ8oglSe2kgc4Wf+4Pd9BGz7SZqlsrzutkPJYML5RH3aPVaEAvIc+zA2mJBfq6HfHPGELR45gUIXzRYKjixex8T93/udr4Gt30YBoEdOPeIcz0wfgTKFxcG8toZ+s7aGojXjE67IR7fXgjJJguOl10dbv4/1D7wEfEWrSLY38byswPFYBNh3YYTCH6ZwxQ4WZlDM6XDj4IgVra6uyad//V87tt6NK1CuN+0Kr/3IJ34IyoQGff6pJRzPn/jRHwabKbnjoFAUh8ICezDUxCWUwB4qNhBEijF2Fjlef/P518CWX7uBnznm3kO1jqIw8fwpsDXPv4jXL3BsyIzb3kYRORqfwFjSVYSgEuU+baGIcfniSorYS1w+OP4tIhKYQur++iLAe4s9cal3PIaiazVFWLDfQ5GogRKPbt2+5hw/98wzUOYn//bfBVtjGmN9ocQoRaML1jiBotiW5WhrKWJChSIyORW6/jVZwZjYKuO14gDLdZU12nbLXY8dU8S9ShbXQa2V62ArujgXNiKMC2VPgKmsrOOyDNsCZWKGRxyFcuSQK+SU5bi+8sVFoxj9+bHTD4Pt0nWcv/s5tsug6goY3c5RrOiwIrQVT6CPp10cV1ET+9B64oVWGXsdxXezMvZ9EuIckPTcdau2PqhFisCQImiVzaHYVueQ12ZtvO/NLYztVlnRNMdRmHR2HgXrjh/z1iWKiFmi2HYKfxElhBBCCCGEEDJU+CBKCCGEEEIIIWSo8EGUEEIIIYQQQshQGWqOaD9J5cJlNx+s38cNZy97+xSvWNwId1vZ+LaM+0TL9T6+t/ytK+772R9YWYQy9QzzD5IC3+G2GT7LXyxPgu3b3obuJ7U8zMv4rvqWKO+9R0reQ+TmsOQ55sWVDdb/9Bw22uIVfK/+a8+5GxQv3cBrtdewHQcdzBENLOYixCXlnXOv3OMfeAeUOXMW33HfTwIRKXvZN0WCvtrJ3ByueoS5AJUI3+mfPzQNNqPkXaZ9t7/KZWynQ/MnwXbmzGNgm5rB8ffVLzwHtuVbrt90Wlj/ySnMgdzeQh/pJej3+aY7JtMMc30GCdqU1DXJlPFx8pSbd3L6NOZKpEp+9u1FrH+ri/Fjcxvv6fGn3PyaT/7U90OZSknJ99snttst+aOvfcmx/dyP/iSUe+xv/Q3n+PAc5iMXCbbl/BEs99CT6JPZwG1fo+S/ZBY73mBqjlglz0eqytToV1dLslO+2u0kmK9TOY75nxUvT7w0fRTKjE+hLcS93SWySv5q5F4/izFuVKZw7jLKmMpvo8+frmL+2IW2u3n8vJKTFJUwpu0noRFplFzfiWLs2CNH3DzoM49iPA2V83ItJ217BWwrNy85xyZDP4pK2Oa9FB0zUDapt4o+RDJwzw203FjlM6sLit7Aq1fBVvaSUI2Sg1oqK2uBHta/XkO9hIGXu6ykMkuorOPWbl8GW1RgvJ6oYJ6j8doosoo+gFEERvaVQnJ/javFS083QZR8x4eOoxbE8cMzYLt2A/NGC+sGL1vg2hBbU6TVwXK3+tjGJ2LUIaiKO/d3M7xWO0Jdj2SA16ocPgW2zMuVxOgp0lRiZbPA8X1zFf0yv+U+Q1lFq2D+NM6Z51/GPOi19gbYbBWvF3pzZG8D1yTNTdTL2Sn8RZQQQgghhBBCyFDhgyghhBBCCCGEkKHCB1FCCCGEEEIIIUPlrg+ixpjjxpg/M8a8Yox52Rjzj+7Yp40xnzfGXLzzNyZ/EfIAQB8now59nIwy9G8y6tDHyaiyE7GiTER+3lr7LWNMQ0SeM8Z8XkT+SxH5grX2l40xvyAivyAi//i7Xaif9OXCpVcd20wDhVAuXXPFXVZamDwbKKoTz64oAkCYly5h302Qvx5jwm41xhTjTEncD5UNk6MIBZgei93k51KsbJieoIjB8RLeU6x8f7Bxy72HlZsofnB9AwUmnm9hgnRzGwWMWp6oVB7g9WfqmMx9aAH76dAhFOZZWcPk507qttEP/HVMwA5k9wnSb2LPfDwwRmqRK2gQlbEN/D3i260tKJME2PflENt4cgLLjTfcZPjxGs5N+UDZnNuib33ofU+B7czDc2B77fwN5/jLf/YilHn22Ytg226jcFAQ4z0V3qbp73zqCShz7tw7wZb0cFxlOdo++CH3Pk+eVIRhDIoMbCpiS63ONtiuXb8NtmNHXaGHqUns30ARFdkFe+LjURTLzKzb99/zrndDucOnTjnHRYa+FkXYx8cePgW25gBjVLnvCkxEiuhJXigbkFtFtEVRNDEplstSd06wqTJHpDjhVOdw8/WtDgqyhBW37zsG67V5+xrYCkU1yWY4pgaddfc8RcWrf+0G2DSZlbIi0vfxJ9EPjjfdz3z8MAr6dGJcA+yCPYvhxoj44adSQV+dmKg7x2ONOpRJM/SHgTLPb67dQtuSKy5SUnw8UXyw08fPjA2WMwHGssJbEg4SvFaueMSHf+CTYLs8hSI2r5+/4ByXF9GfbQvXY7kiDPaOd50DW6niChilGfp4qohvbSprozBQYlaIYkVZ7pZLUuxfVdjs3tkzH7ciYj2BNyVES+CJ1AQWb2S8hnPW2YdxnC8uo2hj6q03xiO8Vm6xv9pruF565SbOrxM1rG/sdUZewc9cryiCXC9fAFtrFdfBm7Eb27uCPriV45zeDDHGmEmcJzJvXVhpYF1nKhiLpo7gemZlE+eJF65eAlvlthuLpidQYG76CI73nXLXX0SttYvW2m/d+XdLRM6LyFER+XER+eydYp8VkZ/YdS0I2Ufo42TUoY+TUYb+TUYd+jgZVe4pR9QYc0pEnhKRb4jIvLX2r/brWBKR+bc451PGmGeNMc92W/hLISEHifv28Q7+ekPIQeJeffzN/j3INTF6Qg4O9xvDU+3nN0IOEPfr4632nrxJRsiesOMHUWPMmIj8roj8nLXWeaK01r7xS7+CtfbT1tqnrbVP1xrKRp+EHBD2xMfr+JoEIQeF3fj4m/27pLw+RMhBYS9ieBziK7CEHBT2wscbY/jqJiH7xY4eRI0xsbzh+L9trf29O+ZlY8zCnf9fEBF8WZqQBwT6OBl16ONklKF/k1GHPk5Gkbt+vW2MMSLyGyJy3lr7z9/0X58TkX8gIr985+8/uNu12s2OfOmLzzi2oK98+9iY9GqJvzLFMX6jsx1g0nGUJWCr5K7QRUsRFOgkmMTbt/japc0V8QtFECMp3ORkmw2gzEaKX2R1elhus4VJ3+1tN3k7VwQRCkW8o1BEn8brKB7x8IIrAvDQIyjAcebxh8B26uHDYCtXq2D7yl+eB9v01Ann+PSjp6BMrAie3Ct76eMmCCSqum1lBX2wVHbLSIGvPOYp9r0mYLTZV0SHvOlDv/wAACAASURBVF9mVUEWxU8jRQwpG+C5mn7O8UfdN4L+zokfgzKbG3ifX/2Lb4PtuedeAVvqtcdHPvphKPMjn/wo2KwiTNTttcFWrbrCALnSZnkP22xmGsfL1BSKWhw+hAn+nY5bjzzFuFOu3f+313vl48aKVFM3Zg8CRYyr4/p8FOJ3nlb55alQ/KqSYVzcWnFFcBZvXIcyRnnFMlBEiEA5TETCQPmO1hPqMILnxSXs9yBSBO2U+evV51xxr1Tx0XPf9zGwFf58KSLlUBvbbuwwWr9FSlsocSJbxthUWcN56fCcK2AxSHAOHdy4CbZ7ZS9juBWR3POJRHklvZu4Pm4Ufx4MMPa0myjGc+PK62BbW3KFo0plFPlbXV0HWzg2C7ZqiLEsRleVvHDvodPFfi6U9JPpOsa2J3/oR8A2/m5XYKj68ktQZu0P/xBsJ07ieuNDH/oI2KwnHtnp4dybKSJ93b6ytlOWptp6yXqxwiiCPvqPlNi234299HERA/VWdApFfHEmg/dhFDGeE0dRzHB6Cv23s+SOhTFlrd9R1ue1Eor4bCp9c355DWyPe/VtLuFz+wVlLE8rc3MzwXTDVa//tZegM8W5BjHek8nxLdLcuGu0ag/vuxXgfDg3vQC242cfB1u0NAa2zVuumNrSrUUos7K5+9TLnaziPyQif09EvmOMeeGO7RflDaf/98aYfygi10Tkb+26FoTsL/RxMurQx8koQ/8mow59nIwkd30QtdZ+RUT56vcNPr631SFk+NDHyahDHyejDP2bjDr0cTKq3JNqLiGEEEIIIYQQcr/wQZQQQgghhBBCyFAZqhZ/FIUyN+kKK0Q5ZsxPzrgJunmECbvTY2hbXcXE5FcvoW1xedU5bq/egjLSQeGF3GDSdKAImtgCbQPvjYpyrlyrwGTlNMRE5HJFEaKIXFu1hnWYmMJE8IUTU2B7+CSKDj1y0hUOOnkCE5/HZ2tgi8v4mRsdTHh/8n3vBtvhebce1YoiPqKI9+wnaZbK0rqb/B5G2BelkiuOECqCKf1uC2wDJXG/XkHxJ7HuufEsiqMEishAoAgTNbdwDG0oglnNlivMMVZHcYITZ3GLs8UVxXYb7/36dVe842tf/zqUeeKdeK1QEeoYpChOUau5IgC+kJCISL2EwkSlCMdtocQAX2xJRCTzRKoyRVCstXFw9nw7NDUtP/NTP+3YbIox6ld+9Ved404fRVv+65//b8BWL6FYVqgIfdS8OeLk+BkoEyix0yg2LYZoNuON0UAZs0GgvDmniJdcvnIZbP/xi59zT2thmx0/h/f5xBNPYF2VN/isZ8sVHw1SFOXpL62CTTL00/VVjAnLi686x5vbKNTTUwRx9htfS6RQlFwmp1wBHaP4TNrDsbu1guuNxZsoLpL0XaGd1OK1lm7htQ6fRB/JFfGVSNumJndti4so5DKw6DfJDF5qfALj//xxd06/fPUalKmPY4ydmUMBpk4fffWUJ8iSKap6m12M65kiRqVojElulPWeN+aN+vvOwdqb1lqL96y4Q98T7YwVhatSgPc2N43rjZPHcG5eXnMFbgqLa5lAEfZZXEOxnE0ldq03MYbmsTdOU4xbN5T+esXg+C4ibA9fBEqUOSFU5yb0G0UbSuLIMyprwk0lFpsc6zEzg+PqkUceBltnyn1uW11ahjLryvPXTuEvooQQQgghhBBChgofRAkhhBBCCCGEDBU+iBJCCCGEEEIIGSp8ECWEEEIIIYQQMlSGqvRyZOGQ/NJ/+ynHFvuJvSISFG4ib1xBwZs8wyTkjU4Ctou3UPTkWy9ecI7/8st/AWXWrqOYRJp2wTbooq2kJCdHmSvI0M8xYd4YRRChPga2qRkUamo0XGGAKU9IQUTk0BwqCkzPojBApYyfWfJERHLBJO20g4n825vYT1kZRQwOH8VEdmNcQQxNLKpQhED2kzAMZdrri25vG8p1t7wk/Rzvo6cJZin+1jh2GmyrS+71t7dQJMTmKBIyUUexgHSAfdhJ0H/jyE1ojwsUnjl/+RLYNpp4n0ePnATb0qJbrtdtQplWex1slSr6czJAX9pquf1kFGGKrY4y3mP05yJHMYIsw/gkXh+0mnhPuSIMs18ERSaVputL/+kr34Ryn3v+C87xZA3jzI1PfhJsj5x6FGyJ8nVpWHbjz1gD+0DR4pJA+e5VCyFZjm1uPQGLVBHp8MuIiOTKfHD+m9hmj05MO8fVGfTbr37tS2B76IMfAluYaQIZri3OlPorvtbaxvF58cJrYLtxG4Vz0pbbCTbDeaNvlHGxz1jPKTRxkXrd9WmrCBd2u1toa6EtUJyw8JRKcosxa3kR23xjFWPgiVPHwaborIh465Slm7gOGmBTSE2ZN6o1FC8crLrx85WXX4YyQYxjtKmIXG1uYDtmnk+nFn3r1u0bYOu0cY4OlXGbGxTEERBEUgLPAcT3cf9YRCRNPZ8rlJhhMGYEviCQiJw6fgRsFy7edI6324qwo8V1xNY29v1WC9f6toznfid3fSJQYn0nxPNEEYVU9IVAKM4XuRMRCbTnHsUWBjjYwsi1RUpbRyHa+l0UaGxG6PeTnqCsiMjU/Jxz3JhCkdNDC3it23/+ZbBp8BdRQgghhBBCCCFDhQ+ihBBCCCGEEEKGCh9ECSGEEEIIIYQMlaHmiBoRibz3oLX30vteTtqgwHeb1zZwk+12H9/1Xr19G2wLM25yxPf/wF+DMrevn8BrLeO10j7mjPWVnJrl61ed40ET7zsQzGfIldzBbgfv0wSurVTB7xiqXbR1bymbOw8wD6JacfM9VhYxT3V1DTe/rk3Nge2pD30EbEGGnxl4GwhnqfIOfXzAckSDQMZrbi5BJVI2l6+77/nXlTzoqr9xsYgYJU+oquQzpCX33EGAY6gUlcAWK3lvubJR9FgVE4xi73qF4kei5HtcuoCbU2dd3BB77rCbB1it4diLSliv3GKYs8rm1P3ErW9WYA5ts4v5sskAx7soOdSDVOkDcdusovRlp48b2e8X+SCTzZtuDlpnDTe3PuPlhB6qYr7Yy898HWzHZg/jhyp5Mv4m4T3Fb0sx+ndaYOwslM3stXy83ItRJlHOU/KNtHGw/jrmSr/j1GPOcWkcc/pf/NZXwdb+82+BzSi5ntstL2+8gzlVgwH6fNpE//u/vv0NsJ0toy7BbNXN/e+F+JkdZUP2/caP2Fr+VuH5Ur+PsaHbw/lVyyGMlLxI8XLvBkq+Yy/Bz7xw/iLYZqbQlw4dxrztjU03f/L6leewWkqMbVQxnoZWy6d0731qAvOgbTYNtuVb18EWi5Kb6F1/awtj8+vnXwRbr4v9VAp2uDy2Xh60sqZV0gT3Hb9OoZJXGMduXyd9jA/dAc5rBU7NEil51hPj7jzf7aOPa/nZGtNT6DdJhnVLCnfMaDE7EpyHlWWcGEUTwGen+aBqjqhy71Hk9lOszHNxGfNZKxUl7xXym0W2mzgW+rkboxtjOG6nFxaU6+8M/iJKCCGEEEIIIWSo8EGUEEIIIYQQQshQ4YMoIYQQQgghhJChctcHUWPMcWPMnxljXjHGvGyM+Ud37L9kjLlljHnhzp8ffvurS8jeQx8nowz9m4w69HEy6tDHyaiyk2zsTER+3lr7LWNMQ0SeM8Z8/s7//Yq19p/u9MOarY786ZdckQMtwX9r092stlxHoYtEEVBptVFU4eZN3PD52LFjzvHJE8egzMICbtg6VsPn9suvozDAjau42ff2mieulGPSt1GENLrbuMF9qOyiG3ib3JbKV6BMYwI3qo3LKJKjbVpsrHv9gZJUvqEIA0wewgTmXor3+cS73wW2qWlX6EjbuNdYvNYu2DMft0UmSWfNsZXLmNg9Nu4KRVQiTCSvKaIQpQiFLkyOY6hWda9XGBSX6nVxA2KrCINVDPrbIMX+zzzRnsigv0UhjqvVW7g59bGT2B6HF1x/uHwZx97GBt5TL8fN0DVhgMzbzB028xYRk2Nyf2yUjcAVYZG4jGoHjarbRtoG1lsDjAH3yJ75d2FD6Wau7545/giUC8tum0yOoXjC5WvXwDaubJRtlDZZu+jGt6vPoQBJqswRiSIc1GsrgjJ9HAd9z6YJ+/hlREQ6KZZ7beky2Ozxk87xtEUfvbCKgnC/+m9/C2wbHWXD964rFNTLsF4fnHsYbE9Po3Df2cos2A5XUfwm88aByXFMxYEmonHP7JmPi4gUnuBMpojp+f7V7eL6Y7uJ/aCEHolj9HHrhYtCicNz8yjulXRRKaa5gXPz0SMTYLtx+bxzvHTtO1DGRFjXbhOvvziHfjM+7s5DE1WMieUpXO9t3sBxlXUxLo7V3PXM4iKWWbmOQmGmwP61StwJrDL/esJpRhHCkx2I2uyAvVun2EL6yhrOx58nN7fRx28sLuH1NaGnQhHj8a4fl1B4Rxt7voiSiMhYHQUOKwMs18ldW5Er4mGFpi6lrDUVkbzAG6dGUarSbL4IkYguVuTbwgjvMVKEKMMQbSWlHU2IdRt44ndbLZwzq4po0k6564OotXZRRBbv/LtljDkvIkd3/YmEHDDo42SUoX+TUYc+TkYd+jgZVe4pR9QYc0pEnhKRv/pZ82eNMS8aY37TGINfZRPygEEfJ6MM/ZuMOvRxMurQx8koseMHUWPMmIj8roj8nLW2KSL/QkQeEZFz8sa3NP/sLc77lDHmWWPMs+3mfb9iRsjbxl74eLd999ddCNkP9sK/mwNlgzhCDgh74eOpsnc3IQeFPVmLdxjHycFhRw+ixphY3nD837bW/p6IiLV22VqbW2sLEfmXIvI+7Vxr7aettU9ba58eG8c8NUIOAnvl47WxPcl3ImRP2Sv/Hi9h/hYhB4G98vFYyZEi5CCwZ2txRXeFkP3irjmi5o2s2t8QkfPW2n/+JvvCnXfWRUR+UkReutu10ryQpS032TlRxB1W11zBkeZVFBxKFAGIXrsFtizBX6gWr7lCF899GcVejJJcnisiPhvra2DbXL4BtsATIgqV7wC0bwUiRaBAzXvP3ITrQY73vd5HoQtNOSFQauInOue5krittc/SIti++Ed/CLbF61fB9tBjjzvHpx47C2Um5lA0417ZSx8PAyPTNTeZ3Bjsi3LhilikCYq53NrEJP1C6dcwwnavxJ4IgEGRkCjAJP1qBfs+VX4FSxQBnXLsCu80avjF08ptHLfHT6Ko0Y/9jafBtrrsju+Lr6Io2OYG1iuvYZs1t7VvhF0fL0XYJyWD4gGF4ve1cU24Bds7Sd36DgYoBtFNFDGde2Av/VuslSBzfSnKcOGee657ZWUZynzl8qtg6/2P/wPWXwmBS1ddoaO1azehTKbEqFQRpkgUMQyclfB6fUW8ovBvXEREea7RBK4anm7ERgvns2e2sR2XV2+DLbOK0JYnkOGLrIiIbN28ALauImjx1PgRsEUlZbx451YtjvUiUgTz7pG99HFrrdjC9RNNMKXtiVz1+hhT2m0lHg3Qu6pVFPrwhVumZ1H859QpFAq7chXFY6TANU6eoP8u3rjuHJsU6x8b/LK1tfw62DqrV8EWBu49BQH6lvrrSIL1HyiCXKnXB1YRxyoL2gZKbC58tSgRsZqQo3+sBCxtjrhX9tLHjQmkUnHHXV9ZK/uCbKuKGOCVRRQD7GfYnoFiyweuD2oiPhqasI9VFsZRiP7VSN3+0dayhVHqr1w/2IEQUUkRYNIoCuUzFbGi2Lv3QLlHzaYJH4lWf2UERt5zgjYOkh6u7XbKTlRzPyQif09EvmOMeeGO7RdF5KeNMefkjceiqyLyM7uuBSH7C32cjDL0bzLq0MfJqEMfJyPJTlRzvyLqd7ryx3tfHUKGD32cjDL0bzLq0MfJqEMfJ6PKPanmEkIIIYQQQggh9wsfRAkhhBBCCCGEDJWd5IjuGWEUyfj0IcfWbGLycyNzE2FLVVT4WrqNgkB2gMnW3a11sG2vuqI9yTaWyZRraSpB5RibsBZjUnCWuGIHtsAEaaMIoQSKmJCW0O0LTxjlPAkVmyJYoSUr+0nT5SoKFhiL5+WKeMegi33+yvPPgW1z3e2XUBHNqE1OgG1fsVbyzE3a1togz11Rhc0NFOSKozrYEkV8YaBd3xOUGTRR8GZ6cgxsx46i+FOqiFoMMrQlPfcz+p1VKHPxIgpmvec9p8A2OYniFLFx22N6AuNCSfH7chXvM+tgO+aemEI53JkYzcbWJtiiPtpSRRTBerbxBgq+ZBkKGO0XeZbJ1qrbr4UivHNr1RXVuZmgoMViG+PAv/n3/wZsmghF4b2hFgrGzijE2Fwuo3CEUbbryJVY6QtkVDQRokgRplBUWGPlDbvVltvPgcV4d3ZyHmzHFeGtUInroXdPNlDatUD/Xlq5CrbLXRTcOTI2B7bxyWnneHIS48vGNgr+7Td+y2hiRaveOJiZm4EyHUVAcUsRODQFxqNKxY1bT577IJSJSyj0tL2NcbdaQaG/1haOv7Vl99zQYsyKFWGfyCryXpqgYeaOIVsoojNKDI+UNUmR4We2mq6Akbbk8YX8RETaO4gxb1QO79165ayiS1Qo6719xYiisoR9seH5yFYT56JQEcYxynhJUm3OddvFKmvbUFljG62RI2X9aRWBHk+ksdCEqpTYGCo+Emvrc89Wr+GcXqnh2m5TGY9WqZv/TKCKFSkiYJowqe+7IiL4iejTRhsbOxSa0uAvooQQQgghhBBChgofRAkhhBBCCCGEDBU+iBJCCCGEEEIIGSp8ECWEEEIIIYQQMlSGKlYk1krhJeXHirDP1LQrQNNu4vPy1NQ02EyhiE6kmCTe9oRbSnUlIT9GsZSkg8nEWR/LBYp4hy3cRN5CS7bWztOKKUnTvlBQoJXRbEqCsdVy9P1uUhLDoxIKGIUFlstaKASV9FHUYen6Vef42sXXoMz8qVNg2096/YG8cvGmY6uNYbtM+UJBigjORAPHxkDTPAgxGT6KXVu7hoImR+YPgS3poY/nuSZohaIspbjhHF+/ch3KbLW2wHZ24jTYNtZRcMNmk24dAhQ/WF9GMaRHJo5jXRUhjXbi+mARDaBMq4u+mxaKaMY2ihX1U7xe1WszTcSslxwcsSIRK4Fx6xgqwghnpo45xzMDFBX73jlFbAq7VHJFb6I+N+Uczz5yDMpEMfqo7eI4622gz4vSz5HXNVmBFauqYkUYAxvj43j9E+49PfzU90AZ28MG6rZQiCzZRtug7fpRZwtjbm8ZRaU2N9CXmynOe4UibDM5787Tm2tLUOax9z4ONvkqmoaHkby4+3za3HRj2Y1LV6FMmuOY77RwPCd9jM/T8yec48PHMU5evXwZbDbFfp2YQMG2laVLYNtad/snCnHuypUFglXEwpQmA1sQaoVQLkUTXxwIjuV+7o7bcgXFnEJFvE5bZ4XKnJwWmlCkX7e7r//2HSuSe+vltXWMg7eX3HhgFUHKSoSPEYM++n2qCNwYT4BS+2XMKp2jiRWFijCRzbFuubm7cFSo+FtoFL9U1uwVz+fKZWUMGbx+XEVfLQpFHMu7dxPgfYeh0hZg0Z9DtHvfifcW6ifsDP4iSgghhBBCCCFkqPBBlBBCCCGEEELIUOGDKCGEEEIIIYSQocIHUUIIIYQQQgghQ2W4YkVGJPISfjPBxOEJT8glzzFhdzY6AbbJ6cNgmz+KIgDHHj7jHC/fvAJlBh0UVUnbKNqwdgsFWdIuipdY44sVYSK8lphslMT9QMsc9sWKDCYrx4pohpqFrNiMJ05kQyyUKXXVBJiispJIrZyaJ247Lt+4AWU2V9fwxH0kCCKplFwRoOs3bkK53pbbLtUSDsV+gj44pYhO1AMUfUk6rlhAEaC4yMa20nY5ipwUOQqYHDm8ALZ+2xVIWrqF5z32xENga3abYFvawjGUeoIeieKn1Tq2TyXGsZbkGBcWm257b6+g8NGgj/4cGOy7UgkFfHp9rMeg5NajN8Ay3S723X5Rm5qQd/3NH3Rs66+jAE3xsjvGjyniCaUSCvtUJtCXTQ3bcv5h1/9OvOcslMkVoZICm1fyHgoTZUoMv/X/tHcvIXbVdxzAv7/7mjuPTCaJIVVjW20FkdIqiCh0FRCsLWjBRbtyIXTTRUu7EQoFoRsXravSIljMojS2tqCUtqASKF1UK9VIfNQkvhKdV5KZzOvmPn9d3DNxzvn9MnNncuec8z9+PzBk7smZe3/n3O95zdz/75yMN0tbW7DHg1LX5qM2OW6nOcteqpUTj+2x8bLT1G3vLbYRVKV8wExrr8aXqVSxTTRW5+22OP2u0xCn5WyfYhuULFyYjj2+tPCRmaf3nt1PZEkBJPseVpyDbrsZX96ZT6fNPOodE50DbNdpntZsx3921jnWvfvWCTNt36TdB3qtSubnbb2Ntfj+v+z8rcKJuNumpOQ0MJLEekyeF131yZzZ2s7G3O3F3xPvnGdkzFs/9gVKbj9J+3ya+Fl1zuMGa/eSnnang9nZC7FpM3N2O1xJHnuc7cBreNlp26ZqHaf5ExLnqV5jIrfnlXPCOOCpLCreebCZx2Z3YsI59xqzjSKT5/GNhj1+N5z9p7egzukzZIC8dZ3rC3EaJHkv2vVOxgfIr3f9Mij+RZSIiIiIiIhSxQtRIiIiIiIiStWWF6IiUheRV0XkhIi8JSKPR9NvFpFXROS0iDwrIs4N1IjyjxmnomPGqciYbyo6ZpyKapAxok0AR1R1RUSqAP4lIn8H8BMAT6rqMRH5LYBHAfxm02dSQBODL3odOzYi+RHlAwf2m3lazliq1SU7fmZs1I6D2T8Zv4n8nrqdp9Gwz3Vx1o71azZtHSsXL5hp3dX480nb+Ty1O0bU+5C49zn9+GPvc+Odtq217NyUXZ0PpovGP/Nfrns3vXXGhJgpQKlsx3y1e86Ygk58Gc7Pz5p53j/1npm2A0PL+Githq998abYtOsn7bi3PaPxY0XDGzPsjW1s2JtO15y1vHfvVOzxofoeM8/igh0TMjpu52vZoV9ortpt5sMz8bGCe52xcfv32zEVZ6Y/NdNKI3adVcfir7nv0JSZ5+ycHfc0t2DHFmvN1tZsx8fazXxq87a6bMfQ1p0bVk9M2PXY7tj3qVGJZ7zt3Iz+WsZebDCcjJe6KI/H18HUYZvT2Q/i2Rop2UPNyBfs+zc5YfcrzYV5M+3cyf/FHp8/+28zT2XSvgeTB68z0+D0IKhN2PzVavFtryuf2Kfq2vGmrVW7X1xZtPu7Rju+oX34sbOfv2TzN+KMN0XF7mOT473GJibNPB1nX7Jwwe4nes44ouVlu29aW4xPKzvH7XPTQ/lg1hDPU9QcP9UZX5XcLlst+957B0DvZvOeUuLG8jMzdiz29Md2/G7pBtsrY8EZz7y0bPc1ybGqTiuIgcfx+ecpA4yVdHtg2EnVil2PjUQGS8653aSzb+717Iv21D172bI2b4idOs+/A0PLeK+nWFmLb4uXnXPZauLY5o3zLDt5GB2z+89S1Z7r9xLXA27/E7dpiTfJyZtTWzKD3vaY3PYAoF63+9mJCec8ohnfD3jjZcdLzmuW7THS3dacaYNxMu6cW6izsZXM/s/Ok3wvt2PLI4D2rR/9qtGXAjgC4Llo+lEAD+24CqIMMeNUdMw4FRnzTUXHjFNRDfSrSBEpi8gbAOYAvAjgDIBF1St/xjkH4MbdKZFo9zHjVHTMOBUZ801Fx4xTEQ10IaqqXVW9A8BhAHcDuG3QFxCRH4jIayLy2uqS/egOUR4MK+MrK/m51QbRRjvN+MZ8L67Yj4YS5cGw9uGda/iIGdFuGlbG83RLMKJtDc5Q1UUAxwHcC2BK5MoN9A4DsANm+j/zlKrepap3jU/a+50R5cm1Znxiwo6BJMqT7WZ8Y76nnHupEeXJte7DvXuGEuXJtWZ8zLn/JVFWtmxWJCIHAbRVdVFERgHcB+AJ9DeChwEcA/AIgOe3ei5VNQ1zqk5TBXNzZ+8XlD2nyVHXDgqGc6PoWmKQ+6TToKVed+pq2t8itQ6tmWllp+BaLb6qV5ds84BmY9VM+2z/8hlvrHLJ/E7B6zLgDFZ27k6tTuOcVqKhR7LpVL8Iu/5LJTsY2muA0HXq6CVe8/Jl58bAq3adbdcwM97pdrFwaTE2bcIZuN+V+DqembM3K6/vcTZPtTdClqbNuBlYv2abNkjZrvPzS/aG9jOztraRktOUpRV//2/4iv2E0OyibXwyf8ku08i4PVBWq/Hn15pdpsUl+xe75XmbkU7PZqmt8fWtap9/1GlY0O3Y7eXCefua3nY7MRl/jcqIXe4lp9HIdg0r42sXZ/H6sV/GpumoXf5Rje9juyW7P7180e4bLs7aLFSrdp03E41h2pdsR636SN1Mm/vA/u617OwXu14zhuQb6Pwat9d0mq55zR+cm6q3evH1ONpzGjc5jVBKy/b56xWbo2aittZ5+3OXW04TpZ5dtz2n8Racw+/kSHy+htMYpOHUul3D3IdDxDQrGaTHjntcdi5q3WZFat/rS4lmQgsnTph5Kuo1x1o0086e/dhMu+A0oUrymguVvJXhzec8X7LRitcYxZtUrtr9x/io3b4/PPVu7PE+p2EZvN5vzvpXr1mR07TKvu+D9ADdvmFmfO/effj2d74bm7a85px/JrJacfZbbacJZqtjV7LXnzO58ipOaLwMetuau4m6jX3ic3rbqKfkbLdes59u4trEr99prLTDpkzqBNrfruy0SsVmdWpqn33NQXaAjpf+8beB5htki7kewFERKaO/b/mjqv5VRN4GcExEfgHgdQBP76hSouwx41R0zDgVGfNNRceMUyFteSGqqm8CuNOZ/j76n1EnChozTkXHjFORMd9UdMw4FdVQbuBFRERERERENCheiBIREREREVGqJDl4fFdfTGQewEcArgNgO6CEI/T6gfCXYbP6v6SqB9MsZh0znhtFrz+TjG/IN1D8dZx3Ra4/D/twoNjrOARFrj8PhY/pqQAABGNJREFUGS/y+g1B6PUDQ8h4qheiV15U5DVVvSv1Fx6S0OsHwl+GvNef9/q2wvqzFUL9IdS4GdafrRDqD6HGzbD+bOW9/rzXtxXWn71hLAM/mktERERERESp4oUoERERERERpSqrC9GnMnrdYQm9fiD8Zch7/XmvbyusP1sh1B9CjZth/dkKof4QatwM689W3uvPe31bYf3Zu+ZlyGSMKBEREREREX1+8aO5RERERERElCpeiBIREREREVGqUr8QFZH7ReR/InJaRB5L+/W3S0R+JyJzInJyw7T9IvKiiJyK/t2XZY2bEZGbROS4iLwtIm+JyI+i6UEsg4jUReRVETkR1f94NP1mEXklytGzIlLLutZ1zHi6mPF0hZZvIOyMh55vgBnfbSHnGwg/46HlG2DG08aMX12qF6IiUgbwawDfAnA7gO+LyO1p1rADzwC4PzHtMQAvq+qtAF6OHudVB8BPVfV2APcA+GG0zkNZhiaAI6r6DQB3ALhfRO4B8ASAJ1X1qwAWADyaYY1XMOOZYMZTEmi+gbAzHnq+AWZ8tz2DcPMNhJ/xYPINMOMZYcavIu2/iN4N4LSqvq+qLQDHADyYcg3boqr/BHAxMflBAEej748CeCjVorZBVadV9b/R98sA3gFwIwJZBu1biR5Woy8FcATAc9H0PNXPjKeMGU9VcPkGws546PkGmPHdFnK+gfAzHli+AWY8dcz41aV9IXojgLMbHp+LpoXmkKpOR9/PADiUZTGDEpEvA7gTwCsIaBlEpCwibwCYA/AigDMAFlW1E82Spxwx4xlixnddUfINBJSPdaHmG2DGMxBUPtaFmvGA8g0w45lixuPYrOgaaf/+N7m/B46ITAD4M4Afq+rSxv/L+zKoaldV7wBwGP3f5N2WcUmfK3nPxzpmnHYq7/kAws43wIxnKYR8AGFnnPnOVt7zsY4Zt9K+EP0EwE0bHh+OpoVmVkSuB4Do37mM69mUiFTRD/7vVfUv0eSglgEAVHURwHEA9wKYEpFK9F95yhEzngFmPDVFyTcQUD6Kkm+AGU9RUPkoSsYDyDfAjGeCGfelfSH6HwC3Rl2WagC+B+CFlGsYhhcAPBJ9/wiA5zOsZVMiIgCeBvCOqv5qw38FsQwiclBEpqLvRwHch/5n648DeDiaLU/1M+MpY8ZTVZR8A+HkI+h8A8x4RkLKR9AZDyzfADOeOmZ8E6qa6heABwC8h/5ni3+W9uvvoN4/AJgG0Eb/88+PAjiAfnerUwBeArA/6zo3qf+b6P+p/00Ab0RfD4SyDAC+DuD1qP6TAH4eTb8FwKsATgP4E4CRrGvdUDMznm79zHi69QaV76jmYDMeer6jZWDGd7feYPMd1R90xkPLd1QbM55u/cz4Vb4keiIiIiIiIiKiVLBZEREREREREaWKF6JERERERESUKl6IEhERERERUap4IUpERERERESp4oUoERERERERpYoXokRERERERJQqXogSERERERFRqv4PrMzsEaiEPJgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1152x432 with 5 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "(x_tan,y_tan,x_tst,y_tst,cimke)=mintak_cifar10()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.) Transfer learning alkalmazása"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sok esetben nincs elég tanítóminta a konvolúciós rétegek tanításához. Gyakoran alkalmazott forgatókönyv ilyenkor más, a feladatunkhoz lehetőleg legjobban hasonlító, de elegendő nagy mintazsámú halmazon betanított konvolúciós rétegekből álló háló, mint jellemzőkiemelő háló alkalmazása. Ennek kimenetét adjuk általában egy 1 rejtett rétegű MLP-nek, melynek feladata ebben az esetben a kapott jellemzők alapján a konkrét probléma megoldása."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Önálló feladat - transfer learninges háló létrehozása"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Egészítsük ki úgy az alábbi kódrélszletet, hogy a 121 rétegű DenseNet-et transfer learninges jellemzőkiemelőként használja fel. Figyeljük meg, hogy a Keras által külön nem támogatott, de a Tensorflow API-ja által tartalmazott, képeket átméretező réteget hogyan hívhatjuk meg (lambda rétegként)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def atmer(kep):\n",
    "    return(K.tf.image.resize_images(kep, (128,128)))\n",
    "\n",
    "from keras.applications import mobilenet\n",
    "\n",
    "def get_tfn(x_t, y_t):\n",
    "    bem = Input(shape=x_t[0].shape, dtype='float32')\n",
    "    x = Lambda(atmer, input_shape=x_t[0].shape)(bem)\n",
    "    \n",
    "    rnt = mobilenet.MobileNet(input_shape=(128, 128, 3), include_top=False, input_tensor=x)\n",
    "    \n",
    "    y = MaxPooling2D(pool_size=(2, 2))(rnt.output)\n",
    "    y = Flatten()(y)\n",
    "    y = Dense(120 , activation='tanh')(y)\n",
    "    y = Dense(y_t.shape[1], activation='softmax')(y)\n",
    "    \n",
    "    for layer in rnt.layers:\n",
    "        layer.trainable=False\n",
    "        \n",
    "    model = Model(inputs=bem, outputs=y)\n",
    "    model.compile(optimizer=SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy', inTop3])\n",
    "    \n",
    "    print(model.summary())\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 32, 32, 3)         0         \n",
      "_________________________________________________________________\n",
      "lambda_2 (Lambda)            (None, 128, 128, 3)       0         \n",
      "_________________________________________________________________\n",
      "conv1_pad (ZeroPadding2D)    (None, 129, 129, 3)       0         \n",
      "_________________________________________________________________\n",
      "conv1 (Conv2D)               (None, 64, 64, 32)        864       \n",
      "_________________________________________________________________\n",
      "conv1_bn (BatchNormalization (None, 64, 64, 32)        128       \n",
      "_________________________________________________________________\n",
      "conv1_relu (ReLU)            (None, 64, 64, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv_dw_1 (DepthwiseConv2D)  (None, 64, 64, 32)        288       \n",
      "_________________________________________________________________\n",
      "conv_dw_1_bn (BatchNormaliza (None, 64, 64, 32)        128       \n",
      "_________________________________________________________________\n",
      "conv_dw_1_relu (ReLU)        (None, 64, 64, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv_pw_1 (Conv2D)           (None, 64, 64, 64)        2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_1_bn (BatchNormaliza (None, 64, 64, 64)        256       \n",
      "_________________________________________________________________\n",
      "conv_pw_1_relu (ReLU)        (None, 64, 64, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv_pad_2 (ZeroPadding2D)   (None, 65, 65, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv_dw_2 (DepthwiseConv2D)  (None, 32, 32, 64)        576       \n",
      "_________________________________________________________________\n",
      "conv_dw_2_bn (BatchNormaliza (None, 32, 32, 64)        256       \n",
      "_________________________________________________________________\n",
      "conv_dw_2_relu (ReLU)        (None, 32, 32, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv_pw_2 (Conv2D)           (None, 32, 32, 128)       8192      \n",
      "_________________________________________________________________\n",
      "conv_pw_2_bn (BatchNormaliza (None, 32, 32, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_pw_2_relu (ReLU)        (None, 32, 32, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_3 (DepthwiseConv2D)  (None, 32, 32, 128)       1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_3_bn (BatchNormaliza (None, 32, 32, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_dw_3_relu (ReLU)        (None, 32, 32, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_3 (Conv2D)           (None, 32, 32, 128)       16384     \n",
      "_________________________________________________________________\n",
      "conv_pw_3_bn (BatchNormaliza (None, 32, 32, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_pw_3_relu (ReLU)        (None, 32, 32, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_pad_4 (ZeroPadding2D)   (None, 33, 33, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_4 (DepthwiseConv2D)  (None, 16, 16, 128)       1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_4_bn (BatchNormaliza (None, 16, 16, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_dw_4_relu (ReLU)        (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_4 (Conv2D)           (None, 16, 16, 256)       32768     \n",
      "_________________________________________________________________\n",
      "conv_pw_4_bn (BatchNormaliza (None, 16, 16, 256)       1024      \n",
      "_________________________________________________________________\n",
      "conv_pw_4_relu (ReLU)        (None, 16, 16, 256)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_5 (DepthwiseConv2D)  (None, 16, 16, 256)       2304      \n",
      "_________________________________________________________________\n",
      "conv_dw_5_bn (BatchNormaliza (None, 16, 16, 256)       1024      \n",
      "_________________________________________________________________\n",
      "conv_dw_5_relu (ReLU)        (None, 16, 16, 256)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_5 (Conv2D)           (None, 16, 16, 256)       65536     \n",
      "_________________________________________________________________\n",
      "conv_pw_5_bn (BatchNormaliza (None, 16, 16, 256)       1024      \n",
      "_________________________________________________________________\n",
      "conv_pw_5_relu (ReLU)        (None, 16, 16, 256)       0         \n",
      "_________________________________________________________________\n",
      "conv_pad_6 (ZeroPadding2D)   (None, 17, 17, 256)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_6 (DepthwiseConv2D)  (None, 8, 8, 256)         2304      \n",
      "_________________________________________________________________\n",
      "conv_dw_6_bn (BatchNormaliza (None, 8, 8, 256)         1024      \n",
      "_________________________________________________________________\n",
      "conv_dw_6_relu (ReLU)        (None, 8, 8, 256)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_6 (Conv2D)           (None, 8, 8, 512)         131072    \n",
      "_________________________________________________________________\n",
      "conv_pw_6_bn (BatchNormaliza (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_6_relu (ReLU)        (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_dw_7 (DepthwiseConv2D)  (None, 8, 8, 512)         4608      \n",
      "_________________________________________________________________\n",
      "conv_dw_7_bn (BatchNormaliza (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_dw_7_relu (ReLU)        (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_7 (Conv2D)           (None, 8, 8, 512)         262144    \n",
      "_________________________________________________________________\n",
      "conv_pw_7_bn (BatchNormaliza (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_7_relu (ReLU)        (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_dw_8 (DepthwiseConv2D)  (None, 8, 8, 512)         4608      \n",
      "_________________________________________________________________\n",
      "conv_dw_8_bn (BatchNormaliza (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_dw_8_relu (ReLU)        (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_8 (Conv2D)           (None, 8, 8, 512)         262144    \n",
      "_________________________________________________________________\n",
      "conv_pw_8_bn (BatchNormaliza (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_8_relu (ReLU)        (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_dw_9 (DepthwiseConv2D)  (None, 8, 8, 512)         4608      \n",
      "_________________________________________________________________\n",
      "conv_dw_9_bn (BatchNormaliza (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_dw_9_relu (ReLU)        (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_9 (Conv2D)           (None, 8, 8, 512)         262144    \n",
      "_________________________________________________________________\n",
      "conv_pw_9_bn (BatchNormaliza (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_9_relu (ReLU)        (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_dw_10 (DepthwiseConv2D) (None, 8, 8, 512)         4608      \n",
      "_________________________________________________________________\n",
      "conv_dw_10_bn (BatchNormaliz (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_dw_10_relu (ReLU)       (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_10 (Conv2D)          (None, 8, 8, 512)         262144    \n",
      "_________________________________________________________________\n",
      "conv_pw_10_bn (BatchNormaliz (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_10_relu (ReLU)       (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_dw_11 (DepthwiseConv2D) (None, 8, 8, 512)         4608      \n",
      "_________________________________________________________________\n",
      "conv_dw_11_bn (BatchNormaliz (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_dw_11_relu (ReLU)       (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_11 (Conv2D)          (None, 8, 8, 512)         262144    \n",
      "_________________________________________________________________\n",
      "conv_pw_11_bn (BatchNormaliz (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_11_relu (ReLU)       (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_pad_12 (ZeroPadding2D)  (None, 9, 9, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_dw_12 (DepthwiseConv2D) (None, 4, 4, 512)         4608      \n",
      "_________________________________________________________________\n",
      "conv_dw_12_bn (BatchNormaliz (None, 4, 4, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_dw_12_relu (ReLU)       (None, 4, 4, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_12 (Conv2D)          (None, 4, 4, 1024)        524288    \n",
      "_________________________________________________________________\n",
      "conv_pw_12_bn (BatchNormaliz (None, 4, 4, 1024)        4096      \n",
      "_________________________________________________________________\n",
      "conv_pw_12_relu (ReLU)       (None, 4, 4, 1024)        0         \n",
      "_________________________________________________________________\n",
      "conv_dw_13 (DepthwiseConv2D) (None, 4, 4, 1024)        9216      \n",
      "_________________________________________________________________\n",
      "conv_dw_13_bn (BatchNormaliz (None, 4, 4, 1024)        4096      \n",
      "_________________________________________________________________\n",
      "conv_dw_13_relu (ReLU)       (None, 4, 4, 1024)        0         \n",
      "_________________________________________________________________\n",
      "conv_pw_13 (Conv2D)          (None, 4, 4, 1024)        1048576   \n",
      "_________________________________________________________________\n",
      "conv_pw_13_bn (BatchNormaliz (None, 4, 4, 1024)        4096      \n",
      "_________________________________________________________________\n",
      "conv_pw_13_relu (ReLU)       (None, 4, 4, 1024)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 2, 2, 1024)        0         \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 4096)              0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 120)               491640    \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 10)                1210      \n",
      "=================================================================\n",
      "Total params: 3,721,714\n",
      "Trainable params: 492,850\n",
      "Non-trainable params: 3,228,864\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 45000 samples, validate on 5000 samples\n",
      "Epoch 1/15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 7968/45000 [====>.........................] - ETA: 17s - loss: 0.9499 - acc: 0.6855 - inTop3: 0.9050"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-1a2149f0d7aa>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mtfn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mget_tfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_tan\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my_tan\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtfn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtanitas_earlystop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_tan\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_tan\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m15\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mhalo_mentes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtfn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'cnn_tf1'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-3-8ab63ffff2e2>\u001b[0m in \u001b[0;36mtanitas_earlystop\u001b[0;34m(model, x_tan, y_tan, epn, tol)\u001b[0m\n\u001b[1;32m     26\u001b[0m     \u001b[0mmodel_checkpoint\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mModelCheckpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'CNN.hdf5'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmonitor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'val_loss'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msave_best_only\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m     \u001b[0mearlystop\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mEarlyStopping\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmonitor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'val_loss'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpatience\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m     \u001b[0mhst\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mx_tan\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0my_tan\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mepn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mearlystop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_checkpoint\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_split\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     29\u001b[0m     \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'CNN.hdf5'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1035\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1036\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1037\u001b[0;31m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1038\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1039\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m    197\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    200\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2664\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2665\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2666\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2667\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2668\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2634\u001b[0m                                 \u001b[0msymbol_vals\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2635\u001b[0m                                 session)\n\u001b[0;32m-> 2636\u001b[0;31m         \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2637\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2638\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m   1449\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_created_with_new_api\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1450\u001b[0m           return tf_session.TF_SessionRunCallable(\n\u001b[0;32m-> 1451\u001b[0;31m               self._session._session, self._handle, args, status, None)\n\u001b[0m\u001b[1;32m   1452\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1453\u001b[0m           return tf_session.TF_DeprecatedSessionRunCallable(\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "tfn=get_tfn(x_tan,y_tan)\n",
    "tfn=tanitas_earlystop(tfn, x_tan, y_tan, 15, 5)\n",
    "halo_mentes(tfn,'cnn_tf1')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Önálló feladat - tanítás gyorsítása "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Eléggé lelassult egy tanító epoch futása, melynek oka, hogy a jellemzőkiemelést végző rétegen át kell kergetni a képeket. Mivel ezt minden epochnál meg kell tenni, ezért ez mindig overhead-et jelent. Érdemes ezért a jellemzőkiemelés eredmnényét egy változóban eltárolni, majd csak az ebben mentett bemenetekre a végső teljesen összekötött rétegeket tanítani. Persze nem alkalmazható a módszer, ha augmentálni is kívánunk futási időben."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_RNkim(x_t, y_t=None):\n",
    "    bem = Input(shape=x_t[0].shape, dtype='float32')\n",
    "    x = Lambda(atmer, input_shape=x_t[0].shape)(bem)\n",
    "    \n",
    "    rnt = mobilenet.MobileNet(input_shape=(128, 128, 3), include_top=False, input_tensor=x)\n",
    "    \n",
    "    y = MaxPooling2D(pool_size=(2, 2))(rnt.output)\n",
    "    y = Flatten()(y)\n",
    "    model = Model(inputs=bem, outputs=y)\n",
    "    x_jel = model.predict(x_t)\n",
    "    \n",
    "    return (x_jel, model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Definiáljukl a ResNet fölé kerülő hálót, mérete maradjon az előzőeknek megfelelő."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_tfMLP(x_t, y_t, bem=None):\n",
    "    \n",
    "    if (bem is None):\n",
    "        bem = Input(shape=x_t[0].shape, dtype='float32')\n",
    "    \n",
    "    y = Dense(120 , activation='tanh')(bem)\n",
    "    y = Dense(y_t.shape[1], activation='softmax')(y)\n",
    "    \n",
    "    model = Model(inputs=bem, outputs=y)\n",
    "    model.compile(optimizer=SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy', inTop3])\n",
    "    \n",
    "    print(model.summary())\n",
    "    return model\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 4096)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 120)               491640    \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 10)                1210      \n",
      "=================================================================\n",
      "Total params: 492,850\n",
      "Trainable params: 492,850\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 45000 samples, validate on 5000 samples\n",
      "Epoch 1/15\n",
      "45000/45000 [==============================] - 6s 125us/step - loss: 0.7673 - acc: 0.7430 - inTop3: 0.9381 - val_loss: 0.8054 - val_acc: 0.7246 - val_inTop3: 0.9322\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.80535, saving model to CNN.hdf5\n",
      "Epoch 2/15\n",
      "45000/45000 [==============================] - 5s 119us/step - loss: 0.6917 - acc: 0.7639 - inTop3: 0.9487 - val_loss: 0.6888 - val_acc: 0.7580 - val_inTop3: 0.9492\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.80535 to 0.68880, saving model to CNN.hdf5\n",
      "Epoch 3/15\n",
      "45000/45000 [==============================] - 5s 119us/step - loss: 0.6280 - acc: 0.7858 - inTop3: 0.9548 - val_loss: 0.6225 - val_acc: 0.7888 - val_inTop3: 0.9578\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.68880 to 0.62247, saving model to CNN.hdf5\n",
      "Epoch 4/15\n",
      "45000/45000 [==============================] - 5s 120us/step - loss: 0.5967 - acc: 0.7975 - inTop3: 0.9588 - val_loss: 0.6483 - val_acc: 0.7832 - val_inTop3: 0.9524\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.62247\n",
      "Epoch 5/15\n",
      "45000/45000 [==============================] - 5s 120us/step - loss: 0.6146 - acc: 0.7869 - inTop3: 0.9578 - val_loss: 0.6536 - val_acc: 0.7750 - val_inTop3: 0.9530\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.62247\n",
      "Epoch 6/15\n",
      "45000/45000 [==============================] - 5s 120us/step - loss: 0.5948 - acc: 0.7972 - inTop3: 0.9603 - val_loss: 0.5572 - val_acc: 0.8096 - val_inTop3: 0.9674\n",
      "\n",
      "Epoch 00006: val_loss improved from 0.62247 to 0.55722, saving model to CNN.hdf5\n",
      "Epoch 7/15\n",
      "45000/45000 [==============================] - 5s 120us/step - loss: 0.5649 - acc: 0.8080 - inTop3: 0.9641 - val_loss: 0.5585 - val_acc: 0.8136 - val_inTop3: 0.9670\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.55722\n",
      "Epoch 8/15\n",
      "45000/45000 [==============================] - 5s 121us/step - loss: 0.5497 - acc: 0.8127 - inTop3: 0.9648 - val_loss: 0.5309 - val_acc: 0.8174 - val_inTop3: 0.9670\n",
      "\n",
      "Epoch 00008: val_loss improved from 0.55722 to 0.53095, saving model to CNN.hdf5\n",
      "Epoch 9/15\n",
      "45000/45000 [==============================] - 5s 120us/step - loss: 0.5379 - acc: 0.8184 - inTop3: 0.9641 - val_loss: 0.5767 - val_acc: 0.7986 - val_inTop3: 0.9676\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.53095\n",
      "Epoch 10/15\n",
      "45000/45000 [==============================] - 5s 119us/step - loss: 0.5341 - acc: 0.8176 - inTop3: 0.9655 - val_loss: 0.5575 - val_acc: 0.8126 - val_inTop3: 0.9658\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.53095\n",
      "Epoch 11/15\n",
      "45000/45000 [==============================] - 5s 121us/step - loss: 0.5166 - acc: 0.8250 - inTop3: 0.9683 - val_loss: 0.5571 - val_acc: 0.8166 - val_inTop3: 0.9656\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 0.53095\n",
      "Epoch 12/15\n",
      "45000/45000 [==============================] - 5s 119us/step - loss: 0.4929 - acc: 0.8321 - inTop3: 0.9706 - val_loss: 0.5197 - val_acc: 0.8222 - val_inTop3: 0.9694\n",
      "\n",
      "Epoch 00012: val_loss improved from 0.53095 to 0.51967, saving model to CNN.hdf5\n",
      "Epoch 13/15\n",
      "45000/45000 [==============================] - 5s 118us/step - loss: 0.4842 - acc: 0.8333 - inTop3: 0.9716 - val_loss: 0.5364 - val_acc: 0.8234 - val_inTop3: 0.9664\n",
      "\n",
      "Epoch 00013: val_loss did not improve from 0.51967\n",
      "Epoch 14/15\n",
      "45000/45000 [==============================] - 5s 118us/step - loss: 0.4721 - acc: 0.8388 - inTop3: 0.9729 - val_loss: 0.5091 - val_acc: 0.8236 - val_inTop3: 0.9732\n",
      "\n",
      "Epoch 00014: val_loss improved from 0.51967 to 0.50905, saving model to CNN.hdf5\n",
      "Epoch 15/15\n",
      "45000/45000 [==============================] - 5s 118us/step - loss: 0.4748 - acc: 0.8381 - inTop3: 0.9708 - val_loss: 0.5103 - val_acc: 0.8320 - val_inTop3: 0.9736\n",
      "\n",
      "Epoch 00015: val_loss did not improve from 0.50905\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xd4VGX2wPHvISSETmjSBRXpbICIBQuKAjaEXRH84a6ALrs2FCs2UCyLZdVVURddV9ZVEAGxoawKil1CEQWko4ZmAGlSA+f3x5nAEFImyUxuyvk8zzyZuXPvnTNR7sl9y3lFVXHOOedyUy7oAJxzzhV/niycc87lyZOFc865PHmycM45lydPFs455/LkycI551yePFk4lwMRuUJE3g46juyISE8RWV7Ux7qyy5OFK9FEZEfY44CI7Ap7PaAw51bVf6nqhaHPSRQRFZFG0YncuZKlfNABOFcYqlol87mIrAauVNUPg4so9kTE/926Iud3Fq5UE5EuIvK1iGwRkbUi8njmxTbsbmGIiKwQkV9F5PGwY/8qIpmJZ1bo55LQXUvv0D7XhI7dJCJTROSoXGK5UkR+FpF0EblVRNaLyKmh9yqKyBgRWSciaSLyiIjEh97rKSLLReRuEdkAPBt2zntFZLOIrBKRvmHbK4rIE6HPWy8iT4lIhRziukVEFohIvQL9kl2Z4MnClXb7gGuBWsBpwIXAlVn26Ql0ADoCg0SkazbnOT30s4WqVlHVqSJyHnA30AdoCGwEXs4uCBHpADwG9AUahR61w3a5F2gPtAM6AV2BW8PebwrEA42BoWHbEoB6wJ+BcSLSLPTeY6HPaAe0AI4HhmcT14PAxUBXVV2fXezOgScLV8qp6jeqOltV96vqCuAF4Iwsuz2oqttUdRV2B5Ec4ekHAGNVdYGq7sYu7mfn8Bd6X2Cyqn6lqnuAuzj8398AYKSqblTVDcD9wB/D3t8D3Keqe1V1V2hbBnBvaNuHwIfAxaE7pyuA61V1i6puBUYD/cPOJyIyBjgZOFtVN0f4nV0Z5W2frlQTkdbA37G7horY//OfZ9kt/C/qnUAVItMAmJH5QlW3iMg27C4j61/pDYCfwvbdJiJbQzEKdnfwY9j+P4bOczBGVd2X5ZzpoSQVfkyD0CMeWGinBkCw5JKpLjAIuFBVt+f9VV1Z53cWrrR7HpgLHKuq1YBR2IUzv7Irz7wWODrzhYjUAKoBa7LZdx3WLJS5bzWgOoBa6ef14ecCmmQ5T3afX1tEErMcszb0WRnYd64RelRX1Vph+24AegOvisgJ2ZzbucN4snClXVVgq6ruEJE2WNt+voWajrYCx4RtHg/8WUTahi7ao4EZObT9TwT+ICIniEgClrQOZDnXSBGpJSJ1gTuB/+YRVjxwt4gkiMhZwDlYU9c+4EXgHyJSW0xjETkny3f6HzAYeDvUp+JcjjxZuNJuGHCliOwAxgCvFeJcI4DXQyOreqnqO8DfgLewv+jrcXg/w0GqOg+4BXgDu2NYhyWfPWHnXgQsBOZjTWUP5xHPauwOYj2WHAap6srQezeEYkoNfc77wHHZxPUucBXwnoi0z+PzXBkmvviRc0VPRJKAzUADVV0XdDzO5cXvLJwrIiLSKzT/oQo2tPVrTxSupPBk4VzR6Ys1GaVhI50KVY7EuaLkzVDOOefy5HcWzjnn8lRqJuXVrl1bmzZtGnQYzjlXosyZM2ejqtbJa79SkyyaNm1Kampq0GE451yJIiI/5r2XN0M555yLgCcL55xzefJk4ZxzLk+lps8iO/v27SMtLY3du3fnvbOLSGJiIo0aNSI+Pj7oUJxzRahUJ4u0tDSqVq1K06ZNCSvV7ApIVdm0aRNpaWk0a9Ys7wOcc6VGqW6G2r17N7Vq1fJEESUiQq1atfxOzbkyqFQnC8ATRZT579O5sqnUJ4s8ZWTA2rWwc2fQkTjnXLEV02QhIj1FZImILBeR7BaLbyIiM0VknogsEJHzwt67PXTcEhHpEcs4WbcONsdmCeItW7bwzDPPFPj4ESNG8OGHHwLwxBNPsNOTmnMuADFLFiIShy02cy7QGrg0tB5yuLuAiaraAVtM/pnQsa1Dr9sAPYFnQueLvvLloWpV+PVXiEFRxcImi1GjRnH22WcDniycc8GJ5Z1FZ2C5qq5U1b3ABOCiLPsotmYx2HrEa0PPLwImqOoeVV0FLA+dLzZq1IA9eyAGHbfDhw9nxYoVJCcnM2zYMLp160bHjh1p164db775JgCrV6+mVatW/PnPf6ZNmzZ0796dXbt2ATBw4EAmTZrEk08+ydq1aznzzDM588wzARg/fjzt2rWjbdu23HbbbVGP3TnnMsVy6GxD4Oew12nAiVn2uQf4n4hcB1QGzg479qssxzbM+gEiMgQYAtCkSZNcg7nhBpg/P4c3tTbsqAgVykFCrqc5THIyPPFE7vuMHj2a77//nvnz55ORkcHOnTupVq0aGzdu5KSTTqJXr14ALFu2jPHjx/P8889zySWXMHnyZC677LKD5xk6dCiPPfYYM2fOpHbt2qxdu5bbbruNOXPmkJSURPfu3Zk6dSq9e/eO/As451yEgu7gvhR4SVUbAecBL4tIxDGp6lhVTVHVlDp18iyamDMpB3HlrLM7hlSVO+64g/bt23P22WezZs0aNmzYAECzZs1ITk4GoFOnTqxevTrXc82ePZuuXbtSp04dypcvz4ABA5g1a1ZM43fOlV2xvLNYAzQOe90otC3cFVifBKr6pYgkArUjPDZf8roDYN02WLMG2reHhHzcXuTDK6+8Qnp6OnPmzCE+Pp6mTZsenLNQoUKFg/vFxcUdbIZyzrniIJZ3FrOB5iLSTEQSsA7rt7Ls8xPQDUBEWgGJQHpov/4iUkFEmgHNgW9iGKv1WwBs2RLV01atWpXt27cDsHXrVurWrUt8fDwzZ87kxx8jqgyc7bk6d+7MJ598wsaNG9m/fz/jx4/njDPOiGrszjmXKWZ3FqqaISLXAtOBOOBFVV0oIqOAVFV9C7gJeF5EhmGd3QPV1nldKCITgUVABnCNqu6PVawAVKwIiYk2Kqpu3aidtlatWnTp0oW2bdtywgkn8MMPP9CuXTtSUlJo2bJlvs41ZMgQevbsSYMGDZg5cyajR4/mzDPPRFU5//zzueiirOMHnHMuOkrNGtwpKSmadfGjxYsX06pVq8hPkpYG69dbz3X5Ul02q1Dy/Xt1zhVbIjJHVVPy2i/oDu7iJSnJfka5Kco550o6TxbhKlWC+HhPFs45l4Uni3Ai1tG9bRvsj20XiXPOlSSeLLJKSoIDByA06sg555wniyNVqQJxcTYqyjnnHODJ4kjlykH16rB1a0wKCzrnXEnkySI7SUlW+iOApqgqVaoAsHbtWi6++OJs9+natStZhwlHIjU1laFDhxYqPudc2eSTCbJTrZp1dm/ZYs8D0KBBAyZNmhTVc6akpJCSkudwauecO4LfWWQnLs6SxJYthW6KGj58OGPGjDn4+p577uH+++/PtlR5uNWrV9O2bVsAdu3aRf/+/WnVqhV9+vQ5rG7UVVddRUpKCm3atGHkyJEHt8+ePZtTTjmF3/3ud3Tu3Jnt27fz8ccfc8EFFwCwefNmevfuTfv27TnppJNYsGABAJ988gnJyckkJyfToUOHg+VFnHNlW9m5s8i1Rnk29u2z9S0qVbLkkZ0IapT369ePG264gWuuuQaAiRMnMn36dIYOHXpEqfKc1rd+9tlnqVSpEosXL2bBggV07Njx4HsPPPAANWvWZP/+/XTr1o0FCxbQsmVL+vXrx2uvvcYJJ5zAtm3bqFix4mHnHDlyJB06dGDq1KnMmDGDP/3pT8yfP59HH32UMWPG0KVLF3bs2EFiYmLkvzPnXKlVdpJFfmWW+8jIyDlZRKBDhw788ssvrF27lvT0dJKSkqhXrx7Dhg1j1qxZlCtX7mCp8nr16mV7jlmzZh3sa2jfvj3t27c/+N7EiRMZO3YsGRkZrFu3jkWLFiEi1K9fnxNOOAGAatk0pX322WdMnjwZgLPOOotNmzaxbds2unTpwo033siAAQP4/e9/T6NGjQr83Z1zpUfZSRZ51ijPxg8/2OS8Nm0K9dF9+/Zl0qRJrF+/nn79+uVaqjw/Vq1axaOPPsrs2bNJSkpi4MCBBTpPuOHDh3P++eczbdo0unTpwvTp0/Nd8NA5V/p4nwU2By/bromkJNi1q9DLrfbr148JEyYwadIk+vbtm+9S5aeffjqvvvoqAN9///3B/oVt27ZRuXJlqlevzoYNG3jvvfcAaNGiBevWrWP27NkAbN++nYwsCzuddtppvPLKKwB8/PHH1K5dm2rVqrFixQratWvHbbfddrBKrnPOlZ07ixzs3g1LlkDjxlCzZpY3a9SAn3+2ju4cmogi0aZNG7Zv307Dhg2pX78+AwYM4MILL4y4VPlVV13FoEGDaNWqFa1ataJTp04A/O53v6NDhw60bNmSxo0b06VLFwASEhJ47bXXuO6669i1axcVK1bkww8/POyc99xzD4MHD6Z9+/ZUqlSJcePGAfDEE08wc+ZMypUrR5s2bTj33HML/L2dc6VHmS9RrgoLF9pI2dat7edhFi2yiXreFHOQlyh3rvTwEuUREoH69a21KdtiszVqwI4dNjrKOefKqDKfLMCanypUgHXrsum7iNFyq845V5KU+mQRSTNb5t3Fzp1WnfwwFStaJvFkAUT2+3TOlT6lOlkkJiayadOmiC5wNWtCQkI2dxe+xsVBqsqmTZt8op5zZVCpHg3VqFEj0tLSSE9Pj2j/3bth7Vrrvzjserh7N6Snw7x5ULlybIItIRITE32innNlUKlOFvHx8TRr1izi/XfvhmbNbFTURx+FvbF/v7VTnXUWTJgQ/UCdc66Yi2kzlIj0FJElIrJcRIZn8/7jIjI/9FgqIlvC3tsf9t5bsYwzU2Ii3HILzJgBX3wR9kZcHFx0EUybBnv2FEUozjlXrMQsWYhIHDAGOBdoDVwqIq3D91HVYaqarKrJwFPAlLC3d2W+p6q9YhVnVn/5C9SqBQ88kOWN3r1tfYsZM4oqFOecKzZieWfRGViuqitVdS8wAbgol/0vBcbHMJ6IVK4MN95oNxFz54a90a2bLbk6dWpgsTnnXFBimSwaAj+HvU4LbTuCiBwNNAPC/2xPFJFUEflKRHrncNyQ0D6pkXZiR+Kaa2wA1GF3F4mJcO658OabZX5UlHOu7CkuQ2f7A5NUNfwqfHRoCvr/AU+IyLFZD1LVsaqaoqopderUiVow1avD0KEwZYqVAjmoTx/YsAG+/jpqn+WccyVBLJPFGqBx2OtGoW3Z6U+WJihVXRP6uRL4GOgQ/RBzNnSotTo9+GDYxvPOg/h4eOONogzFOecCF8tkMRtoLiLNRCQBSwhHjGoSkZZAEvBl2LYkEakQel4b6AIsimGsR6hVC66+2kbKLlsW2li9ug2ffeONQi+36pxzJUnMkoWqZgDXAtOBxcBEVV0oIqNEJHx0U39ggh4+zboVkCoi3wIzgdGqWqTJAqyjOyEBRo8O29i7N6xYkaV9yjnnSrdSXaI8Gq6/Hp55BpYvh6OPxuqBNGgA990Hd90V9c9zzrmi5CXKo+SWW6w81EMPhTbUrw8nneT9Fs65MsWTRR4aNYJBg+Bf/7K6UYCNipo7F376KdDYnHOuqHiyiMBtt9nUikcfDW3oHZr24RP0nHNlhCeLCBxzDAwYAM89Z8VnOf54qzboycI5V0Z4sojQHXdYVdrHHw9t6N0bZs2CTZsCjcs554qCJ4sItWgBl1wCTz8Nv/6K9Vvs3w/vvBN0aM45F3OeLPLhjjus8OxTTwGdOlnvt4+Kcs6VAZ4s8qF9e1vW4oknYPsOsaao//3PFu92zrlSzJNFPt15pzVDPfsslix27YLp04MOyznnYsqTRT6dcAL06AF//zvsTDkdkpJ8VJRzrtTzZFEAd90Fv/wCz78UDxdcAG+/Dfv2BR2Wc87FjCeLAjj1VDjjDHj4Ydh3QR9rl/r006DDcs65mPFkUUB33WXlP/6zvrutouejopxzpZgniwLq1g1OPBHuf7wyB87pYf0WpaSCr3POZeXJooBE4O67YfVq+KJub0hLgzlzgg7LOediwpNFIZx3HiQnw00fX4iWK+ejopxzpZYni0IQsb6Lb1bU4peWp3u/hXOu1PJkUUh9+kCrVvDir31g0SJYujTokJxzLuo8WRRSuXI2q/u5dRfZBm+Kcs6VQp4soqBfP4g/9mgWV+qIerJwzpVCniyioHx5uP12eHVnb+TLL2HduqBDcs65qIppshCRniKyRESWi8jwbN5/XETmhx5LRWRL2HuXi8iy0OPyWMYZDX/8I3x5VB8A9M23Ao7GOeeiK2bJQkTigDHAuUBr4FIRaR2+j6oOU9VkVU0GngKmhI6tCYwETgQ6AyNFJClWsUZDQgL0uasNyzmWX1/0UVHOudIllncWnYHlqrpSVfcCE4CLctn/UmB86HkP4ANV3ayqvwIfAD1jGGtUDL5C+F+lPlRNnQFbtwYdjnPORU0sk0VD4Oew12mhbUcQkaOBZsCM/BwrIkNEJFVEUtPT06MSdGFUrAg1B/cmXvex9IlpQYfjnHNRU1w6uPsDk1R1f34OUtWxqpqiqil16tSJUWj5c+EDJ7FBjmL9P31UlHOu9IhlslgDNA573Si0LTv9OdQEld9ji5XK1eJY26kXHdZNY/5Xu4MOxznnoiKWyWI20FxEmolIApYQjhgmJCItgSTgy7DN04HuIpIU6tjuHtpWIhx/Wx+qsoNpN30UdCjOORcVMUsWqpoBXItd5BcDE1V1oYiMEpFeYbv2ByaoHqrvraqbgfuwhDMbGBXaViJUvvAsdidUpc4XU5k/P+honHOu8ERLyRoMKSkpmpqaGnQYB+39fX+2Tp1Bv9PW8dHHcYgEHZFzzh1JROaoakpe+xWXDu5SJ+H/LqaOplN71mSmTAk6GuecKxxPFrHSpw/arj1/T7idO27aw27v63bOlWCeLGIlLg559BEa713JeT8+w2OPBR2Qc84VnCeLWOreHXr0YFT8fTzzwK+sXRt0QM45VzCeLGLt4YepkrGFm3Y/wO23Bx2Mc84VjCeLWGvfHhk4kOvkKWb9ZxXffBN0QM45l3+eLIrCffcRlxDHY4l3cMMNUEpGKzvnyhBPFkWhYUPkppvos3sCGV9+w/jxeR/inHPFiSeLonLrrWjdujxX5WZuvUX57begA3LOuch5sigqVasi995Lxx2f0mntWzz8cNABOedc5LzcR1HKyIB27Vi75gAt9n7PwqXxNGkSdFDOubLMy30UR+XLw8MP02D7Uq448Dy33hp0QM45FxlPFkXtgguga1ceiL+Haa9t47PPgg7IOefy5smiqInAo49SeWc6D1R9iBtugAMHgg7KOedy58kiCJ06wYABXLX7MTbM+Zlx44IOyDnncufJIigPPEBcOeW5Ondz++2wbVvQATnnXM48WQTl6KOR66/nvI3/od6G+Tz4YNABOedczjxZBOn225GaNXml/s08/piyYkXQATnnXPY8WQSpRg0YMYI26z7ivHLvc8stQQfknHPZ82QRtL/+FY47jrHVb+GtNzKYMSPogJxz7kgxTRYi0lNElojIchEZnsM+l4jIIhFZKCKvhm3fLyLzQ4+3YhlnoBISYPRo6vyykJtrvcQNN9hE7xJLFdLT4fPP4aWX4P777bVzrkQrH6sTi0gcMAY4B0gDZovIW6q6KGyf5sDtQBdV/VVE6oadYpeqJscqvmLl97+HU05h5OIRPP1df154oQp//WvQQeVh2zZYtgyWLj30M/P5li1H7uvFsJwr0SKqDSUixwJpqrpHRLoC7YH/qOqWXI45GbhHVXuEXt8OoKp/C9vnYWCpqr6QzfE7VLVKpF+kRNSGys2XX8Ipp/Dvo+/hlh0jWbYMkpICjmn3blix4vBEkPl8w4ZD+4lAkybQvDkcf7w9Mp8PGwazZ8PPP0N8fHDfxTmXrUhrQ0V6ZzEZSBGR44CxwJvAq8B5uRzTEPg57HUacGKWfY4PBfs5EIcll/dD7yWKSCqQAYxW1alZP0BEhgBDAJqU9Ip8J58MF1/Mn959hLt2D2HUqPo8/ngRfv78+fDJJ4cnhJ9+OnylpqOOsiRw/vmHJ4Rjj4WKFbM/75VXwjvvwPvvw4UXFs13cc5FXaR3FnNVtaOI3ALsVtWnRGSeqnbI5ZiLgZ6qemXo9R+BE1X12rB93gH2AZcAjYBZQDtV3SIiDVV1jYgcA8wAuqlqjoNLS/ydBcDy5dC6NZ8eO5Czlo/lu++gZcsYf+bevTByJDz0kCWGatUO3R2EJ4TmzaF69fyff98+aNQIunSBKVOiH79zrlCifWexT0QuBS4HMv88zKtNYQ3QOOx1o9C2cGnA16q6D1glIkuB5sBsVV0DoKorReRjoANQumciHHccXH01pz71FB0rXs9NN7Xh3Xdj+HmLF8OAATBvHlxxBdx3H9SrZ81K0RIfD5ddBk8+aR3ddepE79zOuSIT6WioQcDJwAOqukpEmgEv53HMbKC5iDQTkQSgP5B1VNNUoCuAiNTGmqVWikiSiFQI294FWERZcPfdSNWqjG98K9OmwXvvxeAzVGHMGOjY0ZqapkyBF16A+vWjmygyDRpkQ7z++9/on9s5VyQiShaqukhVh6rqeBFJAqqq6kN5HJMBXAtMBxYDE1V1oYiMEpFeod2mA5tEZBEwE7hFVTcBrYBUEfk2tH10+CiqUq1WLbjzTo75YRp/avgRN95oLTlRs3699Tlcey2ccQZ89x306RPFD8hG27Zwwgnw738f3gfinCsxIu2z+BjohTVbzQF+AT5X1RtjGl0+lIo+i0y7d0PLlmyNq0nSylQef6Ic118fhfO++aZ1OO/YAY88AtdcE5s7iew8+yxcfTWkplrVXedcsRDtlfKqq+o24PfYkNkTgbMLE6DLRWIiPPgg1VfOY3TbV7jnHti4sRDn27EDhgyB3r2ts3nOHLuzKKpEAdC/P1SoYHcXzrkSJ9JkUV5E6mOjlt6JYTwuU//+0KkTwzbeyb5tuxgxooDn+fpr6NDB+iRuu81et24d1VAjkpRkkw9fecXunJxzJUqkyWIU1r+wQlVnh4azLotdWI5y5eDRR4lf/zOvnvgP/vlP616IWEYGjBplQ1b37oWZM2H0aCsvEpRBg2x295tvBheDc65AIuqzKAlKVZ9FuF690Jkf06L8Chp3rMOHH0bQerRihQ1X/eorGxr79NNW4TZo+/dDs2Z2Z/P++3nv75yLuaj2WYhIIxF5Q0R+CT0mi0ijwofp8vTQQ8iunUxqP4oZM/L4o1wVXnwRkpNtDsWrr9pw1eKQKADi4mDgQPjf/6z8h3OuxIi0Gerf2ByJBqHH26FtLtZatYI//5l2XzzHecct5eabYc+ebPbbuBEuvtgm16WkwIIFcOmlRR5ungYOtKT2n/8EHYlzLh8iTRZ1VPXfqpoRerwE+FTconLPPUhiIi8eNZwVK+CJJ7K8P306tG8Pb79tQ2I/+sgK+xVHxxxj8zt8zoVzJUqkyWKTiFwmInGhx2XAplgG5sIcdRTcdhtHff4Gw7t8yl13hUag7toF118PPXvaaKNvvoGbb7bO8eJs8GDrV/n006Ajcc5FKNJJeUcDT2ElPxT4ArhOVYtNw3Op7eDOtHMnNG9ORv1GnJf0Fb98+C3Taw/gqI2LLGH87W85V34tbn77zUqL/OEPPu/CuYBFtYNbVX9U1V6qWkdV66pqb+APhY7SRa5SJbj/fsrP+Yb3ky4ltVxn9m/8lftOnc6uvz1RchIFQOXKcMkl8PrrNmHQOVfsFaa9otiU+igz/vQnaN+ecq+/RtxFFzD13gWM/Lw7XbsevhZRiTB4sN1hvP560JE45yJQmGRRhLUiHGBDT6dMgbffRiZP5uoRtXnjDfj+ezjxRPtZYpx8MrRoYUN9nXPFXmGShQ9lCcKxx8IFFxycmXfRRdZPvG8fnHKKDYwqEURsGO1nn9nqfM65Yi3XZCEi20VkWzaP7dh8C1cMdOxoJZ+OOcaqjz/7bNARRehPf7KRWy+9FHQkzrk85JosVLWqqlbL5lFVVSNdZc8VgUaN7I/0c8+1SuDDhll1jSDt3m3Lb//ySw47NGhgw37HjQs+WOdcror5gHyXH1WqwNSpcMMNNnGvT59gBhv99hs89piVgbrwQms5GzUqh1gGDYI1a+CDD4o8Tudc5DxZlDJxcfD447Zq6rvvwmmnQVpa0Xz29u3w0EOWJG66yeoFvv46dO8OI0faEuPPPptl5b8LL4SaNX2+hXPFnCeLUurqqy1ZrFhhI6Xmzo3dZ23ZAvfdB02bwvDhthDe559b1ZGLL4bJk+GLL6B5c4urTRuYNClU7aNCBauQO3UqbN4cuyCdc4XiyaIU69nTLtrly9sdRrSXkdi8GUaMsCQxYoQtnfH11/DeezYyK9zJJ8OsWVa+KiEB+vY9tI1Bg2zNjVdfjW6Azrmo8WRRyrVrZxfwNm2sD+Pxxwtfv++XX+wO4uij7Y7i7LNh3jx46y3o3Dnn40Rs1O+339r0ijVrrKbgBXcls6tlsjdFOVeMebIoA+rVg48/tlVNb7zRmoIO6zeI0Lp1dnzTpvDww3bh/+47a1JKTo78PHFxdjOxdKn1cXz2Gdz2w2CYO5d173+b/8CcczEX02QhIj1FZImILBeR4Tnsc4mILBKRhSLyatj2y0VkWehxeSzjLAsqVYKJE+2O4Lnn7EK/dWtkx/78M1x3nXVcP/mkNSEtXgzjx0PbtgWPqWJFuPVWWLkSkq75P/aQwKQL/s2tt8Kvvxb8vM65GFDVmDyAOGAFcAyQAHwLtM6yT3NgHpAUel039LMmsDL0Myn0PCm3z+vUqZO6yPzrX6rly6u2aaO6alXO+61cqTpkiGp8vO1/xRWqy5fHLq4d512sWyvU1gT2aI0aqg8/rLpzZ+w+zzmnCqRqBNf0WN5ZdAaWq+pKVd0LTAAuyrLPn4ExqvorgKpmTt/qAXygqptD730A9IxhrGXK4MG2sumaNTZS6quvDn9/+XLbp3ly7PxIAAAZyUlEQVRzm1x95ZW27YUXbM5ErFS+djDV9mxkyd/f4ZRT7K6jRQuLwefsOResWCaLhkD4ehdpoW3hjgeOF5HPReQrEemZj2MRkSEikioiqenp6VEMvfQ780z48kubyHfmmdZEtXixjWJt0cKamK65xpqInnnGOrNjrnt3aNCApjNe5N13YeZM628ZNMj6RN591xfXcy4oQXdwl8eaoroClwLPi0iNSA9W1bGqmqKqKXXq+Cqv+dWypY2U6tQJ+vWzEVNvvGGd2KtWwT/+AQ2PSNExFBdn9aLeew/WraNrV4tv4kQrHXLBBRzc5pwrWrFMFmuAxmGvG4W2hUsD3lLVfaq6CliKJY9IjnVRULu2TZ4bNgzuuANWr7ZlvOvVCyigQYPgwAF4+WXAhtv27QuLFtms9B9+gJNOsrWTNvnCvs4VmYiWVS3QiUXKYxf/btiFfjbwf6q6MGyfnsClqnq5iNTGOruTsfLnc4COoV3nAp1UNccpvqV+WdWy5NRTYeNGaxeTw5dN2bED/v53ePBBK5745puFG5HlXFkX1WVVC0JVM4BrgenAYmCiqi4UkVEi0iu023Rgk4gsAmYCt6jqplBSuA9LMLOBUbklClfKDBoES5Yc2fOO9bGMHAmffGLLkp98slUKcc7FVszuLIqa31mUItu3WzvYgAEwdmyOu61ZY7PSZ8+Ge++Fu+6y5TGcc5EL/M7CuQKrWtU6KiZMsNuHHDRsaLWl/vhHu9u45JJgSrI7VxZ4snDF06BBdocxeXKuuyUm2tpJjz5qI7m6dLFOeudcdHmycMXT6afbOrERFBcUsfUzpk2DH3+ElBSrheWcix5PFq54ErG7i5kzbdJHBHr0gG++gTp14JxzbDJhKemScy5wnixc8XX55ZY0Xnop4kOOP94GUfXoYTPQ//pXWyrDOVc4nixc8dW4sS2WMW6cTdSLUPXqNv/i9tttMFW3brYGh3Ou4DxZuOJt8GDriJg5M1+HxcXZxL3x42HOHOvHmDcvRjE6VwZ4snDFW+/eUKOGLa1XAP372+JKqjZS6rXXohxfUJYssaJed95pi6A7F2OeLFzxlpgIl14KU6YU+KLYsSOkptrP/v2tBlY+WrWKn23bLIn+8IPdPjVrBqNHw2+/BR2ZK8U8Wbjib/BgKztbiNuCo46CGTPgz3+Gv/0NLrrIrrklzoEDNgtx2TKr2T5vnt0y3X67LTby9NPeo+9iwpOFK/46dbJqgRHMuchNQgL88592PX3vPateu2xZlGIsKvfdB2+9BY89ZvXak5PhnXesra1FC1v/tkULGxTgK0a5KPJk4Yq/zDkXX39ttcoLeaprroEPP7QRUp0726qBJcJbb8E999iaH9ddd/h7XbrYTMTp06FWLRg4ENq1s+Y7n2ziosCThSsZLrsMypcv9N1Fpq5drQBh48Zw7rn2h3qxvqb+8IP9Djp1gueeO6J0O2Dbune3LzZpkn2hP/zBMuIHHxTzL+iKO08WrmSoW9eWynv5Zdi3LyqnbNYMvvjC+opvusn+GN+9Oyqnjq6tWy3IxEQrgFWxYu77i1iS+O47S67p6ZZEzjrL1tJ1rgA8WbiSY9Ag2LDBOhyipEoVeP11K3H+n//YiKmxY4vRwKLMDu0VKyzQxo3zPiZT+fKWAZcsgaeessWkTjkFevWCBQtiFrIrnTxZuJLj3HPtDiNKTVGZypWDESPg7betE/wvf4EGDeD66631J1CjRllgjz0GZ5xRsHNUqADXXmsJ58EH4dNPrWP8//6vBPbwu6B4snAlR3y8de6+805M6ndccIGNRP38c3v+7LPQqpVVHJkyBTIyov6RuXvzTbvlufxyu9gXVuXKNsR25Ur7+eab9gX/8hdISyv8+V2p5snClSyDBtlV+5VXonfO/ftt1aT0dGTfXk45xU6flgYPPABLl1oXQLNmcP/9sH599D46R4sXW/NTSkrOHdoFlZRkX2zFCrj6artTO+4467jZuDF6n+NKFV9W1ZU8J55onb633gq7dlmv9K5dBX+Ed5jXqwevvgpnnnlwU0aGzX975hkbZhsfb8nj6qvh1FOjex0H7Lt17mwz1lNT89dPURCrVx/qtKlUCa68EmrXtva5cuWs0FZ2z3N7L7vn7dvD0UfH9ru4fIt0WVVPFq7kefFFuOKKI7fHxdlIoYI+KlSwtqelS22d1jvvtHOGWbrUdvn3v+2a3q6dJY3LLrPO8kI7cMCml7//Pnz0kS0CVVQWL4a7747d3IyqVS3bnnRS9M/tCsyThSu9VK3dPWtyiI8v/Ll37ICrroL//tc6K/77X6sVksVvv1lF2zFjYP58uw5efrkljlatCvH5I0bYLO2nn7bZg0E4cMAe+/dH7/lvv1kTYnq6zYhMyfPa5IpIpMkCVY3ZA+gJLAGWA8OzeX8gkA7MDz2uDHtvf9j2t/L6rE6dOqlzUXHggOoLL6gmJqrWq6c6c2auu37xhepll6kmJKiC6plnqk6apLp3bz4/d8oUO8GgQXbi0uann1SbNVNNSlKdOzfoaFwIkKqRXM8j2akgDyAOWAEcAyQA3wKts+wzEHg6h+N35OfzPFm4qPv2W9Xjj1ctV071vvtU9+/PdfcNG1T/9jfVJk3sX1aDBqr33qu6dm0En7VwoWqVKqqdO6vu2hWd+IujVavsF1SrluqCBUFH4zTyZBHL0VCdgeWqulJV9wITgIti+HnORVf79tbB3L+/teX37JnrkN26dWH4cGshe/NN688YORKaNIELL7TVYTdtyubALVtshnalSjB5ss3ULq2aNrXyv4mJtoRhIWt9uaITy2TREPg57HVaaFtWfxCRBSIySUTCh30kikiqiHwlIr2z+wARGRLaJzU9PT2KoTsXUrWq9VuMHQuzZtlktk8+yfWQuDibJP3++9Yhfv31NmF60CDr/jjnHOskX7cOa9O/7DJYtcrqOTVqVDTfK0jHHmsJIy7OEsaSJUFH5CIQ9DyLt4Gmqtoe+AAYF/be0WqdLv8HPCEix2Y9WFXHqmqKqqbUqVOnaCJ2ZY+ILYTx9deWPM46y2ZCR7CCUvPm8OijNjp19mwb7fvTT9YR3rAhvHT0SHj3XTaN+Aecdlrsv0txcfzxljAOHLDf5/LlQUfk8hDLZLEGCL9TaBTadpCqblLVPaGXLwCdwt5bE/q5EvgY6BDDWJ3L2+9+Z81Sl1xiw2rPO89G90RAxAYAPfiglRD5/nt4rd8UBqbdz78YTO0RV9Gpk82VC7zESFFp1cqGB+/ZYwlj1aqgI3K5iGWymA00F5FmIpIA9AfeCt9BROqHvewFLA5tTxKRCqHntYEugDduuuBVrWqT9v75T1s/IjnZai3lgwi0kUX0fedy6NyZMxeO4ZFHhIQEuOsuu4a2bm3P580r5ZXF27a1obQ7dljC+OmnoCNyOYhZslDVDOBaYDqWBCaq6kIRGSUivUK7DRWRhSLyLTAUGx0F0ApIDW2fCYxWVU8WrngQgSFD4KuvrFP6zDNtrdZIF/bO7NCuXBmmTOGY1oncfLNVD09LsykW9evbKTt2hGOOsUocX3xRwtcOz0lysq238euvljDWrMn7GFfkfFKec4WxbZsljtdes9FSL79spTJysn+/9X5/8AHMnGkr3OVg40ZbHG/KFNt9715LIr17W7mR00+PzjzEYuPrr633v359u2urXz/PQ1zh+Qxu54qKqhX7u+EGqFPHEkdOSeCuu6xj4tln4a9/jfgjtm6FadMscUybBjt3Qo0a1k/csKGVVG/Q4Mjn1avHoHZVLH3+OfToYeONZ87Mdva8iy5PFs4VtXnzoG9fG/r04INw881WQC/TlCl2S3DllTYUt4BX8Z07rcTStGn2UWvXWsvNli1H7lux4uFJJKfEktfie0Vq1iy7Szv2WEsYud2pRduKFbZ2yLRpVvYl63/DUsiThXNB2LbNksHrr9toqXHj7GK3cKFVy23XzppYKlSI+kfv3GlzN9assQSSmUSy/sxu6dikpEOJ4/jjbWhv69ZRDzFyH31ki4q0bGnPa9aM7ed9/TU88ogl9PLlbULmnDm24Na4cXbHWEp5snAuKKrWzDRsmE3r/uc/bWbejh12AWrQINDQtmw5MpmEP//uO6vc3qcP3HFHgDX/pk+3/p127WzEVI0a0T3/gQNWe/6RR2xEW/XqdjcxdKiVqs/8b1i7NkyYUGrnwRSLQoJF+fDaUK7YSU1VPeYYKxQVH6/6+edBRxSR9HTVu+9WrVHDQu/eXfXjjwOqbfjOO/a7O/FE1a1bo3POXbusUGTLlvYFmzRRffxx1W3bjtx37lzV446z+mD3359nfbCSiKALCRb1w5OFK5a2bFG9+mrVCROCjiTftm5VHT1atW5du1J06aL67rsBJI2pU1XLl1c95RTV7dsLfp7Nm1UfeED1qKPsCyUnq77ySt7lgbduVe3f34455xzV9esLHkMx5MnCORcVO3eqPv30oWq6ycmqEyeqZmQUYRCvv64aF6d6+umqO3bk79hVq1SHDlWtXNm+QI8eqh9+mL+sd+CA6tixh8rWz5iRvxiKsUiTRenu5nfOFVrFirYO0/LltkLgzp1W8aR1a3sdviptzFx8sc1h+ewz68fYtSvvY+bOhUsvtfXFn3nGRqJ9+61VeOzWLX+j0cLrg1Wvbgtj3XuvzZspIzxZOOciEh8PAwdaVfGJE23y+uDBdi1++unIrt+FcumlVud95kybmZjdsC5VeO89SwadOlkH9rBhVndq3Dgb5VQYmWXrBwyAe+6B7t1D5YNLP08Wzrl8iYuz6SRz59p0hMaN4brrbKmK0aNt9HDM/PGP8MILNtHk4outCCHY9PbMZHDeeVb2/JFH4Oef7Wc0S79XqWKf9eKLVqMls1xJKefJwjlXICI2DeGzz2yJjw4d4PbbbfL13XdbuZKYGDzYZsy/+y706wcPPwzNmtltj4hdyFeutAl11avHJgYRW6Bk9mwbWtujh83Oz8iIzecVB5F0bJSEh3dwOxe81FTV3//e+pErVVIdNkw1LS1GH/bUU/ZBoNqtm+r77wczvve331QHD7Y4Tj89hl84Noiwg9sn5Tnnom7RInjoIXjlFauWMXCgLTl7zDFR/qDp023iY4disNzNyy/bpL6KFe15z55BRxSRSCfleTOUcy7qWre21qDly636yX/+Y9tGj47y6KkePYpHogDrT0lNtWq5555r2bFIhooVDU8WzrmYadrURq2uWAHnn299GieeaJ3jpVLLlja89i9/sVurrl2tk70U8GThnIu5hg1h8mR7rFsHnTvbH94xH24bhIoVrQN+/HgrtJWcDG+/HXRUheZ9Fs65IvXrrzZQ6cUXoXlzeP55OOOMoKOKkWXLbMTWvHlw4422/GFCQvb77ttn2XP3bvuZ0/PsttWrZ4twFYBXnXXOFWsffWTXt5UrD7XaxGqka6B277bsOGaMtctVq5Z9AijMbPCTTrI5HwXgycI5V+zt3AkjRsDjjx+qCt6rV9BRxciUKXY7Vb68NVVVrAiJiUc+z25bbu8nJto5C8iThXOuxJg9G664wpr4L7kEnnzSV1QtKsVi6KyI9BSRJSKyXESGZ/P+QBFJF5H5oceVYe9dLiLLQo/LYxmncy5YJ5xgo07vuw+mToVWrWy4bSn5W7ZUiFmyEJE4YAxwLtAauFREsluo8TVVTQ49XggdWxMYCZwIdAZGikhSrGJ1zgUvIcEqZsyfb3MyLr/c5rWtXh10ZA5ie2fRGViuqitVdS8wAbgowmN7AB+o6mZV/RX4ACgZ0yGdc4XSqhXMmmWVbL/4Atq2hX/8o0xVAy+WYpksGgLhs1HSQtuy+oOILBCRSSLSOD/HisgQEUkVkdT09PRoxe2cC1i5craGxsKFcPrpcMMN0KWLvXbBCHpS3ttAU1Vtj909jMvPwao6VlVTVDWlTp06MQnQORecJk2suOx//2ulQzp0sGUkMiuTu6ITy2SxBmgc9rpRaNtBqrpJVTP/s78AdIr0WOdc2SBiaw0tXmzraNx7L3TsCF99FXRkZUssk8VsoLmINBORBKA/8Fb4DiJSP+xlL2Bx6Pl0oLuIJIU6truHtjnnyqg6dayK7TvvwPbtcMop1jzlLdBFI2bJQlUzgGuxi/xiYKKqLhSRUSKSOe1mqIgsFJFvgaHAwNCxm4H7sIQzGxgV2uacK+POP9/6Lq6+2jq+69a1TvEhQ6wy+KpVPuQ2FnxSnnOuxPr2W3j/ffj0U/j8c9iyxbY3bAinnXbo0aaNdZq7I/kMbudcmXLgAHz/vSWOzMfatfZeUpKNpspMHp065VzPr6zxZOGcK9NUbUJfePJYssTeS0y0dTUyk8fJJ0PVqoGGGxhPFs45l8Uvv8Bnn1ni+Owzqxy+fz/ExdmyE5nJ49RTrS+kLPBk4Zxzedi+3YbgZt55fPWVVQsHq/rdty9cfLHN9yitPFk451w+7d1rS75+9JGt6jdvnm0/8cRDiePoo4ONMdo8WTjnXCEtXw6TJsHrrx9aN7xz50OJo2nTQMOLimJRotw550qy446ztcLnzLHEMXq09XHccgs0a2aJ45FHbG5HaefJwjnnInDssXDbbbbuxooVtgysKtx6KxxzDKSk2LaVK4OONDa8Gco55wph1apDTVWzZ9u2jh2tqapvX0sy0bJnD2zaZI/Nmw89r1YN+vUr2Dm9z8I554rY6tWHEsc339i2Dh0OJY7jjrNt+/fbbPPMi33Wi39Or3/7LfvP7dTJ7ngKwpOFc84F6McfDyWOr7+2bY0a2QV/y5ac61eVKwc1a9qjVq1Dj6yvs26rVKlgcXqycM65YuKnnyxxzJ0LNWpkf/HPfF29etHWsYo0WZQvimCcc64sa9IEbrwx6CgKx0dDOeecy5MnC+ecc3nyZOGccy5Pniycc87lyZOFc865PHmycM45lydPFs455/LkycI551yeSs0MbhFJB34MOo4sagMbgw4iH0pSvCUpVihZ8ZakWKFkxVscYz1aVevktVOpSRbFkYikRjKNvrgoSfGWpFihZMVbkmKFkhVvSYo1K2+Gcs45lydPFs455/LkySK2xgYdQD6VpHhLUqxQsuItSbFCyYq3JMV6GO+zcM45lye/s3DOOZcnTxbOOefy5MkiBkSksYjMFJFFIrJQRK4POqa8iEiciMwTkXeCjiUvIlJDRCaJyA8islhETg46ppyIyLDQ/wPfi8h4EUkMOqZwIvKiiPwiIt+HbaspIh+IyLLQz6QgYwyXQ7yPhP5fWCAib4hIjSBjzJRdrGHv3SQiKiK1g4itIDxZxEYGcJOqtgZOAq4RkdYBx5SX64HFQQcRoX8A76tqS+B3FNO4RaQhMBRIUdW2QBzQP9iojvAS0DPLtuHAR6raHPgo9Lq4eIkj4/0AaKuq7YGlwO1FHVQOXuLIWBGRxkB34KeiDqgwPFnEgKquU9W5oefbsYtZw2CjypmINALOB14IOpa8iEh14HTgXwCquldVtwQbVa7KAxVFpDxQCVgbcDyHUdVZwOYsmy8CxoWejwN6F2lQucguXlX9n6pmhF5+BTQq8sCykcPvFuBx4FagRI0u8mQRYyLSFOgAfB1sJLl6Avuf90DQgUSgGZAO/DvUbPaCiFQOOqjsqOoa4FHsL8h1wFZV/V+wUUXkKFVdF3q+HjgqyGDyaTDwXtBB5ERELgLWqOq3QceSX54sYkhEqgCTgRtUdVvQ8WRHRC4AflHVOUHHEqHyQEfgWVXtAPxG8WomOSjU1n8RluAaAJVF5LJgo8oftbH1JeIvYBG5E2sCfiXoWLIjIpWAO4ARQcdSEJ4sYkRE4rFE8YqqTgk6nlx0AXqJyGpgAnCWiPw32JBylQakqWrmndokLHkUR2cDq1Q1XVX3AVOAUwKOKRIbRKQ+QOjnLwHHkycRGQhcAAzQ4jt57FjsD4dvQ//eGgFzRaReoFFFyJNFDIiIYG3qi1X1saDjyY2q3q6qjVS1Kdb5OkNVi+1fv6q6HvhZRFqENnUDFgUYUm5+Ak4SkUqh/ye6UUw747N4C7g89Pxy4M0AY8mTiPTEmlF7qerOoOPJiap+p6p1VbVp6N9bGtAx9P90sefJIja6AH/E/kqfH3qcF3RQpch1wCsisgBIBh4MOJ5she5+JgFzge+wf2/FqtyDiIwHvgRaiEiaiFwBjAbOEZFl2N3R6CBjDJdDvE8DVYEPQv/Wngs0yJAcYi2xvNyHc865PPmdhXPOuTx5snDOOZcnTxbOOefy5MnCOedcnjxZOOecy5MnC+fyQUT2hw2Hni8iUZs9LiJNs6tQ6lxxUD7oAJwrYXapanLQQThX1PzOwrkoEJHVIvKwiHwnIt+IyHGh7U1FZEZorYWPRKRJaPtRobUXvg09MsuAxInI86E1MP4nIhUD+1LOhfFk4Vz+VMzSDNUv7L2tqtoOm1H8RGjbU8C40FoLrwBPhrY/CXyiqr/DalstDG1vDoxR1TbAFuAPMf4+zkXEZ3A7lw8iskNVq2SzfTVwlqquDBWRXK+qtURkI1BfVfeFtq9T1doikg40UtU9YedoCnwQWnQIEbkNiFfV+2P/zZzLnd9ZOBc9msPz/NgT9nw/3q/oiglPFs5FT7+wn1+Gnn/BoaVUBwCfhp5/BFwFB9c/r15UQTpXEP5Xi3P5U1FE5oe9fl9VM4fPJoUq4e4BLg1tuw5b1e8WbIW/QaHt1wNjQ5VI92OJYx3OFVPeZ+FcFIT6LFJUdWPQsTgXC94M5ZxzLk9+Z+Gccy5PfmfhnHMuT54snHPO5cmThXPOuTx5snDOOZcnTxbOOefy9P9nsFCNOCA7MgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "K.clear_session()\n",
    "(x_RN, almod) = get_RNkim(x_tan, y_tan)\n",
    "#x_RN=np.load('transf_bem.hdf5.npy')\n",
    "tfMLP = get_tfMLP(x_RN,y_tan)\n",
    "tfMLP = tanitas_earlystop(tfMLP, x_RN, y_tan, 15, 5)\n",
    "halo_mentes(tfMLP,'tf_MLP')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Eddigi legjobb eredményeket most sikerült elérni: 0.5-ös validációs loss, 82% feletti pontossággal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Önálló feladat - MobileNet, illetve konstruált MLP összekpacsolása egy hálóba"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ahhoz, hogy az egészet egy hálóként tudjuk használni a két részhálót össze kell kapcsolnunk."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def link_transf(al_halo, fel_halo):\n",
    "    bem = Input(shape=al_halo.layers[0].output_shape[1:4], dtype='float32')\n",
    "    x = al_halo(bem)\n",
    "    y = fel_halo(x)\n",
    "    \n",
    "    model=Model(inputs=bem, outputs=y)\n",
    "    model.compile(optimizer=SGD(lr=0, decay=0), loss='categorical_crossentropy', metrics=['accuracy', inTop3])\n",
    "    return (model)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "tran_halo=link_transf(almod, tfMLP)\n",
    "halo_mentes(tran_halo,'transf_CNN')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Teszthalmazon vizsgáljuk meg a háló teljesítményét"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 4s 401us/step\n",
      "Confusion Matrix:\n",
      "[[761   8  38  12  10   1   1  11 148  10]\n",
      " [  6 923   0   2   1   1   9   1  30  27]\n",
      " [ 24   0 810  43  51  23  33   7   7   2]\n",
      " [  9   3  51 721  30 108  47  14  10   7]\n",
      " [ 12   1  55  55 753  27  59  35   2   1]\n",
      " [  2   1  41 139  27 745  18  23   3   1]\n",
      " [  3   2  39  37  12  20 879   1   7   0]\n",
      " [ 12   1  27  38  69  64   9 773   5   2]\n",
      " [ 26   8  10   5   0   4   2   1 935   9]\n",
      " [ 17  75   3  12   0   0   3   8  38 844]]\n",
      "\n",
      "Loss:0.5537201182365418; Acc:0.8144; Top3Acc:0.9647\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'loss': 0.5537201182365418, 'acc': 0.8144, 'top3acc': 0.9647}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "teszt(tran_halo,x_tst,y_tst)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.) Ensemble"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Különböző osztályozási versenyeket megnyerő neurális hálók kivétel nélkül szakértő együttesként, úgynevezett ensemble-ként épülnek fel. Megjegyzés: a dropout, mint exponenciálisan sok szakértőt előállító eljárás is ezen módszerekre vezethető vissza.\n",
    "\n",
    "Mi most a k kereszt validáció során betanított hálókból hozzuk létre a szakértő együttesünket."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ensemble(mf, x, y, epn, tol, db=3, bm=32):\n",
    "    N=x.shape[0] \n",
    "    M=N//db\n",
    "    idx=set(range(N))\n",
    "    idxt=idx\n",
    "    models=[None]*db\n",
    "    \n",
    "    model_chkp=ModelCheckpoint('enhalo.hdf5', monitor='val_loss', save_best_only=True)\n",
    "    earlystop=EarlyStopping(monitor='val_loss', patience=tol)\n",
    "    bem=Input(shape=x[0].shape, dtype='float32')\n",
    "        \n",
    "    for i in range(db):\n",
    "        print(str(i+1)+'. halo tanitasa:')\n",
    "        idxa=set(RS(idxt,M))\n",
    "        idxt=idxt-idxa\n",
    "        idxat=list(idx-idxa)\n",
    "        idxav=list(idxa)\n",
    "        x_val=x[idxav]\n",
    "        y_val=y[idxav]\n",
    "        x_tan=x[idxat]\n",
    "        y_tan=y[idxat]\n",
    "        models[i]=mf(x, y, bem)\n",
    "        hst=models[i].fit(x=x_tan, y=y_tan, batch_size=bm, epochs=epn, callbacks=[model_chkp, earlystop], validation_data=(x_val, y_val))\n",
    "        models[i].load_weights('enhalo.hdf5')\n",
    "        \n",
    "    kimenetek=[ahalo.output for ahalo in models]\n",
    "    y = Average()(kimenetek)\n",
    "    model = Model(inputs=bem, outputs=y)\n",
    "    model.compile(optimizer=SGD(lr=0, decay=0, momentum=1.0), loss='categorical_crossentropy', metrics=['accuracy', inTop3])\n",
    "    return(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1. halo tanitasa:\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_6 (InputLayer)         (None, 4096)              0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 120)               491640    \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 10)                1210      \n",
      "=================================================================\n",
      "Total params: 492,850\n",
      "Trainable params: 492,850\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 33334 samples, validate on 16666 samples\n",
      "Epoch 1/10\n",
      "33334/33334 [==============================] - 5s 156us/step - loss: 0.7472 - acc: 0.7471 - inTop3: 0.9409 - val_loss: 0.6954 - val_acc: 0.7655 - val_inTop3: 0.9465\n",
      "Epoch 2/10\n",
      "33334/33334 [==============================] - 5s 148us/step - loss: 0.6722 - acc: 0.7695 - inTop3: 0.9514 - val_loss: 0.6398 - val_acc: 0.7852 - val_inTop3: 0.9560\n",
      "Epoch 3/10\n",
      "33334/33334 [==============================] - 5s 147us/step - loss: 0.6649 - acc: 0.7719 - inTop3: 0.9521 - val_loss: 0.6601 - val_acc: 0.7810 - val_inTop3: 0.9484\n",
      "Epoch 4/10\n",
      "33334/33334 [==============================] - 5s 147us/step - loss: 0.6394 - acc: 0.7816 - inTop3: 0.9521 - val_loss: 0.6533 - val_acc: 0.7816 - val_inTop3: 0.9488\n",
      "Epoch 5/10\n",
      "33334/33334 [==============================] - 5s 147us/step - loss: 0.6262 - acc: 0.7842 - inTop3: 0.9567 - val_loss: 0.6094 - val_acc: 0.7967 - val_inTop3: 0.9542\n",
      "Epoch 6/10\n",
      "33334/33334 [==============================] - 5s 148us/step - loss: 0.6008 - acc: 0.7910 - inTop3: 0.9603 - val_loss: 0.6407 - val_acc: 0.7778 - val_inTop3: 0.9536\n",
      "Epoch 7/10\n",
      "33334/33334 [==============================] - 5s 147us/step - loss: 0.5859 - acc: 0.7970 - inTop3: 0.9627 - val_loss: 0.6283 - val_acc: 0.7878 - val_inTop3: 0.9534\n",
      "Epoch 8/10\n",
      "33334/33334 [==============================] - 5s 148us/step - loss: 0.5675 - acc: 0.8043 - inTop3: 0.9624 - val_loss: 0.6089 - val_acc: 0.7937 - val_inTop3: 0.9574\n",
      "Epoch 9/10\n",
      "33334/33334 [==============================] - 5s 146us/step - loss: 0.5606 - acc: 0.8091 - inTop3: 0.9630 - val_loss: 0.5944 - val_acc: 0.7980 - val_inTop3: 0.9576\n",
      "Epoch 10/10\n",
      "33334/33334 [==============================] - 5s 147us/step - loss: 0.5396 - acc: 0.8144 - inTop3: 0.9672 - val_loss: 0.5833 - val_acc: 0.8059 - val_inTop3: 0.9563\n",
      "2. halo tanitasa:\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_6 (InputLayer)         (None, 4096)              0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 120)               491640    \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 10)                1210      \n",
      "=================================================================\n",
      "Total params: 492,850\n",
      "Trainable params: 492,850\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 33334 samples, validate on 16666 samples\n",
      "Epoch 1/10\n",
      "33334/33334 [==============================] - 5s 160us/step - loss: 0.7648 - acc: 0.7451 - inTop3: 0.9363 - val_loss: 0.7547 - val_acc: 0.7502 - val_inTop3: 0.9389\n",
      "Epoch 2/10\n",
      "33334/33334 [==============================] - 5s 147us/step - loss: 0.7114 - acc: 0.7566 - inTop3: 0.9457 - val_loss: 0.6871 - val_acc: 0.7675 - val_inTop3: 0.9519\n",
      "Epoch 3/10\n",
      "33334/33334 [==============================] - 5s 147us/step - loss: 0.6739 - acc: 0.7685 - inTop3: 0.9502 - val_loss: 0.6829 - val_acc: 0.7677 - val_inTop3: 0.9497\n",
      "Epoch 4/10\n",
      "33334/33334 [==============================] - 5s 147us/step - loss: 0.6548 - acc: 0.7741 - inTop3: 0.9548 - val_loss: 0.6710 - val_acc: 0.7728 - val_inTop3: 0.9514\n",
      "Epoch 5/10\n",
      "33334/33334 [==============================] - 5s 146us/step - loss: 0.6347 - acc: 0.7832 - inTop3: 0.9555 - val_loss: 0.6434 - val_acc: 0.7866 - val_inTop3: 0.9537\n",
      "Epoch 6/10\n",
      "33334/33334 [==============================] - 5s 148us/step - loss: 0.6059 - acc: 0.7955 - inTop3: 0.9581 - val_loss: 0.6527 - val_acc: 0.7727 - val_inTop3: 0.9530\n",
      "Epoch 7/10\n",
      "33334/33334 [==============================] - 5s 146us/step - loss: 0.5968 - acc: 0.7958 - inTop3: 0.9600 - val_loss: 0.5959 - val_acc: 0.8005 - val_inTop3: 0.9626\n",
      "Epoch 8/10\n",
      "33334/33334 [==============================] - 5s 147us/step - loss: 0.5874 - acc: 0.7986 - inTop3: 0.9621 - val_loss: 0.6016 - val_acc: 0.7944 - val_inTop3: 0.9582\n",
      "Epoch 9/10\n",
      "33334/33334 [==============================] - 5s 146us/step - loss: 0.5529 - acc: 0.8122 - inTop3: 0.9649 - val_loss: 0.5896 - val_acc: 0.8036 - val_inTop3: 0.9617\n",
      "Epoch 10/10\n",
      "33334/33334 [==============================] - 5s 147us/step - loss: 0.5421 - acc: 0.8181 - inTop3: 0.9654 - val_loss: 0.5729 - val_acc: 0.8048 - val_inTop3: 0.9623\n",
      "3. halo tanitasa:\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_6 (InputLayer)         (None, 4096)              0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 120)               491640    \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 10)                1210      \n",
      "=================================================================\n",
      "Total params: 492,850\n",
      "Trainable params: 492,850\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 33334 samples, validate on 16666 samples\n",
      "Epoch 1/10\n",
      "33334/33334 [==============================] - 5s 158us/step - loss: 0.7772 - acc: 0.7408 - inTop3: 0.9345 - val_loss: 0.7459 - val_acc: 0.7481 - val_inTop3: 0.9341\n",
      "Epoch 2/10\n",
      "33334/33334 [==============================] - 5s 147us/step - loss: 0.6871 - acc: 0.7659 - inTop3: 0.9470 - val_loss: 0.7105 - val_acc: 0.7587 - val_inTop3: 0.9387\n",
      "Epoch 3/10\n",
      "33334/33334 [==============================] - 5s 146us/step - loss: 0.6422 - acc: 0.7803 - inTop3: 0.9537 - val_loss: 0.6657 - val_acc: 0.7788 - val_inTop3: 0.9504\n",
      "Epoch 4/10\n",
      "33334/33334 [==============================] - 5s 146us/step - loss: 0.6228 - acc: 0.7865 - inTop3: 0.9548 - val_loss: 0.6473 - val_acc: 0.7853 - val_inTop3: 0.9549\n",
      "Epoch 5/10\n",
      "33334/33334 [==============================] - 5s 145us/step - loss: 0.6049 - acc: 0.7923 - inTop3: 0.9588 - val_loss: 0.6059 - val_acc: 0.7989 - val_inTop3: 0.9569\n",
      "Epoch 6/10\n",
      "33334/33334 [==============================] - 5s 145us/step - loss: 0.5822 - acc: 0.8029 - inTop3: 0.9609 - val_loss: 0.6522 - val_acc: 0.7844 - val_inTop3: 0.9477\n",
      "Epoch 7/10\n",
      "33334/33334 [==============================] - 5s 147us/step - loss: 0.5632 - acc: 0.8096 - inTop3: 0.9612 - val_loss: 0.5840 - val_acc: 0.8009 - val_inTop3: 0.9614\n",
      "Epoch 8/10\n",
      "33334/33334 [==============================] - 5s 148us/step - loss: 0.5457 - acc: 0.8122 - inTop3: 0.9650 - val_loss: 0.6041 - val_acc: 0.7954 - val_inTop3: 0.9594\n",
      "Epoch 9/10\n",
      "33334/33334 [==============================] - 5s 148us/step - loss: 0.5261 - acc: 0.8223 - inTop3: 0.9675 - val_loss: 0.5769 - val_acc: 0.8095 - val_inTop3: 0.9611\n",
      "Epoch 10/10\n",
      "33334/33334 [==============================] - 5s 148us/step - loss: 0.5363 - acc: 0.8195 - inTop3: 0.9658 - val_loss: 0.6080 - val_acc: 0.7944 - val_inTop3: 0.9578\n"
     ]
    }
   ],
   "source": [
    "en_tf_halo=ensemble(get_tfMLP, x_RN, y_tan, 10, 3, 3, 32)\n",
    "en_tf_halo=link_transf(almod, en_tf_halo)\n",
    "halo_mentes(en_tf_halo,'ensemble_CNN')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Értékeljük ki a szakértő együttesünket."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 4s 426us/step\n",
      "Confusion Matrix:\n",
      "[[825  15  18  12   7   1   0  20  85  17]\n",
      " [  6 946   2   4   1   1   1   3  11  25]\n",
      " [ 43   4 755  56  69  19  25  19   6   4]\n",
      " [  7   5  43 751  31  78  32  39   7   7]\n",
      " [ 11   0  53  53 771  17  34  52   8   1]\n",
      " [  3   2  29 139  26 713  17  66   4   1]\n",
      " [  6   7  26  60  21  16 848   4   9   3]\n",
      " [  5   2  12  25  55  26   6 859   7   3]\n",
      " [ 32  21  13   6   3   0   2   1 909  13]\n",
      " [ 11  82   3   5   1   0   1  11  21 865]]\n",
      "\n",
      "Loss:0.5154574652194976; Acc:0.8242; Top3Acc:0.9673\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'loss': 0.5154574652194976, 'acc': 0.8242, 'top3acc': 0.9673}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "teszt(en_tf_halo,x_tst,y_tst)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Egy tizeddel sikerült javítanunk (különböző versenyek van, hogy ennyin múlnak)..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}