{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "data": { "text/html": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Using TensorFlow backend.\n" ] }, { "data": { "text/html": [ "\n", " show code\n", " " ], "text/plain": [ "" ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ "base_dir = 'D:\\\\deep_learning\\\\adv'\n", "%run ../initscript.py\n", "# %run ../display.py\n", "import pandas as pd\n", "import numpy as np\n", "import scipy.stats as st\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "from ipywidgets import *\n", "%matplotlib inline\n", "import tensorflow as tf\n", "tf.logging.set_verbosity(tf.logging.ERROR)\n", "\n", "import os\n", "from keras import optimizers\n", "from keras import backend as K\n", "from keras import models, layers, Input\n", "from keras import initializers\n", "from keras import preprocessing\n", "from keras.utils import to_categorical\n", "\n", "toggle()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Advanced Deep-Learning" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Non-Sequential Models" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "All previous introduced models are *sequential* models.\n", "\n", "\n", "\n", "Non-sequential models are more flexible for many applications, for example, some tasks require several independent inputs, others require multiple outputs. \n", "\n", "Imagine a deep-learning model trying to predict the most likely market price of a second-hand piece of clothing, using the following inputs: user-provided metadata (such as the item's brand, age, and so on), a user-provided text description, and a picture of the item. It requires a model with three input branches as follows.\n", "\n", "" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "_________________________________________________________________\n", "Layer (type) Output Shape Param # \n", "=================================================================\n", "dense_1 (Dense) (None, 32) 2080 \n", "_________________________________________________________________\n", "dense_2 (Dense) (None, 32) 1056 \n", "_________________________________________________________________\n", "dense_3 (Dense) (None, 10) 330 \n", "=================================================================\n", "Total params: 3,466\n", "Trainable params: 3,466\n", "Non-trainable params: 0\n", "_________________________________________________________________\n" ] } ], "source": [ "seq_model = models.Sequential()\n", "seq_model.add(layers.Dense(32, activation='relu', input_shape=(64,)))\n", "seq_model.add(layers.Dense(32, activation='relu'))\n", "seq_model.add(layers.Dense(10, activation='softmax'))\n", "seq_model.summary()" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "_________________________________________________________________\n", "Layer (type) Output Shape Param # \n", "=================================================================\n", "input_1 (InputLayer) (None, 64) 0 \n", "_________________________________________________________________\n", "dense_4 (Dense) (None, 32) 2080 \n", "_________________________________________________________________\n", "dense_5 (Dense) (None, 32) 1056 \n", "_________________________________________________________________\n", "dense_6 (Dense) (None, 10) 330 \n", "=================================================================\n", "Total params: 3,466\n", "Trainable params: 3,466\n", "Non-trainable params: 0\n", "_________________________________________________________________\n" ] } ], "source": [ "input_tensor = Input(shape=(64,))\n", "x = layers.Dense(32, activation='relu')(input_tensor)\n", "x = layers.Dense(32, activation='relu')(x)\n", "output_tensor = layers.Dense(10, activation='softmax')(x)\n", "model = models.Model(input_tensor, output_tensor)\n", "model.summary()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We build a `Model` object with an input and output tensors. Note that `output_tensor` is obtained by repeatedly transforming `input_tensor`. This `Model` object is equivalent to `seq_model` object." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Multiple Inputs Models" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "__________________________________________________________________________________________________\n", "Layer (type) Output Shape Param # Connected to \n", "==================================================================================================\n", "text (InputLayer) (None, None) 0 \n", "__________________________________________________________________________________________________\n", "question (InputLayer) (None, None) 0 \n", "__________________________________________________________________________________________________\n", "embedding_1 (Embedding) (None, None, 64) 640000 text[0][0] \n", "__________________________________________________________________________________________________\n", "embedding_2 (Embedding) (None, None, 32) 320000 question[0][0] \n", "__________________________________________________________________________________________________\n", "lstm_1 (LSTM) (None, 32) 12416 embedding_1[0][0] \n", "__________________________________________________________________________________________________\n", "lstm_2 (LSTM) (None, 16) 3136 embedding_2[0][0] \n", "__________________________________________________________________________________________________\n", "concatenate_1 (Concatenate) (None, 48) 0 lstm_1[0][0] \n", " lstm_2[0][0] \n", "__________________________________________________________________________________________________\n", "dense_7 (Dense) (None, 500) 24500 concatenate_1[0][0] \n", "==================================================================================================\n", "Total params: 1,000,052\n", "Trainable params: 1,000,052\n", "Non-trainable params: 0\n", "__________________________________________________________________________________________________\n" ] } ], "source": [ "text_vocabulary_size = 10000\n", "question_vocabulary_size = 10000\n", "answer_vocabulary_size = 500\n", "\n", "text_input = Input(shape=(None,), dtype='int32', name='text')\n", "embedded_text = layers.Embedding(text_vocabulary_size, 64)(text_input)\n", "encoded_text = layers.LSTM(32)(embedded_text)\n", "\n", "question_input = Input(shape=(None,),dtype='int32',name='question')\n", "embedded_question = layers.Embedding(question_vocabulary_size, 32)(question_input)\n", "encoded_question = layers.LSTM(16)(embedded_question)\n", "\n", "concatenated = layers.concatenate([encoded_text, encoded_question],axis=-1)\n", "\n", "answer = layers.Dense(answer_vocabulary_size,activation='softmax')(concatenated)\n", "\n", "model = models.Model([text_input, question_input], answer)\n", "model.compile(optimizer='rmsprop',loss='categorical_crossentropy',metrics=['acc'])\n", "model.summary()" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 1/1\n", "1000/1000 [==============================] - 9s 9ms/step - loss: 6.2148 - acc: 0.0030\n" ] } ], "source": [ "num_samples = 1000\n", "max_length = 100\n", "text = np.random.randint(1, text_vocabulary_size, size=(num_samples, max_length))\n", "question = np.random.randint(1, question_vocabulary_size, size=(num_samples, max_length))\n", "answers = np.zeros((num_samples, answer_vocabulary_size)) \n", "answers[range(answers.shape[0]), np.random.randint(answers.shape[1], size=answers.shape[0])] = 1\n", "\n", "\n", "model.fit([text, question], answers, epochs=1, batch_size=128);\n", "# model.fit({'text': text, 'question': question}, answers, epochs=10, batch_size=128)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Multi-Output Models" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "vocabulary_size = 50000\n", "num_income_groups = 10\n", "posts_input = Input(shape=(None,), dtype='int32', name='posts')\n", "embedded_posts = layers.Embedding(256, vocabulary_size)(posts_input)\n", "x = layers.Conv1D(128, 5, activation='relu')(embedded_posts)\n", "x = layers.MaxPooling1D(5)(x)\n", "x = layers.Conv1D(256, 5, activation='relu')(x)\n", "x = layers.Conv1D(256, 5, activation='relu')(x)\n", "x = layers.MaxPooling1D(5)(x)\n", "x = layers.Conv1D(256, 5, activation='relu')(x)\n", "x = layers.Conv1D(256, 5, activation='relu')(x)\n", "x = layers.GlobalMaxPooling1D()(x)\n", "x = layers.Dense(128, activation='relu')(x)\n", "age_prediction = layers.Dense(1, name='age')(x)\n", "income_prediction = layers.Dense(num_income_groups,activation='softmax',name='income')(x)\n", "gender_prediction = layers.Dense(1, activation='sigmoid', name='gender')(x)\n", "model = models.Model(posts_input,[age_prediction, income_prediction, gender_prediction])\n", "model.compile(optimizer='rmsprop',\n", " loss={'age': 'mse','income': 'categorical_crossentropy','gender': 'binary_crossentropy'},\n", " loss_weights=[0.25, 1., 10.])\n", "# model.fit(posts, [age_targets, income_targets, gender_targets], epochs=10, batch_size=64)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.3" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false }, "widgets": { "application/vnd.jupyter.widget-state+json": { "state": {}, "version_major": 2, "version_minor": 0 } } }, "nbformat": 4, "nbformat_minor": 2 }