base_dir = 'D:\\deep_learning\\adv'
%run ../initscript.py
# %run ../display.py
import pandas as pd
import numpy as np
import scipy.stats as st
import matplotlib.pyplot as plt
import seaborn as sns
from ipywidgets import *
%matplotlib inline
import tensorflow as tf
tf.logging.set_verbosity(tf.logging.ERROR)

import os
from keras import optimizers
from keras import backend as K
from keras import models, layers, Input
from keras import initializers
from keras import preprocessing
from keras.utils import to_categorical

toggle()

Using TensorFlow backend.

show code

Advanced Deep-Learning

Non-Sequential Models

All previous introduced models are sequential models.

Non-sequential models are more flexible for many applications, for example, some tasks require several independent inputs, others require multiple outputs.

Imagine a deep-learning model trying to predict the most likely market price of a second-hand piece of clothing, using the following inputs: user-provided metadata (such as the item’s brand, age, and so on), a user-provided text description, and a picture of the item. It requires a model with three input branches as follows.

seq_model = models.Sequential()
seq_model.add(layers.Dense(32, activation='relu', input_shape=(64,)))
seq_model.add(layers.Dense(32, activation='relu'))
seq_model.add(layers.Dense(10, activation='softmax'))
seq_model.summary()

_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
dense_1 (Dense)              (None, 32)                2080
_________________________________________________________________
dense_2 (Dense)              (None, 32)                1056
_________________________________________________________________
dense_3 (Dense)              (None, 10)                330
=================================================================
Total params: 3,466
Trainable params: 3,466
Non-trainable params: 0
_________________________________________________________________

input_tensor = Input(shape=(64,))
x = layers.Dense(32, activation='relu')(input_tensor)
x = layers.Dense(32, activation='relu')(x)
output_tensor = layers.Dense(10, activation='softmax')(x)
model = models.Model(input_tensor, output_tensor)
model.summary()

_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
input_1 (InputLayer)         (None, 64)                0
_________________________________________________________________
dense_4 (Dense)              (None, 32)                2080
_________________________________________________________________
dense_5 (Dense)              (None, 32)                1056
_________________________________________________________________
dense_6 (Dense)              (None, 10)                330
=================================================================
Total params: 3,466
Trainable params: 3,466
Non-trainable params: 0
_________________________________________________________________

We build a Model object with an input and output tensors. Note that output_tensor is obtained by repeatedly transforming input_tensor. This Model object is equivalent to seq_model object.

Multiple Inputs Models

text_vocabulary_size = 10000
question_vocabulary_size = 10000
answer_vocabulary_size = 500

text_input = Input(shape=(None,), dtype='int32', name='text')
embedded_text = layers.Embedding(text_vocabulary_size, 64)(text_input)
encoded_text = layers.LSTM(32)(embedded_text)

question_input = Input(shape=(None,),dtype='int32',name='question')
embedded_question = layers.Embedding(question_vocabulary_size, 32)(question_input)
encoded_question = layers.LSTM(16)(embedded_question)

concatenated = layers.concatenate([encoded_text, encoded_question],axis=-1)

answer = layers.Dense(answer_vocabulary_size,activation='softmax')(concatenated)

model = models.Model([text_input, question_input], answer)
model.compile(optimizer='rmsprop',loss='categorical_crossentropy',metrics=['acc'])
model.summary()

__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to
==================================================================================================
text (InputLayer)               (None, None)         0
__________________________________________________________________________________________________
question (InputLayer)           (None, None)         0
__________________________________________________________________________________________________
embedding_1 (Embedding)         (None, None, 64)     640000      text[0][0]
__________________________________________________________________________________________________
embedding_2 (Embedding)         (None, None, 32)     320000      question[0][0]
__________________________________________________________________________________________________
lstm_1 (LSTM)                   (None, 32)           12416       embedding_1[0][0]
__________________________________________________________________________________________________
lstm_2 (LSTM)                   (None, 16)           3136        embedding_2[0][0]
__________________________________________________________________________________________________
concatenate_1 (Concatenate)     (None, 48)           0           lstm_1[0][0]
                                                                 lstm_2[0][0]
__________________________________________________________________________________________________
dense_7 (Dense)                 (None, 500)          24500       concatenate_1[0][0]
==================================================================================================
Total params: 1,000,052
Trainable params: 1,000,052
Non-trainable params: 0
__________________________________________________________________________________________________

num_samples = 1000
max_length = 100
text = np.random.randint(1, text_vocabulary_size, size=(num_samples, max_length))
question = np.random.randint(1, question_vocabulary_size, size=(num_samples, max_length))
answers = np.zeros((num_samples, answer_vocabulary_size))
answers[range(answers.shape[0]), np.random.randint(answers.shape[1], size=answers.shape[0])] = 1


model.fit([text, question], answers, epochs=1, batch_size=128);
# model.fit({'text': text, 'question': question}, answers, epochs=10, batch_size=128)

Epoch 1/1
1000/1000 [==============================] - 9s 9ms/step - loss: 6.2148 - acc: 0.0030

Multi-Output Models

vocabulary_size = 50000
num_income_groups = 10
posts_input = Input(shape=(None,), dtype='int32', name='posts')
embedded_posts = layers.Embedding(256, vocabulary_size)(posts_input)
x = layers.Conv1D(128, 5, activation='relu')(embedded_posts)
x = layers.MaxPooling1D(5)(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.MaxPooling1D(5)(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.GlobalMaxPooling1D()(x)
x = layers.Dense(128, activation='relu')(x)
age_prediction = layers.Dense(1, name='age')(x)
income_prediction = layers.Dense(num_income_groups,activation='softmax',name='income')(x)
gender_prediction = layers.Dense(1, activation='sigmoid', name='gender')(x)
model = models.Model(posts_input,[age_prediction, income_prediction, gender_prediction])
model.compile(optimizer='rmsprop',
              loss={'age': 'mse','income': 'categorical_crossentropy','gender': 'binary_crossentropy'},
              loss_weights=[0.25, 1., 10.])
# model.fit(posts, [age_targets, income_targets, gender_targets], epochs=10, batch_size=64)