The basic model is: input_data -> (lstm rolls up input into single vector) -> relu -> output

import keras
from keras.models import Sequential, Model, load_model
from keras.layers import Dense, Activation, Flatten, LSTM, SimpleRNN, Input, Merge, GRU
from keras.layers.noise import GaussianNoise
from keras.optimizers import Adam
from keras.callbacks import EarlyStopping
from keras.initializers import RandomUniform, TruncatedNormal, glorot_uniform
from keras.callbacks import Callback
from keras.utils import to_categorical

class PlotLossHistory(keras.callbacks.Callback):
    """
    We use this to plot our losses during training. Keras provides
    nice callback functionality.
    """

    def on_train_begin(self, logs={}):
        self.test_loss_softmax = []
        self.test_loss_mse = []
        self.train_loss_softmax = []
        self.train_loss_mse = []

    def on_epoch_end(self, batch, logs={}):
        plt.clf()

        fig = plt.figure()
        ax1 = fig.add_subplot(111)

        self.train_loss_softmax.append( logs['output_softmax_loss'] )
        ax1.plot(self.train_loss_softmax)

        self.test_loss_softmax.append( logs['val_output_softmax_loss'] )
        ax1.plot(self.test_loss_softmax)

        ax2 = ax1.twinx()

        self.train_loss_mse.append( logs['output_predictions_loss'] )
        ax2.plot(self.train_loss_mse)

        self.test_loss_mse.append( logs['val_output_predictions_loss'] )
        ax2.plot(self.test_loss_mse)
        ax2.set_yscale('log')

        #plt.legend(['TRAIN softmax', 'TRAIN mse', 'TEST softmax', 'TEST mse'], loc='upper right')

        plt.savefig('output_losses.png')

# create entry point for model input
inputs = Input(shape=input_shape)

# setup model
# initializer = RandomUniform(-.001, .001)
lstm = LSTM(128)
timesteps_rolled_up = lstm(inputs)
relu = Dense(32, activation="relu")(timesteps_rolled_up)
output_prediction = Dense(num_outputs_linear, activation="linear", name="output_predictions")(relu)
output_softmax = Dense(num_outputs_softmax, activation="softmax", name="output_softmax")(relu)

model = Model(inputs=[inputs], outputs=[output_softmax, output_prediction])


# setup training and compile graph
opt = Adam(lr=0.0001, clipnorm=10.)
model.compile(loss={"output_softmax": "categorical_crossentropy", "output_predictions": "mse"}, 
                        optimizer=opt, 
                        metrics=['accuracy',])

# one last fix to the data -- go from integer to 1-hot vectors suitable for softmax outputs
data_Y1 = to_categorical(np.int32(data_Y1), num_outputs_softmax)
test_Y1 = to_categorical(np.int32(test_Y1), num_outputs_softmax)
# if validation loss stops improving and/or starts overfitting too much, we'll stop training
early_stop = EarlyStopping('val_loss', .0001, patience=3, verbose=1, mode='auto')
# train your model, coffee break
model.fit(data_X, {"output_softmax": np.int32(data_Y1), "output_predictions": data_Y2}, 
        epochs=epochs, batch_size=128,
        validation_data=(test_X, {"output_softmax": np.int32(test_Y1), "output_predictions": test_Y2}), 
        shuffle=True, callbacks=[early_stop, PlotLossHistory()])

print 'done training'
# model.predict(data_X[:1]) # make a single prediction with your model

import tensorflow as tf
sess = tf.Session()

class TFdataset:
    """ 
    this will hold our data and labels and handle shuffling
    """

    def __init__(self, data_X, data_Y1, data_Y2, batch_size):
        self.data_X = data_X
        self.data_Y1 = to_categorical(data_Y1, np.max(data_Y1)+1) # softmax
        self.data_Y2 = data_Y2 # mse
        self.batch_size = batch_size
        pass

    def iter(self):
        batch_idx = 0
        while True:
            # shuffle data
            idxs = np.arange(0, len(self.data_X))
            np.random.shuffle(idxs)
            shuf_features = self.data_X[idxs]
            shuf_Y1 = self.data_Y1[idxs]
            shuf_Y2 = self.data_Y2[idxs]

            for batch_idx in range(0, len(self.data_X), self.batch_size):
                data_batch = shuf_features[batch_idx: batch_idx+self.batch_size]
                data_batch = data_batch.astype('float32')
                Y1_batch = shuf_Y1[batch_idx: batch_idx+self.batch_size]
                Y2_batch = shuf_Y2[batch_idx: batch_idx+self.batch_size]

                yield data_batch, Y1_batch, Y2_batch


# params =========================
BATCH_SIZE = 128
BATCH_SIZE_TEST = 1024
LSTM_SIZE = 128
LEARNING_RATE = .001
training_iters = 4000
display_step=20
TIMESTEPS = 200
DROPOUT = 1. # 1. is no dropout
#=================================

# setup your data iterators
dset_train = TFdataset(data_X, data_Y1, data_Y2, BATCH_SIZE)
dset_test = TFdataset(test_X, test_Y1, test_Y2, BATCH_SIZE_TEST)

data_iterator = dset_train.iter()

# variables we'll use shortly
n_steps = TIMESTEPS
n_input = num_inputs
n_outputs_sm = num_outputs_softmax
n_outputs_mse = num_outputs_linear

# setup input X and output Y graph variables
x = tf.placeholder('float', [None, n_steps, n_input])
y = tf.placeholder('float', [None, n_outputs_sm])

# init information for model weights
weights = {
        'out': tf.Variable(tf.random_uniform([LSTM_SIZE, n_outputs_sm]))
        }
biases = {
        'out': tf.Variable(tf.random_uniform([n_outputs_sm]))
        }

# Feed forward function to get the RNN output. We're using a fancy type of LSTM cell.
def TFEncoderRNN(inp, weights, biases):
    # current_input_shape: (batch_size, n_steps, n_input
    # required shape: 'n_steps' tensors list of shape (batch_size, n_input)
    inp = tf.unstack(inp, n_steps, 1)
    lstm_cell = tf.contrib.rnn.LayerNormBasicLSTMCell(LSTM_SIZE, dropout_keep_prob=DROPOUT)
    outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, inp, dtype=tf.float32)
    return tf.matmul(outputs[-1], weights['out']) + biases['out']

# we'll be able to call this to get our model output
pred = TFEncoderRNN(x, weights, biases)
# define loss
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))

# learning rate decay
global_step = tf.Variable(0, trainable=False)
lr_scheduler = tf.train.exponential_decay(LEARNING_RATE, global_step,
                                            training_iters, 0.96, staircase=True)
# define optimizer
optimizer = tf.train.AdamOptimizer(learning_rate=lr_scheduler).minimize(cost)

# define evaluation params
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

init = tf.global_variables_initializer()

# train model
config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=True)
config.gpu_options.allow_growth = True

sess = tf.Session(config=config)

sess.run(init)
step = 1

print colored('start training', 'blue')
softmax_loss_history = []
training_accuracy = []
softmax_loss_history_test = []
test_accuracy_history = []

# model training loop
data_iterator = dset_train.iter()
test_iterator = dset_test.iter()
while step < training_iters:
    batch_x, batch_sm, batch_mse = data_iterator.next()

    sess.run(optimizer, feed_dict={x: batch_x, y: batch_sm})

    if step % display_step == 0:

        data_iterator = dset_train.iter()
        test_iterator = dset_test.iter()

        # get training stats
        acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_sm})
        loss = sess.run(cost, feed_dict={x: batch_x, y: batch_sm})
        softmax_loss_history.append(loss)
        training_accuracy.append(acc)

        # get test data stats
        test_X, test_softmax, test_mse = test_iterator.next()

        acc_test = sess.run(accuracy, feed_dict={x: test_X, y: test_softmax})
        loss_test = sess.run(cost, feed_dict={x: test_X, y: test_softmax})

        softmax_loss_history_test.append(loss_test)
        test_accuracy_history.append(acc_test)

        print 'Iter %i, Minibatch Loss= %.6f, Training Accuracy= %.5f' % (step, loss, acc)
        pytorch_plot_losses(softmax_loss_history=softmax_loss_history,
                                            mse_loss_history=training_accuracy,
                                            test_losses_softmax=softmax_loss_history_test,
                                            test_losses_mse=test_accuracy_history)
    step += 1

print "Optimization finished!"

from torch.autograd import Variable
import torch.optim as optim
import torch.nn.functional as torch_func
import torch

def pytorch_plot_losses(softmax_loss_history=None, mse_loss_history=None, 
                                        test_losses_softmax=None, test_losses_mse=None):
    plt.clf()
    fig = plt.figure()
    ax1 = fig.add_subplot(111)
    if softmax_loss_history:
        ax1.plot(softmax_loss_history, color="blue")
    if test_losses_softmax:
        ax1.plot(test_losses_softmax, color="green")
    ax2 = ax1.twinx()
    if mse_loss_history:
        ax2.plot(mse_loss_history, color="red")
    if test_losses_mse:
        ax2.plot(test_losses_mse, color="black")
    #ax2.set_yscale('log')
    plt.savefig('output_losses.png')

class EncoderRNN(nn.Module):
    """
    pytorch class to encode a series of timesteps into a single vector
    """
    def __init__(self, num_inputs, hidden_units, num_outputs, batch_size):
        super(EncoderRNN, self).__init__()
        self.batch_size = batch_size
        self.num_inputs = num_inputs
        self.hidden_units = hidden_units
        self.lstm = nn.LSTM(num_inputs, hidden_units, num_layers=1, batch_first=True).cuda()
        self.relu = nn.ReLU().cuda()
        self.fc_out = nn.Linear(hidden_units, num_outputs).cuda()
        n = hidden_units * num_inputs
        self.lstm.weight_hh_l0.data.normal_(0, np.sqrt(2. / n))
        self.lstm.weight_ih_l0.data.normal_(0, np.sqrt(2. / n))
        self.fc_out.weight.data.normal_(0, np.sqrt(2. / n))

    def forward(self, input, hidden):
        output, hiddens = self.lstm(input, hidden)
        ret_output = self.relu(self.fc_out(output[:,-1,:]))
        return ret_output

    def initHidden(self, batch_size):
        result = Variable(torch.zeros(1, batch_size, self.hidden_units))
        return result.cuda()

class DecoderRNN(nn.Module):
    """
    Given an encoded hidden state, can decode into real values or a softmax
    """

    def __init__(self, input_size, hidden_size, output_size, output_type=None):

        super(DecoderRNN, self).__init__()

        self.hidden_size = hidden_size
        self.output_size = output_size
        self.output_type = output_type

        n = hidden_size * input_size
        self.fc1 = nn.Linear(input_size, hidden_size).cuda()
        self.relu = nn.ReLU().cuda()
        self.fc_output = nn.Linear(hidden_size, output_size).cuda()

        self.fc1.weight.data.normal_(0, np.sqrt(2. / n))
        self.fc_output.weight.data.normal_(0, np.sqrt(2. / n))

        if output_type=="softmax":
            self.softmax = nn.LogSoftmax().cuda()
        elif output_type=="linear":
            self.softmax = None
        else:
            raise Exception('Initializing decoder RNN without supplying "output_type"')

    def forward(self, input):
        relu_out = self.relu( self.fc1( input ) )
        output = self.fc_output( relu_out )
        if self.output_type=='softmax':
            # apply softmax to output to get probablity distribution,
            # otherwise just output raw output values
            output = self.softmax(output)
        else:
            output = torch_func.tanh(output)
        return output

class PytorchStockDataset(tdata.Dataset):
    """
    Pytorch wants you to define a fancy dataset wrapper class
    """

    def __init__(self, data_X, Y_softmax, Y_mse, inflate_y=False):
        assert len(data_X) == len(Y_softmax) == len(Y_mse)
        if inflate_y:
            self.data_Y_softmax = to_categorical(np.float32(Y_softmax.squeeze()), np.max(Y_softmax)+1)
            self.data_X = data_X
            self.data_Y_mse = Y_mse
        else:
            self.data_Y_softmax = torch.from_numpy(Y_softmax)
            self.data_X = torch.from_numpy(data_X)
            self.data_Y_mse = torch.from_numpy(Y_mse)

    def __getitem__(self, index):
        x = self.data_X[index]
        y_mse = self.data_Y_mse[index]
        y_softmax = self.data_Y_softmax[index]
        return x, y_mse, y_softmax

    def __len__(self):
        return len(self.data_X)

# params =======================
BATCH_SIZE = 128
BATCH_SIZE_TEST = 1024
LSTM_SIZE = 128
ENCODER_OUTPUT_SIZE = 1024
RELU_SIZE = 512
lr_encoder = .0001
lr_decoder_softmax = .0005
lr_decoder_mse = .0001
# =============================

# setup data for pytorch
dset_train = PytorchStockDataset(data_X, Y_softmax=data_Y1, Y_mse=data_Y2)
dset_test = PytorchStockDataset(test_X, Y_softmax=test_Y1, Y_mse=test_Y2)

data_sampler_train = tdata.DataLoader(dset_train, batch_size=BATCH_SIZE, shuffle=True, drop_last=True)
data_sampler_test = tdata.DataLoader(dset_test, batch_size=BATCH_SIZE_TEST, shuffle=False, drop_last=True)

# init your encoder and two decoders
encoder = EncoderRNN(num_inputs, LSTM_SIZE, ENCODER_OUTPUT_SIZE, batch_size=BATCH_SIZE)
decoder_softmax = DecoderRNN(ENCODER_OUTPUT_SIZE, RELU_SIZE, num_outputs_softmax, output_type='softmax')
decoder_mse = DecoderRNN(ENCODER_OUTPUT_SIZE, RELU_SIZE, num_outputs_linear, output_type='linear')

# create your optimizers
optimizer_enc = torch.optim.Adam(encoder.parameters(), lr=lr_encoder)
optimizer_sm_dec = torch.optim.Adam(decoder_softmax.parameters(), lr=lr_decoder_softmax)
optimizer_mse_dec = torch.optim.Adam(decoder_mse.parameters(), lr=lr_decoder_mse)

# run training for N epochs
batches_per_epoch = 100
epoch_losses_softmax = []
epoch_losses_mse = []
test_losses_softmax = []
test_losses_mse = []

best_iteration = 0
best_validation_loss = 999.
best_encoder_params = None
best_decoder_params = None
best_decoder_paramse_mse = None

for i in range(250):

    data_iterator = iter(data_sampler_train)
    test_iterator = iter(data_sampler_test)

    print 'EPOCH: %d' % i 

    # run each epoch for N training batches
    iter_losses_mse = []
    iter_losses_softmax = []

    for iteration in range(batches_per_epoch):
        batch_X, batch_mse, batch_softmax = data_iterator.next()

        hx = encoder.initHidden(BATCH_SIZE)
        cx = encoder.initHidden(BATCH_SIZE)

        # clear our cached gradient
        optimizer_enc.zero_grad()
        optimizer_mse_dec.zero_grad()
        optimizer_sm_dec.zero_grad()

        # encode our sequence
        var_X = Variable(batch_X, requires_grad=True).cuda() 
        encoder_output = encoder(var_X, (hx, cx))

        # decode our sequence
        decoded_mse = decoder_mse(encoder_output)
        decoded_softmax = decoder_softmax(encoder_output)

        # compute loss from output and labels
        criterion_softmax = nn.CrossEntropyLoss()
        var_y_sm = Variable(batch_softmax, requires_grad=False)[:,0].long().cuda()
        loss_softmax = criterion_softmax(decoded_softmax, var_y_sm)

        # do the same for the second target w/ different cost function
        criterion_mse = nn.MSELoss()
        var_y_mse = Variable(batch_mse, requires_grad=False).cuda()
        loss_mse = criterion_mse(decoded_mse, var_y_mse)

        # add those losses to our history for plotting purposes
        iter_losses_mse.append(loss_mse.data.cpu().numpy()[0])
        iter_losses_softmax.append(loss_softmax.data.cpu().numpy()[0])

        # backprop losses
        loss_mse.backward(retain_variables=True)
        loss_softmax.backward(retain_variables=True)

        # update weights from computed gradient
        optimizer_mse_dec.step()
        optimizer_sm_dec.step()
        optimizer_enc.step()

    #####
    # run validation data
    #####
    test_X, test_mse, test_softmax = test_iterator.next()

    hx = encoder.initHidden(BATCH_SIZE_TEST)
    cx = encoder.initHidden(BATCH_SIZE_TEST)

    # encode our sequence
    test_var = Variable(test_X, requires_grad=False).cuda()
    encoder_test_output = encoder(test_var, (hx, cx))

    # decode our sequence
    decoded_test_mse = decoder_mse(encoder_test_output)
    decoded_test_softmax = decoder_softmax(encoder_test_output)

    # compute loss from output and labels
    criterion_softmax = nn.CrossEntropyLoss()
    var_y_sm_test = Variable(test_softmax, requires_grad=False)[:,0].long().cuda()
    loss_softmax = criterion_softmax(decoded_test_softmax, var_y_sm_test)

    # do the same for the second target w/ different cost function
    criterion_mse = nn.MSELoss()
    var_y_mse_test = Variable(test_mse, requires_grad=False).cuda()
    loss_mse = criterion_mse(decoded_test_mse, var_y_mse_test)

    test_loss_mse = loss_mse.data.cpu().numpy()[0]
    test_loss_softmax = loss_softmax.data.cpu().numpy()[0]

    test_losses_mse.append(test_loss_mse)
    test_losses_softmax.append(test_loss_softmax)



    if test_loss_softmax < best_validation_loss:
        print colored('NEW BEST DETECTED', 'green')
        best_iteration = i
        best_validation_loss = test_loss_softmax
        # save state dicts containing weights
        best_encoder_params = encoder.state_dict()
        best_decoder_params = decoder_softmax.state_dict()
        best_decoder_params_mse = decoder_mse.state_dict()
        # clone params in state dicts so they don't get overwritten
        for key in best_encoder_params.keys():
            best_encoder_params[key] = best_encoder_params[key].clone()
        for key in best_decoder_params.keys():
            best_decoder_params[key] = best_decoder_params[key].clone()
        for key in best_decoder_params_mse.keys():
            best_decoder_params_mse[key] = best_decoder_params_mse[key].clone()

    times_since_last_best = i - best_iteration
    if times_since_last_best > 30:
        # early stopping
        break

        '''
        this was experimental code where i was trying to help overfitting by
        (after a certain number of iterations with no validation loss improvement)
        reload our best saved validation score and add noise to some of the weights,
        reduce the learning rate, and keep optimizing. The thinking here is maybe it
        can get bumped into a better location. This kind of helped? But it's very
        much unnecessary and I'm leaving it in just in case anyone is interested.
        '''
        dropout_factor = .2
        print colored('WE SHOULD REJIGGER THE WEIGHTS, VALIDATION HAS STOPPED IMPROVING', 'red')
        best_iteration = i
        keys = best_encoder_params.keys()
        for key in keys:
            dat = best_encoder_params[key]
            best_encoder_params[key] = dat #+ torch.from_numpy(np.random.uniform(-.0000001,.0000001,dat.size())*np.random.binomial(1.,dropout_factor,dat.size())).cuda().float()
        encoder.load_state_dict(best_encoder_params)

        keys = best_decoder_params.keys()
        for key in keys:
            dat = best_decoder_params[key]
            best_decoder_params[key] = dat #+ torch.from_numpy(np.random.uniform(-.0000001,.0000001,dat.size())*np.random.binomial(1.,dropout_factor,dat.size())).cuda().float()
        decoder_softmax.load_state_dict(best_decoder_params)

        keys = best_decoder_params_mse.keys()
        for key in keys:
            dat = best_decoder_params_mse[key]
            best_decoder_params_mse[key] = dat #+ torch.from_numpy(np.random.uniform(-.0000001,.0000001,dat.size())*np.random.binomial(1.,dropout_factor,dat.size())).cuda().float()
        decoder_mse.load_state_dict(best_decoder_params_mse)
        #[x.size() for x in encoder.parameters()]
        #import pdb; pdb.set_trace()
        #noise = np.random.random()(params.std()*3)

        # when we revert back to a spot near our original place,
        # halve the learning rate also
        if lr_encoder > .00005:
            lr_encoder = lr_encoder * .8
            lr_decoder_softmax = lr_decoder_softmax * .8
            lr_decoder_mse = lr_decoder_mse * .8

        optimizer_enc = torch.optim.Adam(encoder.parameters(), lr=lr_encoder)
        optimizer_sm_dec = torch.optim.Adam(decoder_softmax.parameters(), lr=lr_decoder_softmax)
        optimizer_mse_dec = torch.optim.Adam(decoder_mse.parameters(), lr=lr_decoder_mse)


    ####

    # report losses (hopefully they're decreasing)
    print 'MSE loss: %.8f' % np.mean(iter_losses_mse)
    print 'SMX loss: %.8f' % np.mean(iter_losses_softmax)
    epoch_losses_mse.append( np.mean(iter_losses_mse) )
    epoch_losses_softmax.append( np.mean(iter_losses_softmax) )
    pytorch_plot_losses( epoch_losses_softmax, epoch_losses_mse, test_losses_softmax, test_losses_mse)

# revert weights to "best weights"
print colored('*** reverting weights back to iteration %i, which was our best' % best_iteration, 'yellow')
encoder.load_state_dict(best_encoder_params)
decoder_softmax.load_state_dict(best_decoder_params)
decoder_mse.load_state_dict(best_decoder_params_mse)

print '========== optimization finished! ============'