Keras univariate time-series CNN+LSTM

Question

I'm trying to implement a CNN+LSTM network to predict the next step of a univariate time series presented in this article.

The data shape is (2922,1) that I change to have a window of 250 steps like the example below if instead of 250 the window was 10:

x =
    [[  0.   1.   2. ...   7.   8.   9.]
    [  1.   2.   3. ...   8.   9.  10.]
    [  2.   3.   4. ...   9.  10.  11.]
    ...
    [987. 988. 989. ... 994. 995. 996.]
    [988. 989. 990. ... 995. 996. 997.]
    [989. 990. 991. ... 996. 997. 998.]]

y =
    [[ 10.]
    [ 11.]
    [ 12.]
    ...
    [997.]
    [998.]
    [999.]]

Then I reshape it to fit in the network:

x_train = x_train.reshape((x_train.shape[0], x_train.shape[1], 1))
x_val = x_val.reshape((x_val.shape[0], x_val.shape[1], 1))

>>>Input shape: (1794, 250)
>>>Target shape: (1794, 1)

And build the network model

model = keras.models.Sequential()
model.add(keras.layers.TimeDistributed(keras.layers.Conv1D(filters=512, kernel_size=2)))
model.add(keras.layers.TimeDistributed(keras.layers.MaxPooling1D(pool_size=2, strides=1, padding='valid')))
model.add(keras.layers.TimeDistributed(keras.layers.Flatten()))
model.add(keras.layers.LSTM(250, return_sequences=True))
model.add(keras.layers.Dense(1))

model.compile('adam', 'binary_crossentropy', metrics=['accuracy'])  
model.build((x_train.shape[0], x_train.shape[1], 1))
model.summary()

This gives me the error:

ValueError: Exception encountered when calling layer "time_distributed" (type TimeDistributed).

Input 0 of layer "conv1d" is incompatible with the layer: expected min_ndim=3, found ndim=2. Full shape received: (448500, 1)

Call arguments received:
  • inputs=tf.Tensor(shape=(1794, 250, 1), dtype=float32)
  • training=None
  • mask=None

Here is a fully functional code that gives the same error if anyone wants to try it out.

import pandas as pd
import matplotlib.pyplot as plt
import math
import numpy as np
from tensorflow import keras


def one_step_ahead_sliding_window(array,window_size):
    window = np.zeros((len(array)-window_size,window_size))
    output = np.zeros((len(array)-window_size,1))
    
    for stop in range(window_size,len(array)):
        window[i,:] = array[stop-window_size:stop]
        output[i,0] = array[stop]
        i=i+1

    return window, output


def normalize(data, train_split):

    train_data = data[0 : train_split - 1]
    val_data = data[train_split:]

    mu = data[:train_split].mean(axis=0)
    std = data[:train_split].std(axis=0)
    train_data_normalized = (train_data - mu) / std
    test_data_normalized = (val_data - mu) / std

    return train_data_normalized, test_data_normalized


data = np.arange(2922)

split_train = 0.7
split_test = 0.15
window_size = 250
learning_rate = 0.001
epochs = 400
n_features = 1


train_split = int(split_train * int(data.shape[0]))
test_split = int(split_test * int(data.shape[0]))

train_data_normalized, test_val_data_normalized = normalize(data, train_split)
val_data_normalized = test_val_data_normalized[:int(len(test_val_data_normalized)/2)]
test_data_normalized = test_val_data_normalized[int(len(test_val_data_normalized)/2):]

x_train, y_train = one_step_ahead_sliding_window(train_data_normalized, window_size)
x_val, y_val = one_step_ahead_sliding_window(val_data_normalized, window_size)
x_test, y_test = one_step_ahead_sliding_window(test_data_normalized, window_size)

print("Input shape:", x_train.shape)
print("Target shape:", y_train.shape, end="\n\n")


x_train = x_train.reshape((x_train.shape[0], x_train.shape[1], n_features))
x_val = x_val.reshape((x_val.shape[0], x_val.shape[1], n_features))


model = keras.models.Sequential()
model.add(keras.layers.TimeDistributed(keras.layers.Conv1D(filters=512, kernel_size=2)))
model.add(keras.layers.TimeDistributed(keras.layers.MaxPooling1D(pool_size=2, strides=1, padding='valid')))
model.add(keras.layers.TimeDistributed(keras.layers.Flatten()))
model.add(keras.layers.LSTM(250, return_sequences=True))
model.add(keras.layers.Dense(1))


model.compile('adam', 'binary_crossentropy', metrics=['accuracy'])
              
model.build((x_train.shape[0], x_train.shape[1], 1))

model.summary()