Tensorflow Keras evaluate method return a empty list

Hello i am follow the time series forecasting tutorial in tensorflow https://www.tensorflow.org/tutorials/structured_data/time_series, I have the same project, the only difference is that I am using a different dataset, when evaluating the models, the model.evaluate () method returns an empty list, it does not return a value.

When the model is trained with the fit() method, evaluation values are generated in training. But not with the evaluate() method.

This the class WindowGenerator Class:

class WindowGenerator():
    def __init__(self, input_width, label_width, shift, 
                train_df=train_df, val_df=val_df, 
                test_df=test_df, label_columns=None):
        self.train_df = train_df
        self.val_df = val_df
        self.test_df = test_df
        self.label_columns = label_columns
        if label_columns is not None:
            self.label_columns_indices = {name: i for i, name in enumerate(label_columns)}

        self.column_indices = {name: i for i, name in enumerate(train_df.columns)}
        self.input_width = input_width
        self.label_width = label_width
        self.shift = shift
        self.total_window_size = input_width + shift
        
        self.input_slice = slice(0, input_width)
        self.input_indices = np.arange(self.total_window_size)[self.input_slice]
        self.label_start = self.total_window_size - self.label_width
        self.labels_slice = slice(self.label_start, None)
        self.label_indices = np.arange(self.total_window_size)[self.labels_slice]

    def __repr__(self):
        return 'n'.join([
            f'Total window size: {self.total_window_size}',
            f'Input indices: {self.input_indices}',
            f'Label indices: {self.label_indices}',
            f'Label column name(s): {self.label_columns}'])

    def split_window(self, features):
        inputs = features[:, self.input_slice, :]
        labels = features[:, self.labels_slice, :]
        if self.label_columns is not None:
            labels = tf.stack( 
                [labels[:, :, self.column_indices[name]] for name in self.label_columns], 
                axis=-1)
        
        # Slicing doesn't preserve static shape information, so set the shapes
        # manually. This way the `tf.data.Datasets` are easier to inspect.
        inputs.set_shape([None, self.input_width, None])
        labels.set_shape([None, self.label_width, None])
        
        return inputs, labels

    #WindowGenerator.split_window = split_window

    def plot(self, model=None, plot_col='Item_Cantidad', max_subplots=3):
        inputs, labels = self.example
        plt.pyplot.figure(figsize=(12, 8))
        plot_col_index = self.column_indices[plot_col]
        max_n = min(max_subplots, len(inputs))
        for n in range(max_n):
            plt.pyplot.subplot(3, 1, n+1)
            plt.pyplot.ylabel(f'{plot_col} [normed]')
            plt.pyplot.plot(self.input_indices, inputs[n, :, plot_col_index], label='Inputs', marker='.', zorder=-10)
        
            if self.label_columns:
                label_col_index = self.label_columns_indices.get(plot_col, None)
            else:
                label_col_index = plot_col_index

            if label_col_index is None:
                continue

            plt.pyplot.scatter(self.label_indices, labels[n, :, label_col_index], 
                                edgecolors='k', label='Labels', c='#2ca02c', s=64)

            if model is not None:
                predictions = model(inputs)
                plt.pyplot.scatter(self.label_indices, predictions[n, :, label_col_index], 
                                    marker='X', edgecolors='k', label='Predictions', 
                                    c='#ff7f0e', s=64)
            
            if n == 0:
                plt.pyplot.legend()
        
        plt.pyplot.xlabel('Time [D]')

    #WindowGenerator.plot = plot

    def make_dataset(self, data):
        data = np.array(data, dtype=np.float32)
        ds = tf.keras.preprocessing.timeseries_dataset_from_array( 
            data=data, 
            targets=None, 
            sequence_length=self.total_window_size, 
            sequence_stride=1, 
            shuffle=True, 
            batch_size=32,)

        ds = ds.map(self.split_window)

        return ds

    #WindowGenerator.make_dataset = make_dataset


    @property
    def train(self):
        return self.make_dataset(self.train_df)

    @property
    def val(self):
        return self.make_dataset(self.val_df)

    @property
    def test(self):
        return self.make_dataset(self.test_df)

    @property
    def example(self):
        """Get and cache an example batch of `inputs, labels` for plotting."""
        result = getattr(self, '_example', None)
        if result is None:
            # No example batch was found, so get one from the `.train` dataset
            result = next(iter(self.train))
            #And cache it for next time
            self._example = result
        return result#
    
    #WindowGenerator.train = train
    #WindowGenerator.val = val
    #WindowGenerator.test = test
    #WindowGenerator.example = example

This is the definition of multiwindow object:

OUT_STEPS = 30
multi_window = WindowGenerator(input_width=120,
                                label_width=OUT_STEPS,
                                shift=OUT_STEPS)

This is the training method:

MAX_EPOCHS = 10

def compile_and_fit(model, window, patience=2):
    
    early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss',
                                                    patience=patience,
                                                    mode='min')
    model.compile(loss=tf.losses.MeanSquaredError(),
                optimizer=tf.optimizers.Adam(),
                metrics=[tf.metrics.MeanAbsoluteError()])
    
    history = model.fit(window.train, epochs=MAX_EPOCHS,
                      validation_data=window.val,
                      callbacks=[early_stopping], verbose = 1) 

    return history

This is the model:

MAX_EPOCHS = 10 #1500
CONV_WIDTH = 10
multi_conv_model = tf.keras.Sequential([
    tf.keras.layers.Lambda(lambda x: x[:, -CONV_WIDTH:, :]),
    tf.keras.layers.Conv1D(256, activation='relu', kernel_size=(CONV_WIDTH)),
    tf.keras.layers.Dense(OUT_STEPS*num_features,
                          kernel_initializer=tf.initializers.zeros),
    tf.keras.layers.Reshape([OUT_STEPS, num_features])
])

This is the training and evalue:

history = compile_and_fit(multi_conv_model, multi_window)
   
result = multi_conv_model.evaluate(multi_window.val, verbose=-1)

print(result)

And is the output in the print result:

[]

Answer

I had the same issue when doing this tutorial (knowing what is done in the tutorial might help you understand this issue and my answer).

When you call:

result = multi_conv_model.evaluate(multi_window.val, verbose=-1)

You evaluate your model on the validation dataset (called val_df in the WindowGenerator class). Your issue happened because the size of your validation dataset is smaller than the total size of the window.

When you make your window:

OUT_STEPS = 30
multi_window = WindowGenerator(input_width=120,
                        label_width=OUT_STEPS,
                        shift=OUT_STEPS)

The size of your window is 150, so you should have a validation dataset bigger than 150. The solution to your issue is to use a bigger dataset or resize your window.