CNN seemingly not even trying to learn

Question

I'm trying to train a CNN, but it seems to output the same thing regardless of input, except between training steps, when its output is altered slightly with each training step. I've tried changing the size of the network, the learning rate, the loss function etc. etc. Regardless of what I do, my training graph is always some variation of this:

The data in question is Stock market data, so it's possible it's either beyond the scope of my network or beyond the scope of CNNs in general. However, I am using the same structure as is used in this paper, which achieves solid results, so I should be getting at least some improvement in accuracy.

I don't really know what I've done wrong, and any tips or directions are much appreciated.

Code for the Model:

class ClassifierModel(nn.Module):
def __init__(
    self,
    input_shape, 
    output_shape,
    path=None,
    class_frequencies=None,
    lr=1e-3,
):
    super(ClassifierModel, self).__init__()
    self.path = path

    self.dropout = nn.Dropout(p=0.2)
    self.conv1 = nn.Conv2d(1, 4, 3, padding=1)
    self.maxpool1 = nn.MaxPool2d(2, padding=1, stride=1)
    self.bn1 = nn.BatchNorm2d(4)
    self.conv2 = nn.Conv2d(4, 8, 3, padding=1)
    self.maxpool2 = nn.MaxPool2d(2, padding=1, stride=1)
    self.bn2 = nn.BatchNorm2d(8)
    self.conv3 = nn.Conv2d(8, 8, 3, padding=1)
    self.maxpool3 = nn.MaxPool2d(2, padding=1, stride=1)
    self.bn3 = nn.BatchNorm2d(8)
    self.input_dims = self._calc_input_dims(input_shape)
    self.fc1 = nn.Linear(self.input_dims, 64)
    self.fc2 = nn.Linear(64, 64)
    self.fc3 = nn.Linear(64, output_shape[0])

    # Optimizer etc.
    self.device = T.device("cuda:0" if T.cuda.is_available() else "cpu")
    self.optimizer = optim.Adam(self.parameters(), lr=lr)
    self.loss = nn.MSELoss()
    self.to(self.device)

def _calc_input_dims(self, input_shape):
    """Calculate the input dimensions of the first fc layer"""
    x = T.zeros((1, 1) + input_shape)
    x = self.conv1(x)
    x = self.maxpool1(x)
    x = self.conv2(x)
    x = self.maxpool2(x)
    x = self.conv3(x)
    x = self.maxpool3(x)

    return int(np.prod(x.size()))

def train_(
    self,
    train_data,   
    val_data=None,
    epochs=1,
    graph_path=None,
    progress=False,
):
    """Train the model on the provided data"""
    self.train()
    acc_hist = []
    loss_hist = []
    val_acc_hist = []
    val_loss_hist = []
    for epoch in range(1, epochs + 1):
        # Training loop                                                                                                                                                                                                                                                                           
        acc = 0                                                                                                                                                                                                                                                                                   
        loss = 0                                                                                                                                                                                                                                                                                  
        for X, y in train_data:                                                                                                                                                                                                                                                                   
            l, a = self._training_step(X, y)
            loss += l                                                                                                                                                                                                                                                                      
            acc += a                                                                                                                                                                                                                                                                              
        loss /= len(train_data)                                                                                                                                                                                                                                                                   
        acc /= len(train_data)                                                                                                                                                                                                                                                                    
        loss_hist.append(loss)                                                                                                                                                                                                                                                                    
        acc_hist.append(acc)                                                                                                                                                                                                                                                                      
                                                                                                                                                                                                                                                                                                  
        # Validation loop                                                                                                                                                                                                                                                                         
        v_acc = 0                                                                                                                                                                                                                                                                                 
        v_loss = 0                                                                                                                                                                                                                                                                                
        if val_data:                                                                                                                                                                                                                                                                              
            for X, y in val_data:                                                                                                                                                                                                                                                                 
                l, a = self._validation_step(X, y)                                                                                                                                                                                                                                                
                v_loss += l                                                                                                                                                                                                                                                                       
                v_acc += a                                                                                                                                                                                                                                                                        
            v_loss /= len(val_data)                                                                                                                                                                                                                                                               
            v_acc /= len(val_data)                                                                                                                                                                                                                                                                
            val_loss_hist.append(v_loss)                                                                                                                                                                                                                                                          
            val_acc_hist.append(v_acc)                                                                                                                                                                                                                                                            
                                                                                                                                                                                                                                                                                                  
        if progress:                                                                                                                                                                                                                                                                              
            print(f"Epoch {epoch}: acc: {acc}, loss: {loss}")                                                                                                                                                                                                                                     
            if val_data:                                                                                                                                                                                                                                                                          
                print(f"\tval_acc: {v_acc}, val_loss: {v_loss}")
    if graph_path:
        mu.graph_training(
            {
                "acc": acc_hist,
                "loss": loss_hist,
                "val_acc": val_acc_hist,
                "val_loss": val_loss_hist,
            },
            graph_path,
        )
    
    return acc_hist, loss_hist

def _training_step(self, X, y):
    # TODO: This bit should be handled by data formatting
    # print(X[0][0], y)
    X = T.tensor([[X]], dtype=T.float, device=self.device)
    y = T.tensor([y], dtype=T.float, device=self.device)
    
    self.optimizer.zero_grad()
    y_ = self.forward(X)
    loss_ = self.loss(y_, y)
    loss_.backward()
    self.optimizer.step()
    
    loss = loss_.item()
    acc = int(list(y[0]).index(max(y[0])) == list(y_[0]).index(max(y_[0])))
    return loss, acc
    
def _validation_step(self, X, y):
    # TODO: This bit should be handled by data formatting
    X = T.tensor([[X]], dtype=T.float, device=self.device)
    y = T.tensor([y], dtype=T.float, device=self.device)
    
    y_ = self.forward(X)
    loss_ = self.loss(y_, y)
    
    loss = loss_.item()
    acc = int(list(y[0]).index(max(y[0])) == list(y_[0]).index(max(y_[0])))
    return loss, acc
    
def forward(self, x):
    # Make sure input is a tensor, and push it to gpu if possible
    if type(x) != T.Tensor:
        x = T.tensor(x, dtype=T.float, device=self.device)
    else:
        x.to(self.device)
    x = self.conv1(x)
    # x = F.relu(x)
    x = self.maxpool1(x)
    x = self.dropout(x)
    x = self.bn1(x)
    x = self.conv2(x)
    x = self.maxpool2(x)
    x = self.dropout(x)
    x = self.bn2(x)
    x = self.conv3(x)
    x = self.maxpool3(x)
    x = self.dropout(x)
    x = self.bn3(x)

    x = x.view(x.size()[0], -1)
    x = self.fc1(x)
    x = self.fc2(x)
    x = self.fc3(x)

    return x