apply LSTM on BERT embedding

Question

I use a code to classify toxic tweet I want to modify the following code to used LSTM architecture the code fed BERT embedding to LSTM Layers

class BertClassifier(nn.Module):
    def __init__(self, freeze_bert=False):

        super(BertClassifier, self).__init__()
        # Specify hidden size of BERT, hidden size of our classifier, and number of labels
        D_in, H, D_out = 768, 50, 2

        # Instantiate BERT model
        self.bert = BertModel.from_pretrained('aubmindlab/bert-base-arabertv02-twitter')

        # Instantiate an one-layer feed-forward classifier
        self.classifier = nn.Sequential(
            nn.Linear(D_in, H),
            nn.ReLU(),
            #nn.Dropout(0.5),
            nn.Linear(H, D_out)
        )

        if freeze_bert:
            for param in self.bert.parameters():
                param.requires_grad = False
        

when I use LSTM an error appears that forward function must be modified

def forward(self, input_ids, attention_mask):
        @return   logits (torch.Tensor): an output tensor with shape (batch_size,num_labels)
        
    
        outputs = self.bert(input_ids=input_ids,
                            attention_mask=attention_mask)
        last_hidden_state_cls = outputs[0][:, 0, :]
        logits = self.classifier(last_hidden_state_cls)
        return logits