How to create Albert model?

Question

I want to create a model for hate speech detection. The idea is to have extra layers on top of an AlbertModel. My issue is in creating the Albert model itself. I was basically following this guide (https://www.kaggle.com/dhruv1234/huggingface-tfbertmodel). I tried many different methods and all are failing.

Below is my code and exceptions, lines marked with '*' is where the exceptions are thrown. Everything I tried is the code, with each method throwing a different error

If you can help me fix any of the methods tried it will be great, and if there any other way to do it that would work also.

This is the link for the full repo: https://github.com/eliesid123/NLP-HateSpeechDetection/tree/main/HateSpeechDetection

Source code:

import torch
from transformers import AlbertConfig, AlbertModel
from tensorflow.keras.optimizers import Adam
import tensorflow as tf
import numpy as np

class ModelBuilder:
    def __init__(self,
                 tokenizer,
                 dropout=0.5,
                 learningRate=12,
                 batchSize = 9,
                 epochs = 20,
                 validationSplit = 0.2,
                 inputSize = 64
                ):

        self.Dropout = dropout              
        self.LearningRate = learningRate    
        self.Epochs = epochs
        self.BatchSize = batchSize
        self.InputSize = inputSize
        self.ValidationSplit = validationSplit
        self.AlbertModel = AlbertModel.from_pretrained('albert-xxlarge-v2')
        self.AlbertConfig = AlbertConfig()
        self.Tokenizer = tokenizer
        self.IsCompiled = False
        self.Model = None
        self.ModelSummary = None

def Create(self, PRINT = False):
    #INITIAL KAGGLE GUIDE
    # input_ids = tf.keras.Input(shape=(self.InputSize,),dtype='int64')
    # attention_masks = tf.keras.Input(shape=(self.InputSize,),dtype='float')
    *# output, layer = self.AlbertModel(input_ids,attention_masks)*

    #THIS IS BASED ON HUGGINGFACE DOCUMENTATION
    # tensor = np.ndarray(shape=(self.InputSize,),dtype='long')
    # input_ids = torch.tensor(tensor)
    # tensor = np.ndarray(shape=(self.InputSize,),dtype='float')
    # attention_masks = torch.tensor(tensor)
    *# output = self.AlbertModel(input_ids,attention_masks)*
    
    #THIS IS FROM HUGGINGFACE DOCUMENTATION
    # sampleText = "some line............................................."
    # encoded = self.Tokenizer.EncodeSentece(sampleText)
    # input_ids = encoded['input_ids']
    # attention_masks = encoded['attention_mask']
    *# output = self.AlbertModel(torch.tensor(encoded).unsqueeze(self.BatchSize))*

    output = self.AlbertModel(self.AlbertConfig)

    output = output[1]
    output = tf.keras.layers.Dense(128,activation='relu')(output)
    output = tf.keras.layers.Dropout(self.Dropout)(output)
    output = tf.keras.layers.Dense(64,activation='relu')(output)
    output = tf.keras.layers.Dropout(self.Dropout)(output)
    output = tf.keras.layers.Dense(32,activation='relu')(output)
    output = tf.keras.layers.Dropout(self.Dropout)(output)
    output = tf.keras.layers.Dense(1,activation='sigmoid')(output)

    model = tf.keras.models.Model(inputs = [input_ids, attention_masks] ,outputs = output)

    self.Model = model
    self.ModelSummary = model.summary()
    if PRINT:
        print(self.ModelSummary)

Exceptions:

Using tf.Keras.Input
Exception has occurred: AttributeError
    • 
'KerasTensor' object has no attribute 'size'
File "/home/elie/Desktop/NLP/HateSpeechDetection/src/modelBuilder.py", line 44, in Create output, layer = self.AlbertModel(input_ids,attention_masks) File "/home/elie/Desktop/NLP/HateSpeechDetection/src/main.py", line 16, in main model.Create(PRINT=True) File "/home/elie/Desktop/NLP/HateSpeechDetection/src/main.py", line 21, in <module> main()


Using torch.tensor

Exception has occurred: ValueError
    • 
not enough values to unpack (expected 2, got 1)
File "/home/elie/Desktop/NLP/HateSpeechDetection/src/modelBuilder.py", line 50, in Create output = self.AlbertModel(input_ids,attention_masks) File "/home/elie/Desktop/NLP/HateSpeechDetection/src/main.py", line 16, in main model.Create(PRINT=True) File "/home/elie/Desktop/NLP/HateSpeechDetection/src/main.py", line 21, in <module> main()

Using AlbertTokenizer.encode_plus()

Exception has occurred: ValueError
    • 
could not determine the shape of object type 'BatchEncoding'
File "/home/elie/Desktop/NLP/HateSpeechDetection/src/modelBuilder.py", line 54, in Create output = self.AlbertModel(torch.tensor(encoded).unsqueeze(self.BatchSize)) File "/home/elie/Desktop/NLP/HateSpeechDetection/src/main.py", line 16, in main model.Create(PRINT=True) File "/home/elie/Desktop/NLP/HateSpeechDetection/src/main.py", line 21, in <module> main()

Using AlbertConfig

Exception has occurred: AttributeError
    • 
'AlbertConfig' object has no attribute 'size'
File "/home/elie/Desktop/NLP/HateSpeechDetection/src/modelBuilder.py", line 59, in Create output = self.AlbertModel(self.AlbertConfig) File "/home/elie/Desktop/NLP/HateSpeechDetection/src/main.py", line 16, in main model.Create(PRINT=True) File "/home/elie/Desktop/NLP/HateSpeechDetection/src/main.py", line 21, in <module> main()

Tokenizer class

import numpy as np
from transformers import AlbertTokenizer

class Tokenizor():
    def __init__(self,maxLen=64) -> None:
        self.Tokenizer = AlbertTokenizer.from_pretrained('albert-xxlarge-v2')
        self.MaxLen = maxLen

    def EncodeAll(self,data) :
        input_ids = []
        attention_masks = []

        for i in range(len(data)):
            encoded = self.EncodeSentece(data[i])
            input_ids.append(encoded['input_ids'])
            attention_masks.append(encoded['attention_mask'])
        return np.array(input_ids),np.array(attention_masks)

    def EncodeSentece(self,line):
        encoded = self.Tokenizer.encode_plus(
              line,
              add_special_tokens=True,
              max_length=self.MaxLen,
              pad_to_max_length=True,
              return_attention_mask=True,
            )
        return encoded