Python Socket UDP high latency

Question

Essentially I have made a simple voice chat using UDP socket. When two people are using it, its pretty normal barely any latency. When a third joins in it start lagging like crazy. I'm not sure on how to solve this I have tried to thread the loops and everything.

Here is the server side. The data sent from the client doesnt go above 1024 bytes but its a constant stream of data.

The actual voice chat is here https://github.com/e1pupper/VoiceChat which is by me

class serverHandler(threading.Thread):
    def __init__(self, serverObj, d, a):
        threading.Thread.__init__(self)
        for c in serverObj.connections:
            if a != c:
                serverObj.udp_server.sendto(d, c)
        
class connectionHandler(threading.Thread):
    def __init__(self, serverObj, timeout):
        threading.Thread.__init__(self)
        for addr in serverObj.clients.copy().keys():
            if serverObj.clients[addr] < (time.time() - timeout):
                serverObj.connections.remove(addr)
                serverObj.clients.pop(addr)
                print("Removed!")
        
class Server:
    udp_server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    
    #Use 0.0.0.0 after you have port forwarded
    host = "0.0.0.0" 
    #host = "127.0.0.1"
    port = 55800
    privatekey = b'Hq9vW2opxUZ7ld51Inrz0rvnMhlHtukUnFKrIAyxyOE='
    publickey = b'4vF083_jOpvEdqbXqem8GP96wmawb0KKZLz3o43o-KU='
    pucryp = Fernet(publickey)
    prcryp = Fernet(privatekey)
    
    curclients = []

    version = "0.1"
    
    def __init__(self):
        server_config = (self.host, self.port)
        self.udp_server.bind(server_config)
        self.connections = []
        
        print(f"Server laucnhed {self.host}:{self.port}")
        
        try:
            # Use a get request for api.duckduckgo.com
            raw = requests.get('https://api.duckduckgo.com/?q=ip&format=json')
            # load the request as json, look for Answer.
            # split on spaces, find the 5th index ( as it starts at 0 ), which is the IP address
            answer = raw.json()["Answer"].split()[4]
        # if there are any connection issues, error out
        except Exception as e:
            print('Error: {0}'.format(e))
        # otherwise, return answer
        else:
            print(answer)
    
        self.acceptConnectionT = threading.Thread(target=self.acceptConnection)
        self.acceptConnectionT.daemon = True
        self.acceptConnectionT.start()
        
    def acceptConnection(self):
        self.clients = dict()
        timeout = 1
        while True:
            try:
                d, a = self.udp_server.recvfrom(4096)
                self.clients[a] = time.time()
                connectionHandler(self,timeout)
                if a not in self.connections:
                    if d == b"connection":
                        self.udp_server.sendto(d,a)
                        self.connections.append(a)
                        print(self.connections)
                        print("Get connecting from ", a)
                else:
                    serverHandler(self,d,a)
                    #for c in self.connections:
                    #    if a != c:
                    #        self.udp_server.sendto(d, c)
                #self.udp_server.timeout(0.050)
            except Exception as ex:
                pass