send multiple packets of data using python socket programming

Question

So now I have a server and client script. I'm trying to upload a file from the client to the server. However, the data from the file in the client will be cut out by the HEADER size. How do I send multiple packets under the same send command to the server?

server.py:

import socket
import threading
HEADER=2048
PORT=5050
SERVER=socket.gethostbyname(socket.gethostname())
ADDR=(SERVER,PORT)
FORMAT='utf-8'
DISCONNECT_MESSAGE='!DISCONNECT'
SEPARATOR='<SEPERATE>'
server=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
server.bind(ADDR)
def handle_client(conn,addr):
    print(f'[NEW CONNECTION] {addr} connected.')
    connected=True
    while connected:
        data=conn.recv(HEADER).decode(FORMAT)
        if data==DISCONNECT_MESSAGE:
            connected=False
        else:
            data=data.split(SEPARATOR)
            file=open(data[0],'w')
            file.write(data[1])
            print('file received')
            conn.send('file received'.encode(FORMAT))         
    conn.close()
    print(f'[DISCONNECT] {addr} disconnected')

def start():
    server.listen()
    print(f'[LISTENING] Server is listening on {SERVER}')
    while True:
        conn,addr=server.accept()
        thread=threading.Thread(target=handle_client,args=(conn,addr))
        thread.start()
        print(f'[ACTIVE CONNECTIONS] {threading.activeCount()-1}')
print("[STARTING] server is starting...")
start()

client.py:

import socket
HEADER=2048
PORT=5050
FORMAT='utf-8'
DISCONNECT_MESSAGE='!DISCONNECT'
SEPARATOR='<SEPERATE>'
SERVER=socket.gethostbyname(socket.gethostname())
ADDR=(SERVER,PORT)
client=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
client.connect(ADDR)
def send(msg):
    message=msg.encode(FORMAT)
    client.send(message)
    print(client.recv(HEADER).decode(FORMAT))
file=open('question_pool.csv','r')
data=file.read()
send(f'question_pool.csv{SEPARATOR}{data}')
file.close()
send(DISCONNECT_MESSAGE)

Answer 1

In short, you want to send multiple chunks of any file that is larger than your HEADER size. Split the file into chunks smaller than the HEADER size, and send each chunk individually. Then when all the chunks are set, send a message that says the whole file has been sent so that the server can save it.

Here is my code for the solution described above:
server.py:

import socket
import threading


HEADER = 2048
PORT = 5050
SERVER = socket.gethostbyname(socket.gethostname())
ADDR = (SERVER, PORT)
FORMAT = 'utf-8'
DISCONNECT_MESSAGE = '!DISCONNECT'
SEPARATOR = '<SEPERATE>'
FILE_FINISHED_SENDING = '<!FILE_SENT!>'

server = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
server.bind(ADDR)


def handle_client(conn, addr):
    print(f'[NEW CONNECTION] {addr} connected.')
    connected = True
    current_file = None

    while connected:
        data = conn.recv(HEADER).decode(FORMAT)
        if data == DISCONNECT_MESSAGE:
            connected = False
        elif data == FILE_FINISHED_SENDING:
            current_file.close()
            current_file = None
            conn.send(b'file received.')
        else:
            data = data.split(SEPARATOR)
            if len(data) == 2 and data[1] == '':
                # The name of the file was sent, more will follow.
                current_file = open(data[0], 'w')
                conn.send(b'filename recieved')
            else:
                # File data was send, so write it to the current file
                current_file.write(data[1])
                print('chunk of file recv''d')
                conn.send(b'chunk received')
    conn.close()
    print(f'[DISCONNECT] {addr} disconnected')



def start():
    server.listen()
    print(f'[LISTENING] Server is listening on {SERVER}')
    while True:
        conn, addr = server.accept()
        thread = threading.Thread(target=handle_client, args=(conn,addr))
        thread.start()
        print(f'[ACTIVE CONNECTIONS] {threading.activeCount()-1}')


print("[STARTING] server is starting...")
start()

client.py:

import socket
from pathlib import Path

HEADER = 2048
PORT = 5050
FORMAT = 'utf-8'
DISCONNECT_MESSAGE = '!DISCONNECT'
SEPARATOR = '<SEPERATE>'
FILE_FINISHED_SENDING = '<!FILE_SENT!>'
SERVER = socket.gethostbyname(socket.gethostname())
ADDR = (SERVER, PORT)

client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect(ADDR)


def chunker(string: str, size: int):
    return (string[pos:pos + size] for pos in range(0, len(string), size))


def send(msg):
    message = msg.encode(FORMAT)
    client.send(message)
    print(client.recv(HEADER).decode(FORMAT))


def send_file(filepath: str):
    with open(filepath, 'r', encoding=FORMAT) as f:
        data = f.read()

    first_bits = f'{Path(filepath).name}{SEPARATOR}'  # Easy way of getting just a file's name from its path
    send(first_bits)  # Send the filename to the server
    for chunk in chunker(data, HEADER-48):  # Leave a safe buffer
        # Send each chunk of the file
        send(f"{SEPARATOR}{chunk}")

    # Tell the server that's all for this file.
    # Now it can close the file object.
    send(FILE_FINISHED_SENDING)


send_file("/path/to/file.html")
send(DISCONNECT_MESSAGE)

Tips:

• make sure that your special messages like SEPARATOR, FILE_FINISHED_SENDING and DISCONNECT_MESSAGE are NOT going to appear in the files you are sending. Otherwise things might get wonky.
• You may want to read files as raw bytes when you send them through the socket, instead of reading as strings, encoding, decoding, etc. This way you could send binary files such as .mp3, for example.