Send and receive objects through sockets in Python

Question

I have searched a lot on the Internet, but I haven't been able to find the solution to send an object over the socket and receive it as is. I know it needs pickling which I have already done. And that converts it to bytes and is received on the other hand. But how can I convert those bytes to that type of object?

process_time_data = (current_process_start_time, current_process_end_time)
prepared_process_data = self.prepare_data_to_send(process_time_data)
data_string = io.StringIO(prepared_process_data)
data_string =  pack('>I', len(data_string)) + data_string
self.send_to_server(data_string)

This is the code which is converting the object to StringIO on the client and sending to the server. And on the server side I am getting bytes. Now I am searching for bytes to be converted to StringIO again so that I can get the object value.

In the code, Object is wrapped in StringIO and is being sent over the socket. Is there a better approach?

The server-side code is as follows.

server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
#server.setblocking(0)
server.bind(('127.0.0.1', 50000))
server.listen(5)
inputs = [server]
outputs = []
message_queues = {}

while inputs:
    readable, writeable, exceptional = select.select(inputs, outputs, inputs)
    for s in readable:
        if s is server:
            connection, client_address = s.accept()
            print(client_address)
            connection.setblocking(0)
            inputs.append(connection)
            message_queues[connection] = queue.Queue()
            print('server started...')
        else:
            print('Getting data step 1')
            raw_msglen = s.recv(4)
            msglen = unpack('>I', raw_msglen)[0]
            final_data = b''
            while len(final_data) < msglen:
                data = s.recv(msglen - len(final_data))
                if data:
                    #print(data)
                    final_data += data
                    message_queues[s].put(data)
                    if s not in outputs:
                        outputs.append(s)
                    else:
                        if s in outputs:
                            outputs.remove(s)
                else:
                    break
            inputs.remove(connection)
            #s.close()
            del message_queues[s]

            process_data = ProcessData()
            process_screen = ProcessScreen()

            if final_data is not None:
                try:
                    deserialized_data = final_data.decode("utf-8")
                    print(deserialized_data)
                except (EOFError):
                    break
            else:
                print('final data is empty.')

            print(process_data.project_id)
            print(process_data.start_time)
            print(process_data.end_time)
            print(process_data.process_id)

The two helper functions are as follows:

def receive_all(server, message_length, message_queues, inputs, outputs):
    # Helper function to recv message_length bytes or return None if EOF is hit
    data = b''
    while len(data) < message_length:
        packet = server.recv(message_length - len(data))
        if not packet:
            return None
        data += packet
        message_queues[server].put(data)
        if server not in outputs:
            outputs.append(server)
        else:
            if server in outputs:
                outputs.remove(server)
    inputs.remove(server)
    del message_queues[server]
    return data


def receive_message(server, message_queues, inputs, outputs):
    # Read message length and unpack it into an integer
    raw_msglen = receive_all(server, 4, message_queues, inputs, outputs)
    if not raw_msglen:
        return None
    message_length = unpack('>I', raw_msglen)[0]

    return receive_all(server, message_length, message_queues, inputs, outputs)

And two of the model classes are as follows:

class ProcessData:
    process_id = 0
    project_id = 0
    task_id = 0
    start_time = 0
    end_time = 0
    user_id = 0
    weekend_id = 0

# Model class to send image data to the server
class ProcessScreen:
    process_id = 0
    image_data = bytearray()

Answer 1

You're looking for pickle and the loads and dumps operations. Sockets are basically byte streams. Let us consider the case you have.

class ProcessData:
    process_id = 0
    project_id = 0
    task_id = 0
    start_time = 0
    end_time = 0
    user_id = 0
    weekend_id = 0

An instance of this class needs to be pickled into a data string by doing data_string = pickle.dumps(ProcessData()) and unpickled by doing data_variable = pickle.loads(data) where data is what is received.

So let us consider a case where the client creates an object of ProcessData and sends it to server. Here's what the client would look like. Here's a minimal example.

Client

import socket, pickle

class ProcessData:
    process_id = 0
    project_id = 0
    task_id = 0
    start_time = 0
    end_time = 0
    user_id = 0
    weekend_id = 0


HOST = 'localhost'
PORT = 50007
# Create a socket connection.
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))

# Create an instance of ProcessData() to send to server.
variable = ProcessData()
# Pickle the object and send it to the server
data_string = pickle.dumps(variable)
s.send(data_string)

s.close()
print 'Data Sent to Server'

Now your server which receives this data looks as follows.

Server

import socket, pickle

class ProcessData:
    process_id = 0
    project_id = 0
    task_id = 0
    start_time = 0
    end_time = 0
    user_id = 0
    weekend_id = 0


HOST = 'localhost'
PORT = 50007
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(1)
conn, addr = s.accept()
print 'Connected by', addr

data = conn.recv(4096)
data_variable = pickle.loads(data)
conn.close()
print data_variable
# Access the information by doing data_variable.process_id or data_variable.task_id etc..,
print 'Data received from client'

Running the server first creates a bind on the port and then running the client makes the data transfer via the socket. You could also look at this answer.

Answer 2

An option is to use JSON serialization.

However, Python objects are not serializable, so you have to map your class object into Dict first, using either function vars (preferred) or the built-in __dict__.

Adapting the answer from Sudheesh Singanamalla and based on this answer:

Client

import socket, json

class ProcessData:
    process_id = 0
    project_id = 0
    task_id = 0
    start_time = 0
    end_time = 0
    user_id = 0
    weekend_id = 0


HOST = 'localhost'
PORT = 50007
# Create a socket connection.
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))

# Create an instance of ProcessData() to send to server.
variable = ProcessData()

# Map your object into dict
data_as_dict = vars(variable)

# Serialize your dict object
data_string = json.dumps(data_as_dict)

# Send this encoded object
s.send(data_string.encode(encoding="utf-8"))

s.close()
print 'Data Sent to Server'

Server

import socket, json

class ProcessData:
    process_id = 0
    project_id = 0
    task_id = 0
    start_time = 0
    end_time = 0
    user_id = 0
    weekend_id = 0


HOST = 'localhost'
PORT = 50007
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(1)
conn, addr = s.accept()
print 'Connected by', addr

data_encoded = conn.recv(4096)
data_string = data_encoded.decode(encoding="utf-8")

data_variable = json.loads(data_string)
# data_variable is a dict representing your sent object

conn.close()
print 'Data received from client'

Warning

One important point is that dict mapping of an object instance does not map class variable, only instance variable. See this answer for more information. Example:

class ProcessData:
    # class variables
    process_id = 0
    project_id = 1

    def __init__(self):
        # instance variables
        self.task_id = 2
        self.start_time = 3

obj = ProcessData()
dict_obj = vars(obj)

print(dict_obj)
# outputs: {'task_id': 2, 'start_time': 3}

# To access class variables:
dict_class_variables = vars(ProcessData)

print(dict_class_variables['process_id'])
# outputs: 0

Answer 3

Pickle is not particularly safe for network communications, because it can be used to inject executable code. I suggest you try json instead.

Pseudocode:

import json
to_send = json.dumps(object)
s.sendall(to_send)