Python: Apply function to values in nested dictionary

Question

I have an arbitrarily deep set of nested dictionary:

x = {'a': 1, 'b': {'c': 6, 'd': 7, 'g': {'h': 3, 'i': 9}}, 'e': {'f': 3}}

and I'd like to basically apply a function to all the integers in the dictionaries, so like map, I guess, but for nested dictionaries.

So: map_nested_dicts(x, lambda v: v + 7) would be the sort of goal.

I'm stuck as to the best way to perhaps store the layers of keys to then put the modified value back into its correct position.

What would the best way/approach to do this be?

Answer 1

Visit all nested values recursively:

import collections

def map_nested_dicts(ob, func):
    if isinstance(ob, collections.Mapping):
        return {k: map_nested_dicts(v, func) for k, v in ob.iteritems()}
    else:
        return func(ob)

map_nested_dicts(x, lambda v: v + 7)
# Creates a new dict object:
#    {'a': 8, 'b': {'c': 13, 'g': {'h': 10, 'i': 16}, 'd': 14}, 'e': {'f': 10}}

In some cases it's desired to modify the original dict object (to avoid re-creating it):

import collections

def map_nested_dicts_modify(ob, func):
    for k, v in ob.iteritems():
        if isinstance(v, collections.Mapping):
            map_nested_dicts_modify(v, func)
        else:
            ob[k] = func(v)

map_nested_dicts_modify(x, lambda v: v + 7)
# x is now
#    {'a': 8, 'b': {'c': 13, 'g': {'h': 10, 'i': 16}, 'd': 14}, 'e': {'f': 10}}

If you're using Python 3:

• replace dict.iteritems with dict.items

• replace import collections with import collections.abc

• replace collections.Mapping with collections.abc.Mapping

Answer 2

Just to expand on vaultah's answer, if one of your elements can be a list, and you'd like to handle those too:

import collections

def map_nested_dicts_modify(ob, func):
for k, v in ob.iteritems():
    if isinstance(v, collections.Mapping):
        map_nested_dicts_modify(v, func)
    elif isinstance(v, list):
        ob[k] = map(func, v)
    else:
        ob[k] = func(v)

Answer 3

If you need it to work for both lists and dicts in arbitrary nesting:

def apply_recursive(func, obj):
    if isinstance(obj, dict):  # if dict, apply to each key
        return {k: apply_recursive(func, v) for k, v in obj.items()}
    elif isinstance(obj, list):  # if list, apply to each element
        return [apply_recursive(func, elem) for elem in obj]
    else:
        return func(obj)

Answer 4

If you want to avoid dependencies and you need to map a mixed dictionaries/iterables collection with any combination of nesting and deepness, you can use the following solution:

def map_nested_coll(func,obj):
    if '__iter__' in dir(obj) and type(obj) not in (str,bytes):
        if type(obj) == dict:
            return {k:map_nested_coll(func,v) for k,v in obj.items()}
        else:
            return tuple(map_nested_coll(func,x) for x in obj)
    else:
        return func(obj)

In order to retain simplicity, non-dict iterables are converted to tuples (you can convert to list instead of tuple if you like, but converting to tuples is slightly faster). Also, although strings and bytes are iterables, usually you want to apply func on the whole string or bytes, so they are filtered out and not treated like iterables.

The advantage of this solution is that it works with any kind of iterable (even generators like zip, range and map) and handles edge cases well (see below):

>>> func = lambda x: x/2
>>> map_nested_coll(func, dict(a=1,b=dict(c=2,d=[3,(41,42),5]),e=[6,7]))
{'a': 0.5, 'b': {'c': 1.0, 'd': (1.5, (20.5, 21.0), 2.5)}, 'e': (3.0, 3.5)}
>>> map_nested_coll(func, [1,dict(a=2,b=3),(4,5)])
(0.5, {'a': 1.0, 'b': 1.5}, (2.0, 2.5))
>>> map_nested_itr(func, map(lambda x: 1+x, range(3)))
(0.5, 1.0, 1.5)
>>> map_nested_coll(func, 9)
4.5
>>> map_nested_coll(func, [])
()
>>> map_nested_itr(func, dict())
{}

Answer 5

I have a more general implementation that can accept any number of containers of any type as parameters.

from collections.abc import Iterable
import types
def dict_value_map(fun, *dicts):
    keys = dicts[0].keys()
    for d in dicts[1:]:
        assert d.keys() == keys
    return {k:fun(*(d[k] for d in dicts)) for k in keys}
def collection_map(fun, *collections):
    assert len(collections) > 0
    if isinstance(collections[0], dict):
        return dict_value_map(fun, *collections)
    if isinstance(collections[0], (tuple, list, set)):
        return type(collections[0])(map(fun, *collections))
    else:
        return map(fun, *collections)
iscollection = lambda v:(isinstance(v,Iterable)and(not isinstance(v,str)))

def apply(fun, *collections, at=lambda collections: not iscollection(collections[0])):
    '''
    like standard map, but can apply the fun to inner elements.
    at: a int, a function or sometype. 
    at = 0 means fun(*collections)
    at = somefunction. fun will applied to the elements when somefunction(elements) is True
    at = sometype. fun will applied to the elements when elements are sometype.
    '''
    if isinstance(at, int):
        assert at >= 0
        if at == 0:
            return fun(*collections)
        else:
            return collection_map(lambda *cs:apply(fun, *cs, at=at-1), *collections)
    if isinstance(at, types.FunctionType):
        if at(collections):
            return fun(*collections)
        else:
            return collection_map(lambda *cs:apply(fun, *cs, at=at), *collections)
    else:
        return apply(fun, *collections, at=lambda eles:isinstance(eles[0], at))

examples:

> apply(lambda x:2*x, [(1,2),(3,4)])  
[(2, 4), (6, 8)]

> apply(lambda a,b: a+b, ([1,2],[3,4]), ([5,6],[7,8]))
([6, 8], [10, 12])

> apply(lambda a,b: a+b, ([1,2],[3,4]), ([5,6],[7,8]), at=1)  
([1, 2, 5, 6], [3, 4, 7, 8])

> apply(lambda a,b: a+b, ([1,2],[3,4]), ([5,6],[7,8]), at=0)  
([1, 2], [3, 4], [5, 6], [7, 8])

> apply(lambda a,b:a+b, {'m':[(1,2),[3,{4}]], 'n':5}, {'m':[(6,7),[8,{9}]],'n':10})  
{'m': [(7, 9), [11, {13}]], 'n': 15}

> apply(str.upper, [('a','b'),('c','d')], at=str)  
[('A', 'B'), ('C', 'D')]

and

> apply(lambda v:v+7, {'a': 1, 'b': {'c': 6, 'd': 7, 'g': {'h': 3, 'i': 9}}, 'e': {'f': 3}})
{'a': 8, 'b': {'c': 13, 'd': 14, 'g': {'h': 10, 'i': 16}}, 'e': {'f': 10}}