Utils

Utility functions

Disk Cache

this doesn’t work completely because the decorator is also included in the source..

a.__code__.co_code
b'd\x01S\x00'
hashlib.md5(a.__code__.co_code).hexdigest()
'9cf8bd1d73fd245ea1325fbc4055d3d4'

source

cache_disk

 cache_disk (base_file, rm_cache=False, verbose=False)

Decorator to cache function output to disk

import time
from tempfile import tempdir
cp = Path(tempdir) / "test_cache"
@cache_disk(cp)
def slow_add(a,b):
    time.sleep(1)
    return a + b

this time is the first time so not from the cache

CPU times: user 2.98 ms, sys: 122 µs, total: 3.1 ms
Wall time: 1 s
3

now is much faster beacuse of the cache

CPU times: user 4 µs, sys: 0 ns, total: 4 µs
Wall time: 7.15 µs
3

adding comments change the hash, so the function is still cached

@cache_disk(cp)
def slow_add(a,b):
    time.sleep(1)
    # this is a comment
    return a + b
CPU times: user 1.49 ms, sys: 192 µs, total: 1.68 ms
Wall time: 1 s
3
CPU times: user 8 µs, sys: 1 µs, total: 9 µs
Wall time: 12.9 µs
3

Random

source

reset_seed

 reset_seed (seed=27)

Testing

source

test_close

 test_close (a, b, eps=1e-05)

test that a is within eps of b

Standard Scaler

make a standard scaler that can also inverse transfor standard deviations. see Standardizer for details of implementation

reset_seed()
xx = np.random.random((4, 10))
s = StandardScaler().fit(xx)
s.transform(xx)
array([[ 0.07263978,  0.63279488, -0.9975139 ,  0.50899177,  0.15537652,
         1.45555506,  1.56629646, -1.60237369,  1.51674974,  1.29584745],
       [ 1.58579521,  0.83086419, -0.68281902,  0.51578245, -0.62395756,
        -1.19720248, -0.43000476,  1.1539719 , -0.74724819, -0.85525414],
       [-1.05809926, -1.69049694,  0.0895118 , -1.72684476, -1.08418417,
         0.32617669, -1.16657374,  0.2345773 ,  0.26525847,  0.64349108],
       [-0.60033573,  0.22683787,  1.59082112,  0.70207053,  1.55276521,
        -0.58452927,  0.03028204,  0.21382449, -1.03476002, -1.08408439]])
s.mean_
array([0.40358703, 0.6758362 , 0.77934606, 0.70748673, 0.34417949,
       0.62067044, 0.48500116, 0.54921643, 0.34604713, 0.3660338 ])
s.scale_
array([0.30471427, 0.21926148, 0.04405831, 0.31536161, 0.25229864,
       0.24649441, 0.26061043, 0.21187396, 0.26093989, 0.22927816])

source

StandardScaler.inverse_transform_std

 StandardScaler.inverse_transform_std (x_std)
Details
x_std standard deviations

Info visualization

source

array2df

 array2df (x:torch.Tensor, row_names:Optional[Collection[str]]=None,
           col_names:Optional[Collection[str]]=None, row_var:str='')
Type Default Details
x Tensor 2d tensor
row_names Optional None names for the row
col_names Optional None names for the columns
row_var str name of the first column (the one with row names). This should describe the values of row_name 
import numpy as np
a = np.random.rand(2,3,3)
display(HTML(f"<pre> {repr(a)} </pre>"))
 array([[[0.96646567, 0.58332229, 0.09242191],
        [0.0136295 , 0.83693011, 0.9147879 ],
        [0.70458626, 0.3870066 , 0.7056939 ]],

       [[0.92331116, 0.28815289, 0.68401985],
        [0.5202925 , 0.87736578, 0.92388931],
        [0.48923016, 0.59621396, 0.26427542]]]) 
a
array([[[0.96646567, 0.58332229, 0.09242191],
        [0.0136295 , 0.83693011, 0.9147879 ],
        [0.70458626, 0.3870066 , 0.7056939 ]],

       [[0.92331116, 0.28815289, 0.68401985],
        [0.5202925 , 0.87736578, 0.92388931],
        [0.48923016, 0.59621396, 0.26427542]]])

source

retrieve_names

 retrieve_names (*args)

Tries to retrieve the argument name in the call frame, if there are multiple matches name is ’’

source

maybe_retrieve_callers_name

 maybe_retrieve_callers_name (args)

Tries to retrieve the argument name in the call frame, if there are multiple matches name is ’’

x, y, z = 1, 2, 3

def func(*args):
    return maybe_retrieve_callers_name(args)

print(func(x,y))
['x', 'y']
retrieve_names(a,a)
[['_', 'a', '_35'], ['_', 'a', '_35']]
trap = io.StringIO()
with redirect_stdout(trap):
    print("hello")

trap.getvalue()
'hello\n'

source

show_as_row

 show_as_row (*os, names:Iterable[str]=None, **kwargs)

Shows a interable of tensors on a row

source

row_items

 row_items (**kwargs)

source

pretty_repr

 pretty_repr (o)
type(a)
numpy.ndarray
show_as_row(a,b=a)

b

array([[[0.96646567, 0.58332229, 0.09242191],
        [0.0136295 , 0.83693011, 0.9147879 ],
        [0.70458626, 0.3870066 , 0.7056939 ]],

       [[0.92331116, 0.28815289, 0.68401985],
        [0.5202925 , 0.87736578, 0.92388931],
        [0.48923016, 0.59621396, 0.26427542]]])

a

array([[[0.96646567, 0.58332229, 0.09242191],
        [0.0136295 , 0.83693011, 0.9147879 ],
        [0.70458626, 0.3870066 , 0.7056939 ]],

       [[0.92331116, 0.28815289, 0.68401985],
        [0.5202925 , 0.87736578, 0.92388931],
        [0.48923016, 0.59621396, 0.26427542]]])

from itertools import zip_longest
zip_longest
itertools.zip_longest
func(a,a)
['a', 'a']
b = a.copy()
show_as_row([1], [2])

#0

[1]

#1

[2]

show_as_row(a,b)

a

array([[[0.96646567, 0.58332229, 0.09242191],
        [0.0136295 , 0.83693011, 0.9147879 ],
        [0.70458626, 0.3870066 , 0.7056939 ]],

       [[0.92331116, 0.28815289, 0.68401985],
        [0.5202925 , 0.87736578, 0.92388931],
        [0.48923016, 0.59621396, 0.26427542]]])

b

array([[[0.96646567, 0.58332229, 0.09242191],
        [0.0136295 , 0.83693011, 0.9147879 ],
        [0.70458626, 0.3870066 , 0.7056939 ]],

       [[0.92331116, 0.28815289, 0.68401985],
        [0.5202925 , 0.87736578, 0.92388931],
        [0.48923016, 0.59621396, 0.26427542]]])

show_as_row(a, names='b')

b

array([[[0.96646567, 0.58332229, 0.09242191],
        [0.0136295 , 0.83693011, 0.9147879 ],
        [0.70458626, 0.3870066 , 0.7056939 ]],

       [[0.92331116, 0.28815289, 0.68401985],
        [0.5202925 , 0.87736578, 0.92388931],
        [0.48923016, 0.59621396, 0.26427542]]])

show_as_row(c=a)

c

array([[[0.96646567, 0.58332229, 0.09242191],
        [0.0136295 , 0.83693011, 0.9147879 ],
        [0.70458626, 0.3870066 , 0.7056939 ]],

       [[0.92331116, 0.28815289, 0.68401985],
        [0.5202925 , 0.87736578, 0.92388931],
        [0.48923016, 0.59621396, 0.26427542]]])

source

display_as_row

 display_as_row (dfs:dict[str,pandas.core.frame.DataFrame], title='',
                 hide_idx=True, styler=<function _style_df>)

display multiple dataframes in the same row

source

row_dfs

 row_dfs (dfs:dict[str,pandas.core.frame.DataFrame], title='',
          hide_idx=True, styler=<function _style_df>)
a = HTML(pd.DataFrame([1,2]).to_html(notebook=True))
display_as_row({"test": pd.DataFrame([1,2])}, "hello")

hello

test

0
1
2 
display_as_row({"test": pd.DataFrame([1,2])}, "hello", hide_idx=False)

hello

test

  0
0 1
1 2 
display_as_row({f"test{i}": pd.DataFrame([1,2], columns=[i]) for i in range(10)})

test0

0
1
2

test1

1
1
2

test2

2
1
2

test3

3
1
2

test4

4
1
2

test5

5
1
2

test6

6
1
2

test7

7
1
2

test8

8
1
2

test9

9
1
2

Torch Helpers

inspired from source: https://github.com/pytorch/pytorch/pull/9281

from fastcore.test import *

source

eye_like

 eye_like (x:torch.Tensor)

Return a tensor with same batch size as x, that has a nxn eye matrix in each sample in batch.

Args: x: tensor of shape (B, n, m) or (n,m)

Returns: tensor of shape (B, n, m) or (n,m) that has the same dtype and device as x.

a = torch.ones(3,2,2)
test_close(eye_like(a),
           torch.tensor([[[1., 0.],
                          [0., 1.]],
 
                         [[1., 0.],
                          [0., 1.]],
 
                         [[1., 0.],
                          [0., 1.]]]))
eye_like(torch.ones(3,3))
tensor([[1., 0., 0.],
        [0., 1., 0.],
        [0., 0., 1.]])
eye_like(torch.ones(2,2,3,3))
tensor([[[[1., 0., 0.],
          [0., 1., 0.],
          [0., 0., 1.]],

         [[1., 0., 0.],
          [0., 1., 0.],
          [0., 0., 1.]]],


        [[[1., 0., 0.],
          [0., 1., 0.],
          [0., 0., 1.]],

         [[1., 0., 0.],
          [0., 1., 0.],
          [0., 0., 1.]]]])
eye_like(torch.ones(2,3,4))
tensor([[[1., 0., 0., 0.],
         [0., 1., 0., 0.],
         [0., 0., 1., 0.]],

        [[1., 0., 0., 0.],
         [0., 1., 0., 0.],
         [0., 0., 1., 0.]]])

Is diagonal

e = torch.eye(3,3)
d = torch.diagonal(e, dim1=-2, dim2=-1)
d
torch.diag_embed(d, dim1=-2, dim2=-1)
tensor([[1., 0., 0.],
        [0., 1., 0.],
        [0., 0., 1.]])

source

is_diagonal

 is_diagonal (x:torch.Tensor)

Check that tensor is diagonal respect to the last 2 dimensions

is_diagonal(torch.eye(3,3))
tensor(True)
is_diagonal(torch.ones(3,3))
tensor(False)
is_diagonal(eye_like(torch.ones(2,3,3)))
tensor(True)

Itertools

source

product_dict

 product_dict (**kwargs)

Export