basic data structure
| Python | Pytorch |
|---|---|
| int | intTensor |
| float | floatTensor |
| int array | intTensor[d1, d2, ...] |
| float array | floatTensor [d1, d2, ...] |
| string | -- |
how to denote string
One-hot
[0, 1, 0, 0, ...]
eg: [1, 0] dog [0, 1] cat
Embedding
Word2vec
glove
import torch, numpy as np
# type check
a = torch.randn(2, 3)
a, a.type(), type(a), isinstance(a, torch.FloatTensor)
(tensor([[ 0.8295, -1.3190, -2.3164],
[-0.0908, -0.1295, 0.4569]]),
'torch.FloatTensor',
torch.Tensor,
True)
data = a
print(isinstance(data, torch.cuda.FloatTensor))
data = data.cuda()
print(isinstance(data, torch.cuda.FloatTensor))
False
True
# 标量 dimension = 0
torch.tensor(1.), torch.tensor(1.3)
(tensor(1.), tensor(1.3000))
a = torch.tensor(2.2)
a.shape, len(a.shape), a.size()
(torch.Size([]), 0, torch.Size([]))
# Dim = 1 bias
x = torch.tensor([1.1])
# 由rand默认生成
y = torch.FloatTensor(1)
x, y
(tensor([1.1000]), tensor([7.1276e+10]))
a = torch.ones(2, 2)
a.shape
torch.Size([2, 2])
a = torch.randn(2, 3)
b = torch.rand_like(a)
a, a.shape, b, b.shape
(tensor([[ 0.3870, 1.4658, 0.3883],
[ 1.2135, -0.2156, 0.2044]]),
torch.Size([2, 3]),
tensor([[0.5810, 0.6718, 0.4941],
[0.3202, 0.1144, 0.6279]]),
torch.Size([2, 3]))
a = torch.rand(1, 2, 3)
a, a.shape, a[0][0][0]
(tensor([[[0.6286, 0.4754, 0.4326],
[0.8277, 0.0903, 0.8209]]]),
torch.Size([1, 2, 3]),
tensor(0.6286))
# CNN:[b, c, h ,w]
a = torch.rand(2, 3, 28, 28)
a, a.shape
(tensor([[[[0.5850, 0.3220, 0.2764, ..., 0.2583, 0.2023, 0.0405],
[0.2773, 0.0526, 0.4206, ..., 0.2651, 0.4935, 0.0529],
[0.8322, 0.6635, 0.3953, ..., 0.1945, 0.4148, 0.2811],
...,
[0.9784, 0.6646, 0.7006, ..., 0.0596, 0.4073, 0.7863],
[0.6283, 0.2102, 0.3565, ..., 0.7918, 0.3521, 0.2344],
[0.0729, 0.2326, 0.9389, ..., 0.0093, 0.2136, 0.7329]],
[[0.2355, 0.8272, 0.8288, ..., 0.6993, 0.5001, 0.7555],
[0.0067, 0.2415, 0.6324, ..., 0.7001, 0.8746, 0.6139],
[0.3608, 0.9725, 0.7737, ..., 0.5029, 0.3571, 0.9438],
...,
[0.7190, 0.5326, 0.8478, ..., 0.3921, 0.9292, 0.7709],
[0.9771, 0.5493, 0.0353, ..., 0.3389, 0.8328, 0.0263],
[0.9802, 0.5167, 0.4147, ..., 0.2320, 0.2288, 0.8932]],
[[0.9721, 0.2172, 0.1682, ..., 0.8864, 0.7916, 0.9658],
[0.1837, 0.8630, 0.5140, ..., 0.8898, 0.3345, 0.4016],
[0.2555, 0.9492, 0.0176, ..., 0.6252, 0.7736, 0.4769],
...,
[0.8322, 0.5678, 0.7323, ..., 0.0615, 0.9368, 0.5201],
[0.3365, 0.0571, 0.7331, ..., 0.9178, 0.1235, 0.5731],
[0.2730, 0.0068, 0.8020, ..., 0.9131, 0.1911, 0.2702]]],
[[[0.7454, 0.3667, 0.5293, ..., 0.0206, 0.3729, 0.5392],
[0.5271, 0.5590, 0.8741, ..., 0.3614, 0.8927, 0.4231],
[0.7738, 0.4327, 0.9357, ..., 0.5890, 0.9820, 0.3313],
...,
[0.1871, 0.7113, 0.9185, ..., 0.1552, 0.8232, 0.2313],
[0.4758, 0.7040, 0.3431, ..., 0.9336, 0.1191, 0.2481],
[0.8682, 0.7698, 0.7174, ..., 0.8403, 0.6177, 0.9751]],
[[0.6904, 0.2158, 0.1617, ..., 0.7632, 0.0191, 0.3103],
[0.8171, 0.6158, 0.8286, ..., 0.7803, 0.7403, 0.3246],
[0.0666, 0.1494, 0.5089, ..., 0.2545, 0.0162, 0.2392],
...,
[0.0290, 0.3450, 0.6067, ..., 0.6603, 0.0775, 0.1848],
[0.3419, 0.1592, 0.0742, ..., 0.6999, 0.3127, 0.4675],
[0.5114, 0.6280, 0.0165, ..., 0.7041, 0.7528, 0.0463]],
[[0.1924, 0.0242, 0.0404, ..., 0.1870, 0.2074, 0.7346],
[0.6063, 0.1382, 0.7763, ..., 0.7941, 0.5222, 0.8400],
[0.3330, 0.4453, 0.8121, ..., 0.1684, 0.2837, 0.9024],
...,
[0.3594, 0.5028, 0.8320, ..., 0.6542, 0.8193, 0.2418],
[0.4848, 0.7924, 0.3170, ..., 0.9198, 0.5620, 0.4489],
[0.1019, 0.1506, 0.1362, ..., 0.3712, 0.3361, 0.0715]]]]),
torch.Size([2, 3, 28, 28]))
a.numel() #2*3*28*28
4704
a.dim()
4
Create Tensor
a = np.array([2, 3.3])
torch.from_numpy(a)
tensor([2.0000, 3.3000], dtype=torch.float64)
a = np.ones([2, 3])
torch.from_numpy(a)
tensor([[1., 1., 1.],
[1., 1., 1.]], dtype=torch.float64)
# import from List
p = torch.Tensor(2,3) # recieve shape, generate randn array
q = torch.Tensor([2, 3.]) # recieve existed num
r = torch.tensor([2., 3.2]) # recieve existed num
s = torch.tensor(2.) # recieve existed num
p, q, r, s
(tensor([[0.4848, 0.7924, 0.3170],
[0.9198, 0.5620, 0.4489]]),
tensor([2., 3.]),
tensor([2.0000, 3.2000]),
tensor(2.))
# uninitialized
torch.empty(1), torch.Tensor(2,3)
(tensor([2.3694e-38]),
tensor([[0.8322, 0.6635, 0.3953],
[0.1945, 0.4148, 0.2811]]))
# set default type
print(torch.tensor([1.2, 3]).type())
torch.set_default_tensor_type(torch.DoubleTensor)
torch.tensor([1.2, 3]).type()
torch.FloatTensor
'torch.DoubleTensor'
# rand/rand_like,randint(int low, int high, *size)#random array
a = torch.randn(3, 3)
b = torch.rand_like(a)
a, b, torch.randint(1, 10, (3, 3))
(tensor([[-0.6969, 2.8789, 0.6738],
[ 0.0760, -0.9637, -0.2009],
[ 0.3328, -0.6778, -0.2165]]),
tensor([[0.6262, 0.8183, 0.6408],
[0.9939, 0.9125, 0.6677],
[0.5850, 0.6071, 0.3025]]),
tensor([[9, 5, 1],
[4, 7, 5],
[3, 1, 4]]))
# randn N(0, 1) --> N(u, std)
r = torch.randn(3,3)
n = torch.normal(mean=torch.full([10], 0.), std=torch.arange(1, 0, -0.1)) # mean参数后为浮点数
r, n
(tensor([[ 0.3386, -0.9174, 2.2254],
[ 0.3211, -0.6740, -1.5012],
[-0.1994, -0.4808, 0.1505]]),
tensor([1.0337, 0.5827, 1.0241, 0.6647, 0.1842, 0.8900, 0.1765, 0.0360, 0.0285,
0.0700]))
# Dim 1/rank 1
torch.tensor([1.1, 2.2]), torch.FloatTensor(2), torch.from_numpy(np.ones(2))
(tensor([1.1000, 2.2000]),
tensor([1.4013e-45, 0.0000e+00], dtype=torch.float32),
tensor([1., 1.]))
a = torch.rand(1, 2, 3)
a, a.shape, a[0], list(a.shape)
(tensor([[[0.1973, 0.2405, 0.8402],
[0.2945, 0.4635, 0.3633]]]),
torch.Size([1, 2, 3]),
tensor([[0.1973, 0.2405, 0.8402],
[0.2945, 0.4635, 0.3633]]),
[1, 2, 3])
# Dim 4
a = torch.rand(2, 3, 28, 28)
a, a.shape
(tensor([[[[0.3507, 0.7074, 0.7228, ..., 0.7881, 0.0783, 0.0578],
[0.5794, 0.4129, 0.8776, ..., 0.3997, 0.2519, 0.7105],
[0.3407, 0.1594, 0.2745, ..., 0.4245, 0.3867, 0.1613],
...,
[0.9715, 0.0773, 0.7464, ..., 0.5181, 0.2834, 0.0373],
[0.9817, 0.8491, 0.0954, ..., 0.2347, 0.1792, 0.3911],
[0.6301, 0.0197, 0.5734, ..., 0.2933, 0.6313, 0.0677]],
[[0.0756, 0.4865, 0.8576, ..., 0.7353, 0.7030, 0.9222],
[0.3190, 0.3342, 0.0992, ..., 0.1880, 0.6765, 0.9833],
[0.9906, 0.1591, 0.1585, ..., 0.3735, 0.8033, 0.8630],
...,
[0.4100, 0.1268, 0.3933, ..., 0.7481, 0.2758, 0.9460],
[0.5199, 0.7700, 0.6702, ..., 0.9368, 0.1708, 0.9260],
[0.3144, 0.9081, 0.1070, ..., 0.6255, 0.6888, 0.4264]],
[[0.2917, 0.2633, 0.2501, ..., 0.4316, 0.2877, 0.0931],
[0.1374, 0.2878, 0.7344, ..., 0.4526, 0.6517, 0.8001],
[0.3078, 0.9782, 0.0181, ..., 0.2421, 0.2989, 0.5481],
...,
[0.9766, 0.2363, 0.7844, ..., 0.6815, 0.4563, 0.8133],
[0.6053, 0.0810, 0.3398, ..., 0.2780, 0.3869, 0.1035],
[0.3582, 0.1621, 0.8924, ..., 0.9201, 0.2136, 0.8067]]],
[[[0.3342, 0.4407, 0.3751, ..., 0.1788, 0.9775, 0.6619],
[0.6103, 0.9657, 0.6862, ..., 0.1565, 0.4375, 0.0496],
[0.5700, 0.3863, 0.2647, ..., 0.8094, 0.0087, 0.1453],
...,
[0.0993, 0.4800, 0.5050, ..., 0.3777, 0.7112, 0.6149],
[0.7586, 0.8472, 0.8335, ..., 0.0260, 0.1709, 0.2116],
[0.0196, 0.1113, 0.5924, ..., 0.6154, 0.6695, 0.0448]],
[[0.6665, 0.5068, 0.8970, ..., 0.4390, 0.5921, 0.9491],
[0.8297, 0.5459, 0.1529, ..., 0.2910, 0.5097, 0.6766],
[0.3556, 0.7632, 0.4692, ..., 0.1013, 0.6309, 0.7228],
...,
[0.6769, 0.3452, 0.4071, ..., 0.3114, 0.8521, 0.5124],
[0.7649, 0.5354, 0.9400, ..., 0.1221, 0.2676, 0.5711],
[0.3562, 0.4829, 0.5393, ..., 0.9557, 0.5115, 0.7727]],
[[0.4354, 0.5601, 0.3857, ..., 0.1080, 0.5042, 0.4194],
[0.1533, 0.8310, 0.5890, ..., 0.6971, 0.2073, 0.9577],
[0.2654, 0.9521, 0.7436, ..., 0.4113, 0.0029, 0.7534],
...,
[0.6257, 0.3254, 0.9342, ..., 0.7368, 0.6528, 0.2563],
[0.8983, 0.1168, 0.2811, ..., 0.7723, 0.1052, 0.2291],
[0.3017, 0.3654, 0.6408, ..., 0.3079, 0.9327, 0.3654]]]]),
torch.Size([2, 3, 28, 28]))
# Mixed
a.shape, a.numel(), a.dim()
(torch.Size([2, 3, 28, 28]), 4704, 4)
torch.full([2,3], 7), torch.full([], 7), torch.full([2], 7)
(tensor([[7, 7, 7],
[7, 7, 7]]),
tensor(7),
tensor([7, 7]))
torch.arange(0, 10), torch.arange(0 , 10 ,2)
(tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), tensor([0, 2, 4, 6, 8]))
torch.linspace(0, 10, steps=4)
tensor([ 0.0000, 3.3333, 6.6667, 10.0000])
torch.ones(3, 3), torch.zeros(3, 3), torch.eye(3, 3)
(tensor([[1., 1., 1.],
[1., 1., 1.],
[1., 1., 1.]]),
tensor([[0., 0., 0.],
[0., 0., 0.],
[0., 0., 0.]]),
tensor([[1., 0., 0.],
[0., 1., 0.],
[0., 0., 1.]]))
# randperm
a, b, idx = torch.rand(2, 3), torch.rand(2, 2), torch.randperm(2)
a[idx], b[idx], a, b, idx
(tensor([[0.8900, 0.7087, 0.0572],
[0.0787, 0.9646, 0.2514]]),
tensor([[0.9071, 0.3137],
[0.2353, 0.6256]]),
tensor([[0.8900, 0.7087, 0.0572],
[0.0787, 0.9646, 0.2514]]),
tensor([[0.9071, 0.3137],
[0.2353, 0.6256]]),
tensor([0, 1]))
# Indexing
a = torch.rand(4, 3, 28 ,28)
a[0].shape, a[0][0].shape, a[0, 0, 2, 4]
(torch.Size([3, 28, 28]), torch.Size([28, 28]), tensor(0.3711))
a.shape, a[:2].shape, a[:2, :1, :, :].shape, a[:2, 1:, :, :].shape, a[:2, -1:, :, :].shape
(torch.Size([4, 3, 28, 28]),
torch.Size([2, 3, 28, 28]),
torch.Size([2, 1, 28, 28]),
torch.Size([2, 2, 28, 28]),
torch.Size([2, 1, 28, 28]))
a[:, :, 0:28:2, 0:28:2].shape, a[:, :, ::2, ::2].shape
(torch.Size([4, 3, 14, 14]), torch.Size([4, 3, 14, 14]))
a.shape, a.index_select(0, torch.tensor([0, 2])).shape
(torch.Size([4, 3, 28, 28]), torch.Size([2, 3, 28, 28]))
a[...].shape, a[0, ...].shape, a[:,1,...].shape, a[..., :2].shape
(torch.Size([4, 3, 28, 28]),
torch.Size([3, 28, 28]),
torch.Size([4, 28, 28]),
torch.Size([4, 3, 28, 2]))
x = torch.randn(3, 4)
mask = x.ge(0.5)
torch.masked_select(x ,mask), torch.masked_select(x, mask).shape
(tensor([0.9735, 0.7977, 1.3304, 1.0590, 0.5475]), torch.Size([5]))
src = torch.tensor([[4, 3, 5],[6, 7, 8]])
torch.take(src,torch.tensor([0 , 2, 1]))
tensor([4, 5, 3])
dimension transform
a = torch.rand(4, 1, 28, 28)
a.shape, a.view(4, 28*28), a.view(4, 28*28).shape, a.view(4*28, 28).shape, a.view(4*1, 28, 28).shape
#可能拉伸后损失物理意义
(torch.Size([4, 1, 28, 28]),
tensor([[0.8821, 0.8922, 0.4360, ..., 0.5728, 0.9648, 0.8665],
[0.6512, 0.3380, 0.9317, ..., 0.7917, 0.1386, 0.8332],
[0.8190, 0.0530, 0.1702, ..., 0.5868, 0.6179, 0.6288],
[0.3834, 0.0832, 0.7733, ..., 0.9742, 0.2182, 0.3308]]),
torch.Size([4, 784]),
torch.Size([112, 28]),
torch.Size([4, 28, 28]))
# squeeze unsqueeze 插入维度(下标)[-a.dim()-1, a.dim()+1]=[-5, 5]
a.shape, a.unsqueeze(0).shape, a.unsqueeze(-1).shape, a.unsqueeze(4).shape, a.unsqueeze(-4).shape
(torch.Size([4, 1, 28, 28]),
torch.Size([1, 4, 1, 28, 28]),
torch.Size([4, 1, 28, 28, 1]),
torch.Size([4, 1, 28, 28, 1]),
torch.Size([4, 1, 1, 28, 28]))
# squeeze(idx)挤压shape=1的维度 squeeze()能挤压的都挤压
a.shape, a.squeeze().shape, a.squeeze(1).shape
(torch.Size([4, 1, 28, 28]), torch.Size([4, 28, 28]), torch.Size([4, 28, 28]))
# Expand broadcasting
# repeat memory copied
a = torch.rand(4, 32, 14, 14)
b = torch.rand(1, 32, 1, 1)
b.shape, b.expand(4, 32, 14, 14).shape, b.expand(-1, 32, -1, -1).shape #-1保持不变
(torch.Size([1, 32, 1, 1]),
torch.Size([4, 32, 14, 14]),
torch.Size([1, 32, 1, 1]))
b.shape, b.repeat(4, 32, 1, 1).shape, b.repeat(4, 1, 1, 1).shape #参数为对应dimension需要repeat的次数
(torch.Size([1, 32, 1, 1]),
torch.Size([4, 1024, 1, 1]),
torch.Size([4, 32, 1, 1]))
a = torch.randn(1, 4)
a, a.t()
(tensor([[1.4993, 0.9613, 0.3209, 0.0253]]),
tensor([[1.4993],
[0.9613],
[0.3209],
[0.0253]]))
b = torch.rand(4, 3, 32, 32)
b.shape, b.transpose(1, 3).shape
(torch.Size([4, 3, 32, 32]), torch.Size([4, 32, 32, 3]))
#b.transpose(1, 3).view(4, 3 * 32 * 32).view(4, 3, 32, 32)
"""---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
~\AppData\Local\Temp/ipykernel_1676/240077693.py in <module>
----> 1 b.transpose(1, 3).view(4, 3 * 32 * 32).view(4, 3, 32, 32)
RuntimeError: view size is not compatible with input tensor's size and stride
(at least one dimension spans across two contiguous subspaces). Use .reshape(...) instead."""
a1 = b.transpose(1, 3).contiguous().view(4, 3 * 32 * 32).view(4, 3, 32, 32)
# [B C H W] [B W H C] [B C W H]
a2 = b.transpose(1, 3).contiguous().view(4, 3 * 32 * 32).view(4, 32, 32, 3).transpose(1, 3)
# [B C H W] [B W H C] [B C H W]
a1.shape, a2.shape
(torch.Size([4, 3, 32, 32]), torch.Size([4, 3, 32, 32]))
torch.all(torch.eq(b, a1)), torch.all(torch.eq(b, a2))
(tensor(False), tensor(True))
# permute
a, b = torch.rand(4, 3, 28, 28), torch.rand(4, 3, 28, 32)
a.transpose(1, 3).shape, b.transpose(1, 3).shape, b.transpose(1, 3).transpose(1, 2).shape, b.permute(0, 2, 3, 1).shape
(torch.Size([4, 28, 28, 3]),
torch.Size([4, 32, 28, 3]),
torch.Size([4, 28, 32, 3]),
torch.Size([4, 28, 32, 3]))
Broadcast 自动扩展
Insert 1 dim ahead
Expand dims with size 1 to same size
Feature maps: [4, 32, 14, 14] Bias: [32, 1, 1] =>[1, 32, 1, 1] => [4, 32, 14, 14]
Why broadcasting
- for actual demanding
[class, students, scores]
Add bias for every students: +5 score
[4, 32, 8] + [4, 32, 8]
[4, 32, 8] + '[5.0]' - memory consumption
[4, 32, 8] => 1024
[5.0] => 1
Is it broadcasting-able?
Match from Last dim
if current dim=1, expand to same
if either has no dim, insert one dim and expand to same
otherwise, Not broadcasting-able
Situation :
[4, 32, 14, 14]
S1. [1, 32, 1, 1] => [4, 32, 14, 14]
S2. [14, 14] => [1, 1, 14, 14] => [4, 32, 14, 14]
S3. [2, 32, 14, 14]
Dim 0 has dim, can Not insert and expand to same
Dim 0 has distinct dim, Not size 1
Not broadcasting-able
How to understand this behavior?
match from last dim
When it has no dim
treat it as all own the same
[class, student, scores] + [scores]
When it has dim of size 1
treat it shared by all
[class, student, scores] + [student, 1]
Merge or split
a, b = torch.rand(4, 32, 8), torch.rand(5, 32, 8)
torch.cat([a, b], dim=0).shape
torch.Size([9, 32, 8])
