Download as PDF, PPTX
![NumPyだけによる実装
import numpy as np
epochs = 300
batch_size = 32
D_in = 784
H = 100
D_out = 10
learning_rate = 1.0e-06
# create random input and output data
x = np.random.randn(batch_size, D_in)
y = np.random.randn(batch_size, D_out)
# randomly initialize weights
w1 = np.random.randn(D_in, H)
w2 = np.random.randn(H, D_out)
for epoch in range(epochs):
# forward pass
h = x.dot(w1) # h = x * w1
h_r = np.maximum(h, 0) # h_r = ReLU(h)
y_p = h_r.dot(w2) # y_p = h_r * w2
# compute mean squared error and print loss
loss = np.square(y_p - y).sum()
print(epoch, loss)
# backward pass: compute gradients of loss with respect to w2
grad_y_p = 2.0 * (y_p - y)
grad_w2 = h_r.T.dot(grad_y_p)
# backward pass: compute gradients of loss with respect to w1
grad_h_r = grad_y_p.dot(w2.T)
grad_h = grad_h_r.copy()
grad_h[h < 0] = 0
grad_w1 = x.T.dot(grad_h)
# update weights
w1 -= learning_rate * grad_w1
w2 -= learning_rate * grad_w2
w1 w1 ⌘
@L
@w1
<latexit sha1_base64="kN3sQo8OP8glKG68w4PsUtC/f3k=">AAACMXicbVDLSgNBEJyN73fUo5fBIHgx7IqiR9GLBw8RjArZEHonvcmQ2QczvYaw5Ev8DL/Aq35BbiJ48iecjQGfDQM1VdXTPRWkShpy3ZFTmpqemZ2bX1hcWl5ZXSuvb1ybJNMC6yJRib4NwKCSMdZJksLbVCNEgcKboHdW6Dd3qI1M4isapNiMoBPLUAogS7XKh/2W5ysMCbRO+tze9nwk8EMNIvdT0CRB8YvhF7aWYatccavuuPhf4E1AhU2q1iq/+e1EZBHGJBQY0/DclJp58aRQOFz0M4MpiB50sGFhDBGaZj7+3pDvWKbNw0TbExMfs987coiMGUSBdUZAXfNbK8j/tEZG4XEzl3GaEcbic1CYKU4JL7LibalRkBpYAEJLuysXXbDJkE30x5S2KVYrcvF+p/AXXO9XPbfqXR5UTk4nCc2zLbbNdpnHjtgJO2c1VmeC3bNH9sSenQdn5Lw4r5/WkjPp2WQ/ynn/AO/Rq1o=</latexit>
<latexit sha1_base64="kN3sQo8OP8glKG68w4PsUtC/f3k=">AAACMXicbVDLSgNBEJyN73fUo5fBIHgx7IqiR9GLBw8RjArZEHonvcmQ2QczvYaw5Ev8DL/Aq35BbiJ48iecjQGfDQM1VdXTPRWkShpy3ZFTmpqemZ2bX1hcWl5ZXSuvb1ybJNMC6yJRib4NwKCSMdZJksLbVCNEgcKboHdW6Dd3qI1M4isapNiMoBPLUAogS7XKh/2W5ysMCbRO+tze9nwk8EMNIvdT0CRB8YvhF7aWYatccavuuPhf4E1AhU2q1iq/+e1EZBHGJBQY0/DclJp58aRQOFz0M4MpiB50sGFhDBGaZj7+3pDvWKbNw0TbExMfs987coiMGUSBdUZAXfNbK8j/tEZG4XEzl3GaEcbic1CYKU4JL7LibalRkBpYAEJLuysXXbDJkE30x5S2KVYrcvF+p/AXXO9XPbfqXR5UTk4nCc2zLbbNdpnHjtgJO2c1VmeC3bNH9sSenQdn5Lw4r5/WkjPp2WQ/ynn/AO/Rq1o=</latexit>
<latexit sha1_base64="kN3sQo8OP8glKG68w4PsUtC/f3k=">AAACMXicbVDLSgNBEJyN73fUo5fBIHgx7IqiR9GLBw8RjArZEHonvcmQ2QczvYaw5Ev8DL/Aq35BbiJ48iecjQGfDQM1VdXTPRWkShpy3ZFTmpqemZ2bX1hcWl5ZXSuvb1ybJNMC6yJRib4NwKCSMdZJksLbVCNEgcKboHdW6Dd3qI1M4isapNiMoBPLUAogS7XKh/2W5ysMCbRO+tze9nwk8EMNIvdT0CRB8YvhF7aWYatccavuuPhf4E1AhU2q1iq/+e1EZBHGJBQY0/DclJp58aRQOFz0M4MpiB50sGFhDBGaZj7+3pDvWKbNw0TbExMfs987coiMGUSBdUZAXfNbK8j/tEZG4XEzl3GaEcbic1CYKU4JL7LibalRkBpYAEJLuysXXbDJkE30x5S2KVYrcvF+p/AXXO9XPbfqXR5UTk4nCc2zLbbNdpnHjtgJO2c1VmeC3bNH9sSenQdn5Lw4r5/WkjPp2WQ/ynn/AO/Rq1o=</latexit>
<latexit sha1_base64="kN3sQo8OP8glKG68w4PsUtC/f3k=">AAACMXicbVDLSgNBEJyN73fUo5fBIHgx7IqiR9GLBw8RjArZEHonvcmQ2QczvYaw5Ev8DL/Aq35BbiJ48iecjQGfDQM1VdXTPRWkShpy3ZFTmpqemZ2bX1hcWl5ZXSuvb1ybJNMC6yJRib4NwKCSMdZJksLbVCNEgcKboHdW6Dd3qI1M4isapNiMoBPLUAogS7XKh/2W5ysMCbRO+tze9nwk8EMNIvdT0CRB8YvhF7aWYatccavuuPhf4E1AhU2q1iq/+e1EZBHGJBQY0/DclJp58aRQOFz0M4MpiB50sGFhDBGaZj7+3pDvWKbNw0TbExMfs987coiMGUSBdUZAXfNbK8j/tEZG4XEzl3GaEcbic1CYKU4JL7LibalRkBpYAEJLuysXXbDJkE30x5S2KVYrcvF+p/AXXO9XPbfqXR5UTk4nCc2zLbbNdpnHjtgJO2c1VmeC3bNH9sSenQdn5Lw4r5/WkjPp2WQ/ynn/AO/Rq1o=</latexit>
w2 w2 ⌘
@L
@w2
<latexit sha1_base64="XDVYNpwB7UZogd7iSX6oklwpHpw=">AAACMXicbVDLSgNBEJz1bXxFPXoZDIIXw64oehS9ePAQwZhANoTeSa8Ozj6Y6TWEJV/iZ/gFXvULchPBkz/hbAz4iA0DNVXV0z0VpEoact2hMzU9Mzs3v7BYWlpeWV0rr29cmyTTAusiUYluBmBQyRjrJElhM9UIUaCwEdydFXrjHrWRSXxF/RTbEdzEMpQCyFKd8mGvs+8rDAm0Tnrc3vZ8JPBDDSL3U9AkQfGLwTe2lkGnXHGr7qj4JPDGoMLGVeuU3/1uIrIIYxIKjGl5bkrtvHhSKByU/MxgCuIObrBlYQwRmnY++t6A71imy8NE2xMTH7E/O3KIjOlHgXVGQLfmr1aQ/2mtjMLjdi7jNCOMxdegMFOcEl5kxbtSoyDVtwCElnZXLm7BJkM20V9TuqZYrcjF+5vCJLjer3pu1bs8qJycjhNaYFtsm+0yjx2xE3bOaqzOBHtgT+yZvTiPztB5dd6+rFPOuGeT/Srn4xP07atd</latexit>
<latexit sha1_base64="XDVYNpwB7UZogd7iSX6oklwpHpw=">AAACMXicbVDLSgNBEJz1bXxFPXoZDIIXw64oehS9ePAQwZhANoTeSa8Ozj6Y6TWEJV/iZ/gFXvULchPBkz/hbAz4iA0DNVXV0z0VpEoact2hMzU9Mzs3v7BYWlpeWV0rr29cmyTTAusiUYluBmBQyRjrJElhM9UIUaCwEdydFXrjHrWRSXxF/RTbEdzEMpQCyFKd8mGvs+8rDAm0Tnrc3vZ8JPBDDSL3U9AkQfGLwTe2lkGnXHGr7qj4JPDGoMLGVeuU3/1uIrIIYxIKjGl5bkrtvHhSKByU/MxgCuIObrBlYQwRmnY++t6A71imy8NE2xMTH7E/O3KIjOlHgXVGQLfmr1aQ/2mtjMLjdi7jNCOMxdegMFOcEl5kxbtSoyDVtwCElnZXLm7BJkM20V9TuqZYrcjF+5vCJLjer3pu1bs8qJycjhNaYFtsm+0yjx2xE3bOaqzOBHtgT+yZvTiPztB5dd6+rFPOuGeT/Srn4xP07atd</latexit>
<latexit sha1_base64="XDVYNpwB7UZogd7iSX6oklwpHpw=">AAACMXicbVDLSgNBEJz1bXxFPXoZDIIXw64oehS9ePAQwZhANoTeSa8Ozj6Y6TWEJV/iZ/gFXvULchPBkz/hbAz4iA0DNVXV0z0VpEoact2hMzU9Mzs3v7BYWlpeWV0rr29cmyTTAusiUYluBmBQyRjrJElhM9UIUaCwEdydFXrjHrWRSXxF/RTbEdzEMpQCyFKd8mGvs+8rDAm0Tnrc3vZ8JPBDDSL3U9AkQfGLwTe2lkGnXHGr7qj4JPDGoMLGVeuU3/1uIrIIYxIKjGl5bkrtvHhSKByU/MxgCuIObrBlYQwRmnY++t6A71imy8NE2xMTH7E/O3KIjOlHgXVGQLfmr1aQ/2mtjMLjdi7jNCOMxdegMFOcEl5kxbtSoyDVtwCElnZXLm7BJkM20V9TuqZYrcjF+5vCJLjer3pu1bs8qJycjhNaYFtsm+0yjx2xE3bOaqzOBHtgT+yZvTiPztB5dd6+rFPOuGeT/Srn4xP07atd</latexit>
<latexit sha1_base64="XDVYNpwB7UZogd7iSX6oklwpHpw=">AAACMXicbVDLSgNBEJz1bXxFPXoZDIIXw64oehS9ePAQwZhANoTeSa8Ozj6Y6TWEJV/iZ/gFXvULchPBkz/hbAz4iA0DNVXV0z0VpEoact2hMzU9Mzs3v7BYWlpeWV0rr29cmyTTAusiUYluBmBQyRjrJElhM9UIUaCwEdydFXrjHrWRSXxF/RTbEdzEMpQCyFKd8mGvs+8rDAm0Tnrc3vZ8JPBDDSL3U9AkQfGLwTe2lkGnXHGr7qj4JPDGoMLGVeuU3/1uIrIIYxIKjGl5bkrtvHhSKByU/MxgCuIObrBlYQwRmnY++t6A71imy8NE2xMTH7E/O3KIjOlHgXVGQLfmr1aQ/2mtjMLjdi7jNCOMxdegMFOcEl5kxbtSoyDVtwCElnZXLm7BJkM20V9TuqZYrcjF+5vCJLjer3pu1bs8qJycjhNaYFtsm+0yjx2xE3bOaqzOBHtgT+yZvTiPztB5dd6+rFPOuGeT/Srn4xP07atd</latexit>
@L
@w2
=
@L
@yp
@yp
@w2
=
1
NO
2 (yp y) hr
@L
@w1
=
@L
@yp
@yp
@hr
@hr
@w1
=
1
NO
2 (yp y) w2x
L =
1
NO
X
(yp y)
2
00_numpy.py](https://image.slidesharecdn.com/0729yokota-210727025128/85/15-A-2021-18-320.jpg)
![PyTorch の導入
import torch
epochs = 300
batch_size = 32
D_in = 784
H = 100
D_out = 10
learning_rate = 1.0e-06
# create random input and output data
x = torch.randn(batch_size, D_in)
y = torch.randn(batch_size, D_out)
# randomly initialize weights
w1 = torch.randn(D_in, H)
w2 = torch.randn(H, D_out)
for epoch in range(epochs):
# forward pass: compute predicted y
h = x.mm(w1)
h_r = h.clamp(min=0)
y_p = h_r.mm(w2)
# compute and print loss
loss = (y_p - y).pow(2).sum().item()
print(t, loss)
# backward pass: compute gradients of loss with respect to w2
grad_y_p = 2.0 * (y_p - y)
grad_w2 = h_r.t().mm(grad_y_p)
# backward pass: compute gradients of loss with respect to w1
grad_h_r = grad_y_p.mm(w2.t())
grad_h = grad_h_r.clone()
grad_h[h < 0] = 0
grad_w1 = x.t().mm(grad_h)
# update weights
w1 -= learning_rate * grad_w1
w2 -= learning_rate * grad_w2
np.random torch
np torch
x.dot(w1) x.mm(w1)
np.maximum(h, 0) h.clamp(min=0)
np.square(y_p-y) (y_p-y).pow(2)
copy() clone()
01_tensors.py](https://image.slidesharecdn.com/0729yokota-210727025128/85/15-A-2021-19-320.jpg)
![自動微分の導入
# randomly initialize weights
w1 = torch.randn(D_in, H)
w2 = torch.randn(H, D_out)
for epoch in range(epochs):
# forward pass: compute predicted y
h = x.mm(w1)
h_r = h.clamp(min=0)
y_p = h_r.mm(w2)
# compute and print loss
loss = (y_p - y).pow(2).sum().item()
print(t, loss)
# backward pass: compute gradients of loss
with respect to w2
grad_y_p = 2.0 * (y_p - y)
grad_w2 = h_r.t().mm(grad_y_p)
# backward pass: compute gradients of loss
with respect to w1
grad_h_r = grad_y_p.mm(w2.t())
grad_h = grad_h_r.clone()
grad_h[h < 0] = 0
grad_w1 = x.t().mm(grad_h)
# update weights
w1 -= learning_rate * grad_w1
w2 -= learning_rate * grad_w2
01_tensor.py 02_autograd.py
# randomly initialize weights
w1 = torch.randn(D_in, H, requires_grad=True)
w2 = torch.randn(H, D_out, requires_grad=True)
for epoch in range(epochs):
# forward pass: compute predicted y
h = x.mm(w1)
h_r = h.clamp(min=0)
y_p = h_r.mm(w2)
# compute and print loss
loss = (y_p - y).pow(2).sum()
print(t, loss.item())
# backward pass
loss.backward()
with torch.no_grad():
# update weights
w1 -= learning_rate * w1.grad
w2 -= learning_rate * w2.grad
# initialize weights
w1.grad.zero_()
w2.grad.zero_()
@L
@w1
=
@L
@yp
@yp
@hr
@hr
@w1
=
1
NO
2(yp y)w2x
<latexit sha1_base64="V1OkoDW7pmfxcKKULrjvVELuV8s=">AAACmHicfVHdSsMwGE3r//ybeqc3wSHMC0dTBL0Rhl6oIDrFqbCNkmbpFkx/SFJnKX0TX8wH8D1Mt8JcFT8InJzzfTkfJ27EmVSW9WmYc/MLi0vLK5XVtfWNzerW9pMMY0Fom4Q8FC8ulpSzgLYVU5y+RIJi3+X02X29yPXnNyokC4NHlUS05+NBwDxGsNKUU/3oegKTtBthoRjm8Cab4pGDsrN/9MSJspKcU9Pb0BHlhpyadagUFihLb527zK7rN46Sw5FjvzvVmtWwxgV/A1SAGiiq5VS/uv2QxD4NFOFYyg6yItVLczfCaVbpxpJGmLziAe1oGGCfyl46TjGDB5rpQy8U+gQKjtmfEyn2pUx8V3f6WA1lWcvJv7ROrLzTXsqCKFY0IBMjL+ZQhTD/EthnghLFEw0wEUzvCskQ60yU/rgZl77MV8t0Lqicwm/wZDeQ1UD3x7XmeZHQMtgD+6AOEDgBTXAFWqANiGEadQMZtrlrNs1L83rSahrFzA6YKfPhG92OzlM=</latexit>
<latexit sha1_base64="V1OkoDW7pmfxcKKULrjvVELuV8s=">AAACmHicfVHdSsMwGE3r//ybeqc3wSHMC0dTBL0Rhl6oIDrFqbCNkmbpFkx/SFJnKX0TX8wH8D1Mt8JcFT8InJzzfTkfJ27EmVSW9WmYc/MLi0vLK5XVtfWNzerW9pMMY0Fom4Q8FC8ulpSzgLYVU5y+RIJi3+X02X29yPXnNyokC4NHlUS05+NBwDxGsNKUU/3oegKTtBthoRjm8Cab4pGDsrN/9MSJspKcU9Pb0BHlhpyadagUFihLb527zK7rN46Sw5FjvzvVmtWwxgV/A1SAGiiq5VS/uv2QxD4NFOFYyg6yItVLczfCaVbpxpJGmLziAe1oGGCfyl46TjGDB5rpQy8U+gQKjtmfEyn2pUx8V3f6WA1lWcvJv7ROrLzTXsqCKFY0IBMjL+ZQhTD/EthnghLFEw0wEUzvCskQ60yU/rgZl77MV8t0Lqicwm/wZDeQ1UD3x7XmeZHQMtgD+6AOEDgBTXAFWqANiGEadQMZtrlrNs1L83rSahrFzA6YKfPhG92OzlM=</latexit>
<latexit sha1_base64="V1OkoDW7pmfxcKKULrjvVELuV8s=">AAACmHicfVHdSsMwGE3r//ybeqc3wSHMC0dTBL0Rhl6oIDrFqbCNkmbpFkx/SFJnKX0TX8wH8D1Mt8JcFT8InJzzfTkfJ27EmVSW9WmYc/MLi0vLK5XVtfWNzerW9pMMY0Fom4Q8FC8ulpSzgLYVU5y+RIJi3+X02X29yPXnNyokC4NHlUS05+NBwDxGsNKUU/3oegKTtBthoRjm8Cab4pGDsrN/9MSJspKcU9Pb0BHlhpyadagUFihLb527zK7rN46Sw5FjvzvVmtWwxgV/A1SAGiiq5VS/uv2QxD4NFOFYyg6yItVLczfCaVbpxpJGmLziAe1oGGCfyl46TjGDB5rpQy8U+gQKjtmfEyn2pUx8V3f6WA1lWcvJv7ROrLzTXsqCKFY0IBMjL+ZQhTD/EthnghLFEw0wEUzvCskQ60yU/rgZl77MV8t0Lqicwm/wZDeQ1UD3x7XmeZHQMtgD+6AOEDgBTXAFWqANiGEadQMZtrlrNs1L83rSahrFzA6YKfPhG92OzlM=</latexit>
<latexit sha1_base64="V1OkoDW7pmfxcKKULrjvVELuV8s=">AAACmHicfVHdSsMwGE3r//ybeqc3wSHMC0dTBL0Rhl6oIDrFqbCNkmbpFkx/SFJnKX0TX8wH8D1Mt8JcFT8InJzzfTkfJ27EmVSW9WmYc/MLi0vLK5XVtfWNzerW9pMMY0Fom4Q8FC8ulpSzgLYVU5y+RIJi3+X02X29yPXnNyokC4NHlUS05+NBwDxGsNKUU/3oegKTtBthoRjm8Cab4pGDsrN/9MSJspKcU9Pb0BHlhpyadagUFihLb527zK7rN46Sw5FjvzvVmtWwxgV/A1SAGiiq5VS/uv2QxD4NFOFYyg6yItVLczfCaVbpxpJGmLziAe1oGGCfyl46TjGDB5rpQy8U+gQKjtmfEyn2pUx8V3f6WA1lWcvJv7ROrLzTXsqCKFY0IBMjL+ZQhTD/EthnghLFEw0wEUzvCskQ60yU/rgZl77MV8t0Lqicwm/wZDeQ1UD3x7XmeZHQMtgD+6AOEDgBTXAFWqANiGEadQMZtrlrNs1L83rSahrFzA6YKfPhG92OzlM=</latexit>
微分を自動的に計算してくれる](https://image.slidesharecdn.com/0729yokota-210727025128/85/15-A-2021-20-320.jpg)
![活性化関数の自作 03_function.py
import torch
for epoch in range(epochs):
# forward pass: compute predicted y
h = x.mm(w1)
h_r = h.clamp(min=0)
y_p = h_r.mm(w2)
02_autograd.py
import torch
class ReLU(torch.autograd.Function):
@staticmethod
def forward(ctx, input):
ctx.save_for_backward(input)
return input.clamp(min=0)
@staticmethod
def backward(ctx, grad_output):
input, = ctx.saved_tensors
grad_input = grad_output.clone()
grad_input[input<0] = 0
return grad_input
for epoch in range(epochs):
# forward pass: compute predicted y
relu = ReLU.apply
h = x.mm(w1)
h_r = relu(h)
y_p = h_r.mm(w2)
.
.
.
.
.
.
y=f(x)
ReLU (Rectified Linear Unit)](https://image.slidesharecdn.com/0729yokota-210727025128/85/15-A-2021-21-320.jpg)
![Validationデータによる検証
08_validate.py
def validate():
model.eval()
val_loss, val_acc = 0, 0
for data, target in val_loader:
output = model(data)
loss = criterion(output, target)
val_loss += loss.item()
pred = output.data.max(1)[1]
val_acc += 100. * pred.eq(target.data).cpu().sum() / target.size(0)
val_loss /= len(val_loader)
val_acc /= len(val_loader)
print('nValidation set: Average loss: {:.4f}, Accuracy: {:.1f}%n'.format(
val_loss, val_acc))
学習時に使うデータ
ハイパラやモデル
を変えて試すとき
に使うデータ
最終的な精度の評価
に使うデータ
Validation dataのloss
予測クラスがラベルと一致しているか？
パーセンテージに変換
sum()はGPUでやると遅いのでCPUで](https://image.slidesharecdn.com/0729yokota-210727025128/85/15-A-2021-26-320.jpg)
![train(), main()関数の形で書く
09_train.py
def train(train_loader,model,criterion,optimizer,epoch):
model.train()
t = time.perf_counter()
for batch_idx, (data, target) in enumerate(train_loader):
output = model(data)
loss = criterion(output, target)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if batch_idx % 200 == 0:
print('Train Epoch: {} [{:>5}/{} ({:.0%})]tLoss: {:.6f}t Time:{:.4f}'.format(
epoch, batch_idx * len(data), len(train_loader.dataset),
batch_idx / len(train_loader), loss.data.item(),
time.perf_counter() - t))
t = time.perf_counter()
def main():
epochs = 10
batch_size = 32
learning_rate = 1.0e-02
train_dataset = datasets.MNIST('./data',
train=True,
download=True,
transform=transforms.ToTensor())
val_dataset = datasets.MNIST('./data',
train=False,
transform=transforms.ToTensor())
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)
val_loader = torch.utils.data.DataLoader(dataset=validation_dataset,
batch_size=batch_size,
shuffle=False)
model = TwoLayerNet(D_in, H, D_out)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
for epoch in range(epochs):
model.train()
train(train_loader,model,criterion,optimizer,epoch)
validate(val_loader,model,criterion)](https://image.slidesharecdn.com/0729yokota-210727025128/85/15-A-2021-27-320.jpg)
![分散並列
12_distributed.py
import os
import torch
import torch.distributed as dist
master_addr = os.getenv("MASTER_ADDR", default="localhost")
master_port = os.getenv('MASTER_PORT', default='8888')
method = "tcp://{}:{}".format(master_addr, master_port)
rank = int(os.getenv('OMPI_COMM_WORLD_RANK', '0'))
world_size = int(os.getenv('OMPI_COMM_WORLD_SIZE', '1'))
dist.init_process_group("nccl", init_method=method, rank=rank, world_size=world_size)
print('Rank: {}, Size: {}'.format(dist.get_rank(),dist.get_world_size()))
ngpus = 4
device = rank % ngpus
x = torch.randn(1).to(device)
print('rank {}: {}'.format(rank, x))
dist.broadcast(x, src=0)
print('rank {}: {}'.format(rank, x))
通信に用いるホストアドレスとポート番号を指定
OpenMPI環境変数からrankとsizeを取得
PyTorchにこれらを設定
PyTorchによる集団通信
.bashrcに以下を記入
if [ -f "$SGE_JOB_SPOOL_DIR/pe_hostfile" ]; then
export MASTER_ADDR=`head -n 1 $SGE_JOB_SPOOL_DIR/pe_hostfile | cut -d " " -f 1`
fi
mpirun -np 4 python 12_distributed.py](https://image.slidesharecdn.com/0729yokota-210727025128/85/15-A-2021-30-320.jpg)
![分散並列MNIST
13_ddp.py
def print0(message):
if torch.distributed.is_initialized():
if torch.distributed.get_rank() == 0:
print(message, flush=True)
else:
print(message, flush=True)
train_sampler = torch.utils.data.distributed.DistributedSampler(
train_dataset,
num_replicas=torch.distributed.get_world_size(),
rank=torch.distributed.get_rank())
model = DDP(model, device_ids=[rank])
.
.
.
.
.
.
全プロセスがprintすると見づらいので１プロセスだけprintするようなprint関数を定義
train dataの読み込みで異なるプロセスが異なるデータを読むようにする
モデルをDDP()に通すことで分散並列計算を行う](https://image.slidesharecdn.com/0729yokota-210727025128/85/15-A-2021-31-320.jpg)
![ProgressMeter
15_meter.py
def train(train_loader,model,criterion,optimizer,epoch,device):
batch_time = AverageMeter('Time', ':.4f')
train_loss = AverageMeter('Loss', ‘:.6f')
progress = ProgressMeter(
len(train_loader),
[train_loss, batch_time],
prefix="Epoch: [{}]".format(epoch))
class ProgressMeter(object):
def __init__(self, num_batches, meters, prefix="", postfix=""):
self.batch_fmtstr = self._get_batch_fmtstr(num_batches)
self.meters = meters
self.prefix = prefix
self.postfix = postfix
def display(self, batch):
entries = [self.prefix + self.batch_fmtstr.format(batch)]
entries += [str(meter) for meter in self.meters]
entries += self.postfix
print0('t'.join(entries))
def _get_batch_fmtstr(self, num_batches):
num_digits = len(str(num_batches // 1))
fmt = '{:' + str(num_digits) + 'd}'
return '[' + fmt + '/' + fmt.format(num_batches) + ']'
前にprintするもの
後にprintするもの
printしたい変数
printしたいものを連結
[ 今のbatch / 全batch数 ] のような表示をしたい](https://image.slidesharecdn.com/0729yokota-210727025128/85/15-A-2021-34-320.jpg)
![Weights and Biases
pip install wandb
wandb login
import wandb
os.environ['MASTER_ADDR'] = 'localhost'
os.environ['MASTER_PORT'] = '8888'
rank = int(os.getenv('OMPI_COMM_WORLD_RANK', '0'))
world_size = int(os.getenv('OMPI_COMM_WORLD_SIZE', '1'))
dist.init_process_group("nccl", rank=rank, world_size=world_size)
device = torch.device('cuda',rank)
if torch.distributed.get_rank() == 0:
wandb.init(project="example-project")
wandb.config.update(args)
epochs = args.epochs
batch_size = args.batch_size
learning_rate = args.lr * world_size
for epoch in range(epochs):
model.train()
train_loss, train_acc = train(train_loader,model,criterion,optimizer,epoch,device)
val_loss, val_acc = validate(val_loader,model,criterion,device)
if torch.distributed.get_rank() == 0:
wandb.log({
'train_loss': train_loss,
'train_acc': train_acc,
'val_loss': val_loss,
'val_acc': val_acc
})
wandbで記録したい変数
trainとvalidateがlossとaccuracyを返すようにする
wandbの初期化
argsを渡すと実験条件を勝手に記録してくれる
16_wandb.py](https://image.slidesharecdn.com/0729yokota-210727025128/85/15-A-2021-35-320.jpg)
![train_dataset = datasets.CIFAR10('./data',
train=True,
download=True,
transform=transforms.ToTensor())
val_dataset = datasets.CIFAR10('./data',
train=False,
download=True,
transform=transforms.ToTensor())
CIFAR10
17_cifar10.py
model = VGG('VGG19').to(device)
model = DDP(model, device_ids=[rank % 4])
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
データセットの変更
モデルの変更](https://image.slidesharecdn.com/0729yokota-210727025128/85/15-A-2021-36-320.jpg)
![transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
transform_val = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
データ拡張
18_augmentation.py
輝度値の正規化](https://image.slidesharecdn.com/0729yokota-210727025128/85/15-A-2021-37-320.jpg)
![Sweep
sweep.yaml
program: wrapper.py
method: grid
metric:
goal: minimize
name: val_loss
parameters:
epochs:
values: [100]
batch_size:
values: [32]
learning_rate:
values: [0.005, 0.01, 0.02, 0.05, 0.1]
momentum:
values: [0.85, 0.9, 0.95]
weight_decay:
values: [1.0e-4, 2.0e-4, 5.0e-4, 1.0e-3, 2.0e-3]
wandb sweep sweep.yaml](https://image.slidesharecdn.com/0729yokota-210727025128/85/15-A-2021-39-320.jpg)
![[DL輪読会]“Spatial Attention Point Network for Deep-learning-based Robust Autono...](https://cdn.slidesharecdn.com/ss_thumbnails/20210729kokiyamane-210730035349-thumbnail.jpg?width=640&height=640&fit=bounds)
![[DL輪読会]Objects as Points](https://cdn.slidesharecdn.com/ss_thumbnails/20190614centernetkuboshizuma-190614004246-thumbnail.jpg?width=640&height=640&fit=bounds)
![[DL輪読会]DropBlock: A regularization method for convolutional networks](https://cdn.slidesharecdn.com/ss_thumbnails/dlyokota20190222-190222002832-thumbnail.jpg?width=640&height=640&fit=bounds)
第15回 配信講義 計算科学技術特論A（2021）
- 1.
- 2. スパコンでしかできない深層学習 TPU v3 ImageNet SOTA:90.45% Top 1 Accuracy 10,000 TPUv3 core days R e s N e t - 5 0 D i s t i l B E R T E L M o B E R T - L a r g e G P T - 2 M e g a t r o n L M T u r i n g - N L G G P T - 3 S w i t c h T r a n s f o r m e r 10 7 10 8 10 9 10 10 10 11 10 12 10 13 Number of parameters x100,000 計算量上等
- 3. 転移学習・事前学習 異なるドメイン 白黒 カラー 昼 夜 イラスト実物 航空写真 地図 手書き文字 標識の文字 異なるタスク 特徴抽出器 分類器 <latexit sha1_base64="xcrXaV3a9ekZ37DSREKY8qQ5X8s=">AAACFnicbVDLSgMxFM3UV62vUXe6GSyCqzJTRV0W3bisYB/QKSWTuW1Dk8yQZIQyFPwMv8CtfoE7cevWD/A/zLSzsK0HQg7n3Jt7c4KYUaVd99sqrKyurW8UN0tb2zu7e/b+QVNFiSTQIBGLZDvAChgV0NBUM2jHEjAPGLSC0W3mtx5BKhqJBz2OocvxQNA+JVgbqWcf+YkIQQYSE0j9oYqzu+pyPpn07LJbcadwlomXkzLKUe/ZP34YkYSD0IRhpTqeG+tuiqWmhMGk5CcKzPsjPICOoQJzUN10+oeJc2qU0OlH0hyhnan6tyPFXKkxD0wlx3qoFr1M/M/rJLp/3U2piBMNgswG9RPm6MjJAnFCKoFoNjYEE0nNrg4ZYpOHNrHNTQlVtlqWi7eYwjJpViveZeX8/qJcu8kTKqJjdILOkIeuUA3doTpqIIKe0At6RW/Ws/VufVifs9KClfccojlYX7/786CX</latexit> | {z } <latexit sha1_base64="MIdm9ei+GGrZHxMeP/xC8IlDuQg=">AAACFnicbVC7TsMwFHV4lvIKsMESUSExVQkgYKxgYSwSfUhNVDnObWvVdiLbQaqiSnwGX8AKX8CGWFn5AP4Dp81AW45k+eice32vT5gwqrTrfltLyyura+uljfLm1vbOrr2331RxKgk0SMxi2Q6xAkYFNDTVDNqJBMxDBq1weJv7rUeQisbiQY8SCDjuC9qjBGsjde1DPxURyFBiApk/UEl+ey7n43HXrrhVdwJnkXgFqaAC9a7940cxSTkITRhWquO5iQ4yLDUlDMZlP1Vg3h/iPnQMFZiDCrLJH8bOiVEipxdLc4R2JurfjgxzpUY8NJUc64Ga93LxP6+T6t51kFGRpBoEmQ7qpczRsZMH4kRUAtFsZAgmkppdHTLAJg9tYpuZEql8tTwXbz6FRdI8q3qX1fP7i0rtpkiohI7QMTpFHrpCNXSH6qiBCHpCL+gVvVnP1rv1YX1OS5esoucAzcD6+gX6V6CW</latexit> | {z } 膨大で汎用なデータ で事前学習 <latexit sha1_base64="ujCWtkvHIVCkz8mFfaCwG99Ca6c=">AAACBnicbVDLSsNAFL2pr1pfVZdugkWoICVRUTdCURcuK9g20IQymUzboZNJmJkIJXTvF7jVL3Anbv0NP8D/cNJmYVsPXDiccy/3cPyYUaks69soLC2vrK4V10sbm1vbO+XdvZaMEoFJE0csEo6PJGGUk6aiihEnFgSFPiNtf3ib+e0nIiSN+KMaxcQLUZ/THsVIacm9u3ZT56RRdY7dcbdcsWrWBOYisXNSgRyNbvnHDSKchIQrzJCUHduKlZcioShmZFxyE0lihIeoTzqachQS6aWTzGPzSCuB2YuEHq7Mifr3IkWhlKPQ15shUgM572Xif14nUb0rL6U8ThThePqolzBTRWZWgBlQQbBiI00QFlRnNfEACYSVrmnmSyCzaFkv9nwLi6R1WrMvamcP55X6Td5QEQ7gEKpgwyXU4R4a0AQMMbzAK7wZz8a78WF8TlcLRn6zDzMwvn4BRWiZFQ==</latexit> D = {X, P(X)} <latexit sha1_base64="hiiuoNNYpJMJfQxrYsc4gEgEUoI=">AAACCHicbVDLSsNAFJ3UV62vqks3g0WoICVRUTdC0Y3LCn3SxDKZTNqhk0mYmQgl9gf8Arf6Be7ErX/hB/gfTtosbOuBC4dz7uUejhsxKpVpfhu5peWV1bX8emFjc2t7p7i715RhLDBp4JCFou0iSRjlpKGoYqQdCYICl5GWO7xN/dYjEZKGvK5GEXEC1OfUpxgpLT3Ur+2kc1Ird57ax/a4VyyZFXMCuEisjJRAhlqv+GN7IY4DwhVmSMquZUbKSZBQFDMyLtixJBHCQ9QnXU05Coh0kknqMTzSigf9UOjhCk7UvxcJCqQcBa7eDJAayHkvFf/zurHyr5yE8ihWhOPpIz9mUIUwrQB6VBCs2EgThAXVWSEeIIGw0kXNfPFkGi3txZpvYZE0TyvWReXs/rxUvckayoMDcAjKwAKXoAruQA00AAYCvIBX8GY8G+/Gh/E5Xc0Z2c0+mIHx9QsGepoP</latexit> T = {Y, P(Y |X)} 数字 顔 物体 少量のクラスとデータ でファインチューニング source target 転移学習 ファインチューニング 後の層ほど粗粒度の特徴を学習 教師なしでもこれらの特徴量を 学習することはできる
- 4. Papers with code MLPerftarget score: 75.9 https://paperswithcode.com ImageNet-1k 事前学習済のニューラルネットモデルが落ちている
- 5. 分散並列化 データ並列 テンソル並列 層並列 データを分散 モデルは冗長 勾配を通信 バッチが巨大化 例：Horovod データは冗長 モデルは分散 活性を通信 通信頻度が多い 例：MeshTensorFlow データは冗長 モデルは分散 活性を通信 計算が逐次的 例：GPipe “Demystifying Parallel and Distributed Deep Learning: An In-Depth Concurrency Analysis”, Ben-nun and Hoefler, ACM Computing Surveys, Article No.: 65
- 6. 深層学習における領域分割 科学技術計算 深層学習 疎な結合 3次元空間 密な結合 超高次元空間 ネットワーク + 結合なし 超高次元空間 疎行列 密テンソル メッシュ data <latexit sha1_base64="t4pFG0WMcp/4tdSKdCyIwXRV0xU=">AAACGnicbVDLSsNAFJ3UV62vqEtBBouQbkoiim6EohuXFewD2lAmk0k7dPJgZiKGkJ2f4Re41S9wJ27d+AH+h5M2gm09MHA4517umeNEjAppml9aaWl5ZXWtvF7Z2Nza3tF399oijDkmLRyykHcdJAijAWlJKhnpRpwg32Gk44yvc79zT7igYXAnk4jYPhoG1KMYSSUN9MO+j+TI8dIku/SMoTEyfoWHrFarDfSqWTcngIvEKkgVFGgO9O++G+LYJ4HEDAnRs8xI2inikmJGsko/FiRCeIyGpKdogHwi7HTyjwweK8WFXsjVCyScqH83UuQLkfiOmsxDinkvF//zerH0LuyUBlEsSYCnh7yYQRnCvBToUk6wZIkiCHOqskI8QhxhqaqbueKKPFqmerHmW1gk7ZO6dVY3b0+rjauioTI4AEfAABY4Bw1wA5qgBTB4BM/gBbxqT9qb9q59TEdLWrGzD2agff4AYkWhKg==</latexit> y =f(g(h(x))) <latexit sha1_base64="99l4eI0BvW/3fqCqOhXud5ttEeI=">AAACHXicbVDLSsNAFJ3UV62vqEs3o0VwVRJRdCMU3bizgn1AE8JkMmmHTiZhZiKUkLWf4Re41S9wJ27FD/A/nLRZ2NYDFw7n3Mu99/gJo1JZ1rdRWVpeWV2rrtc2Nre2d8zdvY6MU4FJG8csFj0fScIoJ21FFSO9RBAU+Yx0/dFN4XcfiZA05g9qnBA3QgNOQ4qR0pJnHjqUKy9z7iIyQDkMoZMMKQzgVIBX0PLMutWwJoCLxC5JHZRoeeaPE8Q4jQhXmCEp+7aVKDdDQlHMSF5zUkkShEdoQPqachQR6WaTV3J4rJUAhrHQxRWcqH8nMhRJOY583RkhNZTzXiH+5/VTFV66GeVJqgjH00VhyqCKYZELDKggWLGxJggLqm+FeIgEwkqnN7MlkMVpuc7Fnk9hkXROG/Z5w7o/qzevy4Sq4AAcgRNggwvQBLegBdoAgyfwAl7Bm/FsvBsfxue0tWKUM/tgBsbXL0tBoYI=</latexit> Z ⌦ f d⌦ = 0 <latexit sha1_base64="mvbgWU6hGLlhR+mJyQ8PA6kuQ+4=">AAACUXicbVBNa9tAEB0rbZM6/XDSYy9LTSGFYqTQ0l4KIbn00EMKdRKwjBmtVvaS3ZXYHQWM2F+Wn5FTjjnk0v6C3rqSXWiSDiz7eG+GefOySklHcXzdizYePX6yufW0v/3s+YuXg53dE1fWlosxL1VpzzJ0QkkjxiRJibPKCtSZEqfZ+VGrn14I62RpftCyElONcyMLyZECNRuM06xUuVvq8DUpLQSh/5JqpIXVDdq5lsanrtazJmfIiKFnncpRNd/8Xoezoln69+wvJv9uNhjGo7gr9hAkazCEdR3PBrdpXvJaC0NcoXOTJK5oGgyQ5Er4flo7USE/x7mYBGhQCzdtuvM9exuYnBWlDc8Q69h/JxrUrr0wdLYW3X2tJf+nTWoqPk8baaqahOGrRUWtGJWszZLl0gpOahkAciuDV8YXaJFTSPzOlty11nzIJbmfwkNwsj9KPo7i7x+GB4frhLbgNbyBPUjgExzAVziGMXC4hBv4Cb96V73fEUTRqjXqrWdewZ2Ktv8At6u2uA==</latexit> ✓ = argmin X data L(y, t) 入力 出力 ラベル 損失 パラメータ 合成関数 保存則
- 7.
- 8. 通信パターン 科学技術計算 深層学習 袖領域を通信 send, recv Local EssentialTree AlltoAllv パラメータを平均 AllReduce データ分散 テンソル分散 層分散 袖領域を通信 send, recv 活性を通信 send, recv
- 9. 深層学習における通信量 活性: 節点数 xバッチサイズ パラメータ: 辺の数 状態量= パラメータ数 <latexit sha1_base64="YVlcMILk4UwtY3+ASi6gidoAwGA=">AAACQ3icbVDLSuxAEO34uL6u11GXbhoHwYswJKLoRhBd6MKFgqPCZBgqnc5MM92d0F0RhpBP8jP8AleCLl25E7eCnXEWvgqaPpxTRZ06USaFRd+/98bGJyb/TE3PzM79nf+3UFtcurBpbhhvslSm5ioCy6XQvIkCJb/KDAcVSX4Z9Q8r/fKaGytSfY6DjLcVdLVIBAN0VKd2FCrAXpQUquwUuBGUe2EXlIJPNG6EHCHUEEmgJ+thlMrYDpT7ihB7TnIt/zu1ut/wh0V/gmAE6mRUp53aUxinLFdcI5NgbSvwM2wXYFAwycvZMLc8A9aHLm85qEFx2y6GB5d0zTExTVLjnkY6ZD9PFKBsZdF1VnfY71pF/qa1ckx224XQWY5cs49FSS4pprRKj8bCcIZy4AAwI5xXynpggKHL+MuW2FbWSpdL8D2Fn+BisxFsN/yzrfr+wSihabJCVsk6CcgO2SfH5JQ0CSM35I48kEfv1nv2XrzXj9YxbzSzTL6U9/YOLZKzhg==</latexit> mt+1 = mt + ⌘rL(✓t) <latexit sha1_base64="/jqvnaxgFI+xf3TbDhlcpGeos8o=">AAACOHicbVDLSsNAFJ3UV62vqEs3wSIIYklE0YVC0Y3LCvYBTQiTyaQdOnkwcyOUkJ/xM/wCt7pz50bErV/gpO3CPi4Mc+ace7lnjpdwJsE0P7TS0vLK6lp5vbKxubW9o+/utWScCkKbJOax6HhYUs4i2gQGnHYSQXHocdr2BneF3n6iQrI4eoRhQp0Q9yIWMIJBUa5+bXsx9+UwVFdmQ58Czt0MTqz8ZpECp3aIoe8FWagerl41a+aojHlgTUAVTarh6l+2H5M0pBEQjqXsWmYCToYFMMJpXrFTSRNMBrhHuwpGOKTSyUa/zI0jxfhGEAt1IjBG7P+JDIeysKs6C49yVivIRVo3heDKyViUpEAjMl4UpNyA2CgiM3wmKAE+VAATwZRXg/SxwARUsFNbfFlYy1Uu1mwK86B1VrMuaubDebV+O0mojA7QITpGFrpEdXSPGqiJCHpGr+gNvWsv2qf2rf2MW0vaZGYfTZX2+wdpUq/Y</latexit> ✓t+1 = ✓t mt Momentum SGD
- 10. 省メモリ local update parameters remainscattered until AllGather
- 11. 二次最適化 https://losslandscape.com SGD 重みWとバイアスbを合わせてθとする ミニバッチごとに損失関数の形状は変化する momentum SGDsemi-implicit Euler風に書くと Nesterov momentum RMSProp Adam <latexit sha1_base64="IOyR436oWLZbrKn8O0Lz+NybarM=">AAACMXicbVDLSsNAFJ34rPVVdekmWARFLInvjSC6ceGigrVCU8rNdGqHTiZh5kYoIV/iZ/gFbvUL3Ingyp9w0kaw1QvDnDnnXu6Z40eCa3ScN2ticmp6ZrYwV5xfWFxaLq2s3uowVpTVaChCdeeDZoJLVkOOgt1FikHgC1b3exeZXn9gSvNQ3mA/Ys0A7iXvcApoqFbp0MMuQ2gluOOmp/kDdz1zeRJ8AV4A2KUgkqt060febpXKTsUZlP0XuDkok7yqrdKn1w5pHDCJVIDWDdeJsJmAQk4FS4terFkEtAf3rGGghIDpZjL4XmpvGqZtd0JljkR7wP6eSCDQuh/4pjMzq8e1jPxPa8TYOWkmXEYxMkmHizqxsDG0s6zsNleMougbAFRx49WmXVBA0SQ6sqWtM2upycUdT+EvuN2ruEeV/euD8tl5nlCBrJMNskVcckzOyCWpkhqh5JE8kxfyaj1Zb9a79TFsnbDymTUyUtbXNx2yq3o=</latexit> ✓t+1 = ✓t ⌘rL(✓t) <latexit sha1_base64="EMAKBAJcozE6InSyo+X7qysKpy0=">AAACLnicbVDLSgMxFM34flt16SZYBEUoMyrqRhDduHChYFXolOFOmrbBJDMkdwplmP/wM/wCt/oFggtxJfgZZmoXvg4EDufcyz05cSqFRd9/8UZGx8YnJqemZ2bn5hcWK0vLVzbJDON1lsjE3MRguRSa11Gg5Dep4aBiya/j25PSv+5xY0WiL7Gf8qaCjhZtwQCdFFW2e1GOW0FxGHZAKaC9CLdCjhBqiCWECrDLQOZnxUaIXadHuBlVqn7NH4D+JcGQVMkQ51HlPWwlLFNcI5NgbSPwU2zmYFAwyYuZMLM8BXYLHd5wVIPitpkP/lbQdae0aDsx7mmkA/X7Rg7K2r6K3WQZ1v72SvE/r5Fh+6CZC51myDX7OtTOJMWElkXRljCcoew7AswIl5WyLhhg6Or8caVly2iF6yX43cJfcrVdC/ZqOxe71aPjYUNTZJWskQ0SkH1yRE7JOakTRu7IA3kkT9699+y9em9foyPecGeF/ID38QkKrqnh</latexit> vt+1 = vt + ⌘rL(✓t) <latexit sha1_base64="so4WvEfNBhzQ2+k0DIcQ37xIf6o=">AAACGXicbVDLSsNAFJ3UV62vqEsRgkUQxJKoqBuh6MZlBfuANoTJZNoOnUzCzE2hhK78DL/ArX6BO3Hryg/wP5y0WdjqgYFzz7mXe+f4MWcKbPvLKCwsLi2vFFdLa+sbm1vm9k5DRYkktE4iHsmWjxXlTNA6MOC0FUuKQ5/Tpj+4zfzmkErFIvEAo5i6Ie4J1mUEg5Y8c78DfQrYS+HYGV/nBZwMp4Jnlu2KPYH1lzg5KaMcNc/87gQRSUIqgHCsVNuxY3BTLIERTselTqJojMkA92hbU4FDqtx08o2xdaiVwOpGUj8B1kT9PZHiUKlR6OvOEENfzXuZ+J/XTqB75aZMxAlQQaaLugm3ILKyTKyASUqAjzTBRDJ9q0X6WGICOrmZLYHKTstyceZT+EsapxXnonJ2f16u3uQJFdEeOkBHyEGXqIruUA3VEUGP6Bm9oFfjyXgz3o2PaWvByGd20QyMzx85KqEn</latexit> ✓t+1 = ✓t vt+1 <latexit sha1_base64="4YkWyJvS7AvVbbmFVeff+OrbRbQ=">AAACMnicbVDLSsNAFJ34rO+qSzeDRaiIJVGpbgTRjQsXClaFpoab6dQOTiZh5qZQQv7Ez/AL3OoP6E7EnR/hpHbh68DA4Zx7uWdOmEhh0HWfnZHRsfGJydLU9Mzs3PxCeXHpwsSpZrzBYhnrqxAMl0LxBgqU/CrRHKJQ8svw9qjwL3tcGxGrc+wnvBXBjRIdwQCtFJTrvSDDDS/f93U3pr0AN6reZsHXfQWhBD8C7DKQ2Ule9bHLEQJcv94KyhW35g5A/xJvSCpkiNOg/O63Y5ZGXCGTYEzTcxNsZaBRMMnzaT81PAF2Cze8aamCiJtWNvhfTtes0qadWNunkA7U7xsZRMb0o9BOFnHNb68Q//OaKXb2WplQSYpcsa9DnVRSjGlRFm0LzRnKviXAtLBZKeuCBoa20h9X2qaIlttevN8t/CUXWzWvXts+26kcHA4bKpEVskqqxCO75IAck1PSIIzckQfySJ6ce+fFeXXevkZHnOHOMvkB5+MTv8+qnQ==</latexit> vt+1 = ⇢vt + (1 ⇢)rL(✓t)2 <latexit sha1_base64="jPT370zvmdQBBRtoAnxlbXWW5BM=">AAACTnicbVBNixNBFOyJX3H9inr00hiElUCYUVm9CItePHhYxewupMPwpvMmaba7Z7b7zUJo5n/5M7x68Sb6C7yJ9iQ5uLsWNBRV7/Gqq6i18pSmX5PelavXrt/o39y5dfvO3XuD+w8OfdU4iRNZ6codF+BRK4sTUqTxuHYIptB4VJy87fyjM3ReVfYTrWqcGVhYVSoJFKV88NHkgUZZ+1oswBjgJqeRKB3IIJCgDcKfOgpnm6GRwNorXdm2FRYKDcIALSXo8L7dFbSMGzk9zQfDdJyuwS+TbEuGbIuDfPBdzCvZGLQkNXg/zdKaZgEcKamx3RGNxxrkCSxwGqkFg34W1n9v+ZOozHlZufgs8bX670YA4/3KFHGyC+svep34P2/aUPlqFpStG0IrN4fKRnOqeFcknyuHkvQqEpBOxaxcLiE2R7Huc1fmvovWxl6yiy1cJofPxtne+PmHF8P9N9uG+uwRe8x2WcZesn32jh2wCZPsM/vGfrCfyZfkV/I7+bMZ7SXbnYfsHHr9v7xetzQ=</latexit> mt+1 = mt + ⌘ p vt+1 + ✏ rL(✓t) <latexit sha1_base64="4YEOH6DoswNYNGJQU7KvSQoMDRc=">AAACGXicbVDLSsNAFJ3UV62vqEsRBosgiCVRUTdC0Y3LCvYBbQiTybQdOpOEmRuhhK78DL/ArX6BO3Hryg/wP0zaLGzrgYFzz7mXe+d4keAaLOvbKCwsLi2vFFdLa+sbm1vm9k5Dh7GirE5DEaqWRzQTPGB14CBYK1KMSE+wpje4zfzmI1Oah8EDDCPmSNILeJdTAqnkmvsd6DMgbgLH9ug6L+BETgTXLFsVaww8T+yclFGOmmv+dPyQxpIFQAXRum1bETgJUcCpYKNSJ9YsInRAeqyd0oBIpp1k/I0RPkwVH3dDlb4A8Fj9O5EQqfVQemmnJNDXs14m/ue1Y+heOQkPohhYQCeLurHAEOIsE+xzxSiIYUoIVTy9FdM+UYRCmtzUFl9np2W52LMpzJPGacW+qJzdn5erN3lCRbSHDtARstElqqI7VEN1RNETekGv6M14Nt6ND+Nz0low8pldNAXj6xcqpaEe</latexit> ✓t+1 = ✓t mt+1 <latexit sha1_base64="hcu7JIK5zJuWJREk9Bj+qzd8oyg=">AAACNXicbVDLSsNAFJ34flt16SZYhIpYEhUfC0F048KFglWhKeFmOrVDZyZh5kYoId/iZ/gFbnXtwp269Rec1Cx8XRg495x7uWdOlAhu0POenaHhkdGx8YnJqemZ2bn5ysLipYlTTVmDxiLW1xEYJrhiDeQo2HWiGchIsKuod1zoV7dMGx6rC+wnrCXhRvEOp4CWCiv7Msxw3c8PgoghhL4Mcb3mb5TdWqAgEhBIwC4FkZ3mtQC7hYRrYaXq1b1BuX+BX4IqKessrLwF7ZimkimkAoxp+l6CrQw0cipYPhWkhiVAe3DDmhYqkMy0ssEXc3fVMm23E2v7FLoD9vtGBtKYvozsZGHW/NYK8j+tmWJnr5VxlaTIFP061EmFi7Fb5OW2uWYURd8CoJpbry7tggaKNtUfV9qmsJbbXPzfKfwFl5t1f6e+db5dPTwqE5ogy2SF1IhPdskhOSFnpEEouSMP5JE8OffOi/PqvH+NDjnlzhL5Uc7HJwCnq78=</latexit> mt+1 = 1mt + (1 1)rL(✓t) <latexit sha1_base64="b0A57HXrWLeK1gdAKZRl7DCDL2w=">AAACN3icbVDLSsNAFJ34rO+qSzfBIlTEklRRQYSiGxcuFKwKTQ0306kdnEzCzE2hhHyMn+EXuNWlK1eKW//ASc3C14WBc8+5l3vmBLHgGh3n2RoZHRufmCxNTc/Mzs0vlBeXLnSUKMqaNBKRugpAM8ElayJHwa5ixSAMBLsMbo9y/bLPlOaRPMdBzNoh3Eje5RTQUH55v++nuOFmB17AEPx638eNqrtZdOuehECAFwL2KIj0JKt62MslXL+u++WKU3OGZf8FbgEqpKhTv/zqdSKahEwiFaB1y3VibKegkFPBsmkv0SwGegs3rGWghJDpdjr8ZGavGaZjdyNlnkR7yH7fSCHUehAGZjK3q39rOfmf1kqwu9dOuYwTZJJ+HeomwsbIzhOzO1wximJgAFDFjVeb9kABRZPrjysdnVvLTC7u7xT+got6zd2pbZ1tVxqHRUIlskJWSZW4ZJc0yDE5JU1CyR15II/kybq3Xqw36/1rdMQqdpbJj7I+PgF9f6x3</latexit> vt+1 = 2vt + (1 2)rL(✓t)2 <latexit sha1_base64="MjuWH5G898k351kRZEFDIU570Os=">AAACHHicbVDLSsNAFJ34rPVVdelmsAi6sCQq6kYQ3bhwoWC10JRyM5naoZNJmLkRSujWz/AL3OoXuBO3gh/gfzhps9DqgYHDOfdyz5wgkcKg6346E5NT0zOzpbny/MLi0nJlZfXGxKlmvM5iGetGAIZLoXgdBUreSDSHKJD8Nuid5f7tPddGxOoa+wlvRXCnREcwQCu1KxTaeLzjKwgk+BFgl4HMLgZbPnY5Wm+7Xam6NXcI+pd4BamSApftypcfxiyNuEImwZim5ybYykCjYJIPyn5qeAKsB3e8aamCiJtWNvzJgG5aJaSdWNunkA7VnxsZRMb0o8BO5mHNuJeL/3nNFDtHrUyoJEWu2OhQJ5UUY5rXQkOhOUPZtwSYFjYrZV3QwNCW9+tKaPJoA9uLN97CX3KzW/MOantX+9WT06KhElknG2SLeOSQnJBzcknqhJEH8kSeyYvz6Lw6b877aHTCKXbWyC84H9+Vz6Jo</latexit> at = rL(✓t) <latexit sha1_base64="6uRgnqbwem00uROeNjK2GJZ+8vY=">AAACHnicbZDPSsNAEMY39f//qkcvwSIIhZKoqBeh6MWjglWhKWGy3bRLd5OwOymUkLuP4RN41SfwJl71AXwPNzUHa/1g4eObGWb2FySCa3ScT6syMzs3v7C4tLyyura+Ud3cutVxqihr0VjE6j4AzQSPWAs5CnafKAYyEOwuGFwU9bshU5rH0Q2OEtaR0It4yCmgifzq7tDPsO7mZ0Mf6+CjFyqgmccQ8szrgZSQ+9Wa03DGsqeNW5oaKXXlV7+8bkxTySKkArRuu06CnQwUcipYvuylmiVAB9BjbWMjkEx3svFfcnvPJF07jJV5Edrj9PdEBlLrkQxMpwTs67+1Ivyv1k4xPO1kPEpSZBH9WRSmwsbYLsDYXa4YRTEyBqji5lab9sHQQINvYktXF6cVXNy/FKbN7UHDPW4cXh/VmucloUWyQ3bJPnHJCWmSS3JFWoSSB/JEnsmL9Wi9Wm/W+09rxSpntsmErI9vHfqj0w==</latexit> vt+1 = vt + at ⌘ <latexit sha1_base64="+wvl9bAzEQ9hRPZ+KyDyIC4ih2s=">AAACH3icbVBLSgNBEO2Jvxh/UZduBoMgBMKMiroRgm5cRjAfSEKo6XSSJt0zQ3dNIAw5gMfwBG71BO7EbQ7gPexJZmESHzS8eq+Kqn5eKLhGx5lambX1jc2t7HZuZ3dv/yB/eFTTQaQoq9JABKrhgWaC+6yKHAVrhIqB9ASre8OHxK+PmNI88J9xHLK2hL7Pe5wCGqmTL7RwwBA6MRbdyV1aYHE0F1p9kBJMl1NyZrBXiZuSAklR6eR/Wt2ARpL5SAVo3XSdENsxKORUsEmuFWkWAh1CnzUN9UEy3Y5nn5nYZ0bp2r1AmeejPVP/TsQgtR5Lz3RKwIFe9hLxP68ZYe+2HXM/jJD5dL6oFwkbAztJxu5yxSiKsSFAFTe32nQACiia/Ba2dHVy2sTk4i6nsEpqFyX3unT5dFUo36cJZckJOSXnxCU3pEweSYVUCSUv5I28kw/r1fq0vqzveWvGSmeOyQKs6S8iWKPA</latexit> ✓t+1 = ✓t + vt+1 <latexit sha1_base64="xhnZcIhN39z2hCSemrfnC5bRMYE=">AAACMnicbVDLSsNAFJ34rPVVdekmWARBLEmV6kYounFZwT6gqWEynbRDJw9nboQS8id+hl/gVn9AdyLu/AgnaRa29TADh3Pu5d57nJAzCYbxri0sLi2vrBbWiusbm1vbpZ3dlgwiQWiTBDwQHQdLyplPm8CA004oKPYcTtvO6Dr1249USBb4dzAOac/DA5+5jGBQkl2qOXYMx2ZyabkCk9iSDwJi88RyKOD7zLGrSTKjqFcqGxUjgz5PzJyUUY6GXfq2+gGJPOoD4VjKrmmE0IuxAEY4TYpWJGmIyQgPaFdRH3tU9uLsvkQ/VEpfdwOhvg96pv7tiLEn5dhzVKWHYShnvVT8z+tG4F70YuaHEVCfTAa5Edch0NOw9D4TlAAfK4KJYGpXnQyxCgpUpFNT+jJdLc3FnE1hnrSqFbNWOb09K9ev8oQKaB8doCNkonNURzeogZqIoCf0gl7Rm/asfWif2tekdEHLe/bQFLSfX6nFqyE=</latexit> bt+1 = q 1 t+1 2 1 t+1 1 初期バイアス補正項 慣性項 慣性項+正規化 勾配分散項 慣性項 勾配分散項 <latexit sha1_base64="so4WvEfNBhzQ2+k0DIcQ37xIf6o=">AAACGXicbVDLSsNAFJ3UV62vqEsRgkUQxJKoqBuh6MZlBfuANoTJZNoOnUzCzE2hhK78DL/ArX6BO3Hryg/wP5y0WdjqgYFzz7mXe+f4MWcKbPvLKCwsLi2vFFdLa+sbm1vm9k5DRYkktE4iHsmWjxXlTNA6MOC0FUuKQ5/Tpj+4zfzmkErFIvEAo5i6Ie4J1mUEg5Y8c78DfQrYS+HYGV/nBZwMp4Jnlu2KPYH1lzg5KaMcNc/87gQRSUIqgHCsVNuxY3BTLIERTselTqJojMkA92hbU4FDqtx08o2xdaiVwOpGUj8B1kT9PZHiUKlR6OvOEENfzXuZ+J/XTqB75aZMxAlQQaaLugm3ILKyTKyASUqAjzTBRDJ9q0X6WGICOrmZLYHKTstyceZT+EsapxXnonJ2f16u3uQJFdEeOkBHyEGXqIruUA3VEUGP6Bm9oFfjyXgz3o2PaWvByGd20QyMzx85KqEn</latexit> ✓t+1 = ✓t vt+1 <latexit sha1_base64="PPIS633nLP5qR61KoXjzVGg7aSY=">AAACOXicbVBNa9tAFFylH3HdplXbYy9LTcDFxEhtSXsxmPaSQw8u1InBMuJpvbYX767E7pPBCP2a/oz8glyTU489FEKu+QNd2YbWcQYWhpn3eLOTZFJYDIJf3t6Dh48e79ee1J8+O3j+wn/56tSmuWG8z1KZmkEClkuheR8FSj7IDAeVSH6WzL9W/tmCGytS/QOXGR8pmGoxEQzQSbHfWcQFtsKyE01BKaCLGFsRR4g0JBIiBThjIItvZTPCmdNjPPo3+S72G0E7WIHuknBDGmSDXuz/icYpyxXXyCRYOwyDDEcFGBRM8rIe5ZZnwOYw5UNHNShuR8XqmyU9dMqYTlLjnka6Uv/fKEBZu1SJm6xy27teJd7nDXOcfB4VQmc5cs3Whya5pJjSqjM6FoYzlEtHgBnhslI2AwMMXbNbV8a2ila6XsK7LeyS0/ft8Lj94fvHRvfLpqEaeUPekiYJySfSJSekR/qEkZ/kglySK+/c++1dezfr0T1vs/OabMG7/QvV+65E</latexit> vt+1 = vt + ⌘rL(✓t vt) <latexit sha1_base64="9xePHLSefWqCHhmPtXC5CMrtgpA=">AAACRnicbVDBShxBEK3ZGLNqNJt49DK4BARxmdFgcgks8SKeDGRV2FmWmt4at7GnZ+yuEZZh/imfkS8I5KQXr7kFr+nZnUPUPGh49V4VVf3iXEnLQfDLa71Yern8qr2yuvZ6feNN5+27M5sVRtBAZCozFzFaUlLTgCUrusgNYRorOo+vjmr//IaMlZn+xrOcRileaplIgeykceck4ikxjkveDavPTcF7Eap8ilFiUJTpwqzKyF4bLm+acjei3EqV6SpeKONON+gFc/jPSdiQLjQ4HXfuo0kmipQ0C4XWDsMg51GJhqVQVK1GhaUcxRVe0tBRjSnZUTn/c+W/d8rETzLjnmZ/rv47UWJq7SyNXWeKPLVPvVr8nzcsOPk0KqXOCyYtFouSQvmc+XWA/kQaEqxmjqAw0t3qiym6oNjF/GjLxNan1bmET1N4Ts72e+Fh7+Drh27/S5NQG7ZgG3YghI/Qh2M4hQEI+A4/4RbuvB/eb++P97BobXnNzCY8Qgv+Avees/A=</latexit> ✓t+1 = ✓t ↵ mt+1 p vt+1 + ✏ bt+1 <latexit sha1_base64="F2TbglfUftVNHTGJ+8kLA8s1+4w=">AAACHXicbZDNSsNAFIUn9a/Wv6pLN9EiCIWSqKjLohuXFWwtNCFMppN26GQSZ24KJWTtY/gEbvUJ3Ilb8QF8DydtF7b1wMDh3Hu5dz4/5kyBZX0bhaXlldW14nppY3Nre6e8u9dSUSIJbZKIR7LtY0U5E7QJDDhtx5Li0Of0wR/c5PWHIZWKReIeRjF1Q9wTLGAEg4688qETSExSO0sd9SghHXopVO2s6tBYMR6JLPPKFatmjWUuGntqKmiqhlf+cboRSUIqgHCsVMe2YnBTLIERTrOSkygaYzLAPdrRVuCQKjcdfyUzj3XSNYNI6ifAHKd/J1IcKjUKfd0ZYuir+Voe/lfrJBBcuSkTcQJUkMmiIOEmRGbOxewySQnwkTaYSKZvNUkfazag6c1s6ar8tJyLPU9h0bROa/ZF7ezuvFK/nhIqogN0hE6QjS5RHd2iBmoigp7QC3pFb8az8W58GJ+T1oIxndlHMzK+fgHm46O/</latexit> 1 p vt+1 + ✏ <latexit sha1_base64="Nn6v29FOHJIjbc58kcgKfC/AV+Y=">AAACFXicbVDLSsNAFJ34rPUVdSVuBotQKZZERV0WBdFdBfuANobJZNIOnUzCzEQoIfgZfoFb/QJ34ta1H+B/mLRZ2NYDFw7n3Mu99zgho1IZxrc2N7+wuLRcWCmurq1vbOpb200ZRAKTBg5YINoOkoRRThqKKkbaoSDIdxhpOYOrzG89EiFpwO/VMCSWj3qcehQjlUq2vlu+tmNVMZNKl4SSsoDD28OH+MhMbL1kVI0R4Cwxc1ICOeq2/tN1Axz5hCvMkJQd0wiVFSOhKGYkKXYjSUKEB6hHOinlyCfSikcvJPAgVVzoBSItruBI/TsRI1/Koe+knT5SfTntZeJ/XidS3oUVUx5GinA8XuRFDKoAZnlAlwqCFRumBGFB01sh7iOBsEpTm9jiyuy0LBdzOoVZ0jyummfVk7vTUu0yT6gA9sA+KAMTnIMauAF10AAYPIEX8AretGftXfvQPsetc1o+swMmoH39AmRHnnY=</latexit> (Ft+1 + ✏I) 1 <latexit sha1_base64="mk8+CmGkDwKAKrj130pZ19sxy/w=">AAACF3icbVDLSsNAFJ34rPUVddnNYBEqxZpUUZdFQXRXwT6gjWEymbZDJ5MwMxFKyMLP8Avc6he4E7cu/QD/w6TNwrYeuHA4517uvccJGJXKML61hcWl5ZXV3Fp+fWNza1vf2W1KPxSYNLDPfNF2kCSMctJQVDHSDgRBnsNIyxlepX7rkQhJfX6vRgGxPNTntEcxUolk64XStR2pshmXuySQlPkc3h4+REfmcTW29aJRMcaA88TMSBFkqNv6T9f1cegRrjBDUnZMI1BWhISimJE43w0lCRAeoj7pJJQjj0grGj8Rw4NEcWHPF0lxBcfq34kIeVKOPCfp9JAayFkvFf/zOqHqXVgR5UGoCMeTRb2QQeXDNBHoUkGwYqOEICxocivEAyQQVkluU1tcmZ6W5mLOpjBPmtWKeVY5uTst1i6zhHKgAPZBCZjgHNTADaiDBsDgCbyAV/CmPWvv2of2OWld0LKZPTAF7esXWWme6w==</latexit> (Ft+1 + ✏I) 1/2 対角近似 Fisher行列 ＋正則化 自然勾配法 二次最適化と 一次最適化の 中間
- 12. 0.9 0.1 0 Labradoodle Fried chicken 1 <latexit sha1_base64="Qrf0MYIwlAOrUIJFBxNweXaH96A=">AAAB/3icbVDLSsNAFL2pr1pfVZduBovgqiSi6EYounFZwbSFNpTJZNIOnUzCzEQsoQu/wK1+gTtx66f4Af6HkzYL23pg4HDOvdwzx084U9q2v63Syura+kZ5s7K1vbO7V90/aKk4lYS6JOax7PhYUc4EdTXTnHYSSXHkc9r2R7e5336kUrFYPOhxQr0IDwQLGcHaSO7T9bBv96s1u25PgZaJU5AaFGj2qz+9ICZpRIUmHCvVdexEexmWmhFOJ5VeqmiCyQgPaNdQgSOqvGwadoJOjBKgMJbmCY2m6t+NDEdKjSPfTEZYD9Wil4v/ed1Uh1dexkSSairI7FCYcqRjlP8cBUxSovnYEEwkM1kRGWKJiTb9zF0JVB5tYnpxFltYJq2zunNRt+/Pa42boqEyHMExnIIDl9CAO2iCCwQYvMArvFnP1rv1YX3ORktWsXMIc7C+fgH/kJbJ</latexit> x= h0 <latexit sha1_base64="oeS8g7Am64cZNl7f2teu7TnWjwI=">AAACBHicdVDLSgMxFM3UV62vqks3wSK4GjKl1XYhFN24rGAf2A5DJpO2oZnMkGSEUrr1C9zqF7gTt/6HH+B/mGlHsKIHLhzOuZd77/FjzpRG6MPKrayurW/kNwtb2zu7e8X9g7aKEkloi0Q8kl0fK8qZoC3NNKfdWFIc+px2/PFV6nfuqVQsErd6ElM3xEPBBoxgbaS7xEMXHQ+NPOQVS8iu16r1Sg0iG82RkvJZvepAJ1NKIEPTK372g4gkIRWacKxUz0GxdqdYakY4nRX6iaIxJmM8pD1DBQ6pcqfzi2fwxCgBHETSlNBwrv6cmOJQqUnom84Q65H67aXiX14v0YOaO2UiTjQVZLFokHCoI5i+DwMmKdF8YggmkplbIRlhiYk2IS1tCVR62szk8v08/J+0y7ZTtdFNpdS4zBLKgyNwDE6BA85BA1yDJmgBAgR4BE/g2XqwXqxX623RmrOymUOwBOv9C3UxmK8=</latexit> u0 = W0h0 <latexit sha1_base64="i98NF53nvMx1GTvIOlT02vBIGAA=">AAACBHicdVDLSsNAFL2pr1pfVZdugkVwFRKt2o1QdOOygn1gG8JkMmmHTiZhZiKU0K1f4Fa/wJ249T/8AP/DSVuhFT0wcDjnXu6Z4yeMSmXbn0ZhaXllda24XtrY3NreKe/utWScCkyaOGax6PhIEkY5aSqqGOkkgqDIZ6TtD69zv/1AhKQxv1OjhLgR6nMaUoyUlu5Tz7lse87Ac7xyxbHsCUzbOq9W7dOaJjPlx6rADA2v/NULYpxGhCvMkJRdx06UmyGhKGZkXOqlkiQID1GfdDXlKCLSzSaJx+aRVgIzjIV+XJkTdX4jQ5GUo8jXkxFSA/nby8W/vG6qwpqbUZ6kinA8PRSmzFSxmX/fDKggWLGRJggLqrOaeIAEwkqXtHAlkHm08Xwv/5PWieWcWfZttVK/mjVUhAM4hGNw4ALqcAMNaAIGDk/wDC/Go/FqvBnv09GCMdvZhwUYH988fpiK</latexit> u1 = W1h1 <latexit sha1_base64="4dcnyKt/ee7kGXZ6S3uBkWEAkPc=">AAACHXicdVDLSsNAFJ3UV62vqEs3o0VwVRKt2mXRjcsK9gFNCJPJpB06mYSZiVBC136GX+BWv8CduBU/wP9w0ka0ohcGzj3nvub4CaNSWda7UVpYXFpeKa9W1tY3NrfM7Z2OjFOBSRvHLBY9H0nCKCdtRRUjvUQQFPmMdP3RZa53b4mQNOY3apwQN0IDTkOKkdKUZ+47oUA4cxIkFEUMpp49+c6GOvPMql2zpgGt2lm9bp00NCiYL6kKimh55ocTxDiNCFeYISn7tpUoN8tHYkYmFSeVJEF4hAakryFHEZFuNv3KBB5qJoBhLPTjCk7Znx0ZiqQcR76ujJAayt9aTv6l9VMVNtyM8iRVhOPZojBlUMUw9wUGVBCs2FgDhAXVt0I8RNobpd2b2xLI/LQ5X/4HneOafVqzruvV5kXhUBnsgQNwBGxwDprgCrRAG2BwBx7AI3gy7o1n48V4nZWWjKJnF8yF8fYJDnqjNw==</latexit> @u1 @h1 <latexit sha1_base64="rQepUHmc6aWrxxB1wV5j6ZVGC6k=">AAACHXicdVDLSsNAFJ3UV62vqEs3o0VwVRKt2mXRjcsK9gFNCJPJpB06mYSZiVBC136GX+BWv8CduBU/wP9w0ka0ohcGzj3nvub4CaNSWda7UVpYXFpeKa9W1tY3NrfM7Z2OjFOBSRvHLBY9H0nCKCdtRRUjvUQQFPmMdP3RZa53b4mQNOY3apwQN0IDTkOKkdKUZ+47oUA4cxIkFEUMpp49+c66OvPMql2zpgGt2lm9bp00NCiYL6kKimh55ocTxDiNCFeYISn7tpUoN8tHYkYmFSeVJEF4hAakryFHEZFuNv3KBB5qJoBhLPTjCk7Znx0ZiqQcR76ujJAayt9aTv6l9VMVNtyM8iRVhOPZojBlUMUw9wUGVBCs2FgDhAXVt0I8RNobpd2b2xLI/LQ5X/4HneOafVqzruvV5kXhUBnsgQNwBGxwDprgCrRAG2BwBx7AI3gy7o1n48V4nZWWjKJnF8yF8fYJ8zGjJg==</latexit> @u1 @W1 <latexit sha1_base64="60kCDCJfdCUFlI7azaDnN8WmG14=">AAACHXicdVDLSsNAFJ3UV62vqEs3o0VwVRJRbHdFNy4r2Ae0IUwmk3boZBJmJkIJWfsZfoFb/QJ34lb8AP/DSRuxFT0wcObc9/FiRqWyrA+jtLS8srpWXq9sbG5t75i7ex0ZJQKTNo5YJHoekoRRTtqKKkZ6sSAo9BjpeuOrPN69I0LSiN+qSUycEA05DShGSkuueTgIBMLpIEZCUcTgyLWzn1/iWplrVq2aNQWcI41G3a43oF0oVVCg5ZqfAz/CSUi4wgxJ2betWDlp3hIzklUGiSQxwmM0JH1NOQqJdNLpKRk81ooPg0joxxWcqvMVKQqlnISezgyRGsnfsVz8K9ZPVFB3UsrjRBGOZ4OChEEVwdwX6FNBsGITTRAWVO8K8Qhpb5R2b2GKL/PVcl++j4f/k85pzT6vWTdn1eZl4VAZHIAjcAJscAGa4Bq0QBtgcA8ewRN4Nh6MF+PVeJulloyiZh8swHj/AiX8o0g=</latexit> @h1 @u0 <latexit sha1_base64="2g++4FK2qtbTNVizFSiWmGPzmRk=">AAACHXicdVDLSsNAFJ34rPUVdelmtAiuQlJabXdFNy4r2Ac0IUwmk3bo5MHMRCihaz/DL3CrX+BO3Iof4H84aSNa0QMD555779x7j5cwKqRpvmtLyyura+uljfLm1vbOrr633xVxyjHp4JjFvO8hQRiNSEdSyUg/4QSFHiM9b3yZ53u3hAsaRzdykhAnRMOIBhQjqSRXP7IDjnBmJ4hLihhMXXP6HfVU5OoV02g26s1aA5qGOUNOqmfNugWtQqmAAm1X/7D9GKchiSRmSIiBZSbSyfIvMSPTsp0KkiA8RkMyUDRCIRFONjtlCk+U4sMg5upFEs7Unx0ZCoWYhJ6qDJEcid+5XPwrN0hl0HAyGiWpJBGeDwpSBmUMc1+gTznBkk0UQZhTtSvEI6S8kcq9hSm+yFfLffk6Hv5PulXDqhvmda3SuigcKoFDcAxOgQXOQQtcgTboAAzuwAN4BE/avfasvWiv89Ilreg5AAvQ3j4BLYSjTA==</latexit> @u0 @W0 <latexit sha1_base64="+jqY1jG3/sRBUYetLzlPwiYD6Ak=">AAACBnicdVDLSsNAFL3xWeur6tLNYBHqpiQq2i6EohuXFewD2hAmk0k7dPJgZiKU0L1f4Fa/wJ249Tf8AP/DSRuhFT0wcDjnXu6Z48acSWWan8bS8srq2npho7i5tb2zW9rbb8soEYS2SMQj0XWxpJyFtKWY4rQbC4oDl9OOO7rJ/M4DFZJF4b0ax9QO8CBkPiNYaak/dKwr3zEriWOeOKWyWTWnQHOkXq9ZtTqycqUMOZpO6avvRSQJaKgIx1L2LDNWdoqFYoTTSbGfSBpjMsID2tM0xAGVdjrNPEHHWvGQHwn9QoWm6vxGigMpx4GrJwOshvK3l4l/eb1E+TU7ZWGcKBqS2SE/4UhFKCsAeUxQovhYE0wE01kRGWKBidI1LVzxZBZtonv5+Tz6n7RPq9ZF9ezuvNy4zhsqwCEcQQUsuIQG3EITWkAghid4hhfj0Xg13oz32eiSke8cwAKMj289EpkS</latexit> h1 = f0(u0) <latexit sha1_base64="YsNX+xavKPnYRsukvqtfuULWxoM=">AAACBHicbVDLSsNAFJ3UV62vqks3g0Wom5C0anUhFN24rGAf2IYwmUzaoZNJmJkIpXTrF7jVL3Anbv0PP8D/cNIGsdUDA4dz7uWeOV7MqFSW9WnklpZXVtfy64WNza3tneLuXktGicCkiSMWiY6HJGGUk6aiipFOLAgKPUba3vA69dsPREga8Ts1iokToj6nAcVIaek+vgxcu5y49rFbLFmmNQW0zNNaxbqowh/FzkgJZGi4xa+eH+EkJFxhhqTs2lasnDESimJGJoVeIkmM8BD1SVdTjkIinfE08QQeacWHQST04wpO1d8bYxRKOQo9PRkiNZCLXir+53UTFZw7Y8rjRBGOZ4eChEEVwfT70KeCYMVGmiAsqM4K8QAJhJUuae6KL9NoE92LvdjCX9KqmPaZWb09KdWvsoby4AAcgjKwQQ3UwQ1ogCbAgIMn8AxejEfj1Xgz3mejOSPb2QdzMD6+Af4MmGY=</latexit> p = f1(u1) 深層ニューラルネットの学習 2 2 10 2 5 5 5 5 15 <latexit sha1_base64="eGta8ATILI7rVM5edzp7/uMnFog=">AAAB+3icbVDLSsNAFJ3UV62vqks3g0VwVRIVdVl047IF+4A2lMnkph06mYSZiRBCvsCtfoE7cevH+AH+h5M2C1s9MHA4517umePFnClt219WZW19Y3Orul3b2d3bP6gfHvVUlEgKXRrxSA48ooAzAV3NNIdBLIGEHoe+N7sv/P4TSMUi8ajTGNyQTAQLGCXaSJ10XG/YTXsO/Jc4JWmgEu1x/XvkRzQJQWjKiVJDx461mxGpGeWQ10aJgpjQGZnA0FBBQlBuNg+a4zOj+DiIpHlC47n6eyMjoVJp6JnJkOipWvUK8T9vmOjg1s2YiBMNgi4OBQnHOsLFr7HPJFDNU0MIlcxkxXRKJKHadLN0xVdFtNz04qy28Jf0LprOdfOyc9Vo3ZUNVdEJOkXnyEE3qIUeUBt1EUWAntELerVy6816tz4WoxWr3DlGS7A+fwCA+ZVy</latexit> y <latexit sha1_base64="CX39qY1yvYuKVy5jRO31RUVPsKU=">AAACG3icbVDLSsNAFJ34rPUVdenCwSK4CkmrVndFNy4r2Ac0IUwmk3bo5MHMRCihSz/DL3CrX+BO3LrwA/wPJ21QWz0wcDjnvuZ4CaNCmuaHtrC4tLyyWlorr29sbm3rO7ttEacckxaOWcy7HhKE0Yi0JJWMdBNOUOgx0vGGV7nfuSNc0Di6laOEOCHqRzSgGEklufqBHXCEMztBXFLEYDL+4alrjV29YhrmBNA0TutV86IGvxWrIBVQoOnqn7Yf4zQkkcQMCdGzzEQ6WT4SMzIu26kgCcJD1Cc9RSMUEuFkk4+M4ZFSfBjEXL1Iwon6uyNDoRCj0FOVIZIDMe/l4n9eL5XBuZPRKEklifB0UZAyKGOYpwJ9ygmWbKQIwpyqWyEeIJWMVNnNbPFFflqeizWfwl/SrhrWmVG7Oak0LouESmAfHIJjYIE6aIBr0AQtgME9eARP4Fl70F60V+1tWrqgFT17YAba+xfXKqKf</latexit> @p @u1 <latexit sha1_base64="U0Wku2zBzTyNreP8fA04lNpnsb8=">AAACOnicbVC7SgNBFJ31GeNr1dJmMAg2CbsqaqMEbSwsIpgHZGO4OztJhsw+mJkVwrJ/42f4Bbba2AoWYusHOJukyMMDA4dz7p1773EjzqSyrA9jYXFpeWU1t5Zf39jc2jZ3dmsyjAWhVRLyUDRckJSzgFYVU5w2IkHBdzmtu/2bzK8/USFZGDyoQURbPnQD1mEElJba5pXjg+oR4MldeunI2H9MPFCQjijhIGVaHDg87OJo0i+OpLZZsErWEHie2GNSQGNU2uaX44Uk9mmghp83bStSrQSEYoTTNO/EkkZA+tClTU0D8KlsJcM7U3yoFQ93QqFfoPBQnexIwJdy4Lu6MrtKznqZ+J/XjFXnopWwIIoVDchoUCfmWIU4Cw17TFCi+EATIILpXTHpgQCidLRTUzyZrZbqXOzZFOZJ7bhkn5VO7k8L5etxQjm0jw7QEbLROSqjW1RBVUTQM3pFb+jdeDE+jW/jZ1S6YIx79tAUjN8/EDGwDg==</latexit> L = data X class X y log p = data X log p <latexit sha1_base64="NXhC3ff4B32CgQ5BYZuIAyqz5Qg=">AAACJHicbVDLSsNAFJ3UV62vqEs3g0XoqiQq6kYounHhooJ9QBPKZDJph04mYWYilJBf8DP8Arf6Be7EhRt3/oeTNqBtPTBwOPd15ngxo1JZ1qdRWlpeWV0rr1c2Nre2d8zdvbaMEoFJC0csEl0PScIoJy1FFSPdWBAUeox0vNF1Xu88ECFpxO/VOCZuiAacBhQjpaW+Wbt0AoFw6sRIKIoYdEKkhhix9DbLftVO1jerVt2aAC4SuyBVUKDZN78dP8JJSLjCDEnZs61YuWm+EDOSVZxEkhjhERqQnqYchUS66eRHGTzSig+DSOjHFZyofydSFEo5Dj3dmfuV87Vc/K/WS1Rw4aaUx4kiHE8PBQmDKoJ5PNCngmDFxpogLKj2CvEQ6YSUDnHmii9za3ku9nwKi6R9XLfP6id3p9XGVZFQGRyAQ1ADNjgHDXADmqAFMHgEz+AFvBpPxpvxbnxMW0tGMbMPZmB8/QAvl6Z4</latexit> = @L @W <latexit sha1_base64="RlrtYxiGwNDm/OSOojM6YjHJMWs=">AAACI3icbVC7TsMwFHV4lvIKMLJYVAimKgEEjBUsDAxFog+piSrHdVqrjmPZDlIV5RP4DL6AFb6ADbEwMPIfOG0kaMuRLB2d+zo+gWBUacf5tBYWl5ZXVktr5fWNza1te2e3qeJEYtLAMYtlO0CKMMpJQ1PNSFtIgqKAkVYwvM7rrQciFY35vR4J4keoz2lIMdJG6tpHXigRTj2BpKaIQS9CeoARS2+z7FcVWdeuOFVnDDhP3IJUQIF61/72ejFOIsI1ZkipjusI7af5QsxIVvYSRQTCQ9QnHUM5iojy0/GHMnholB4MY2ke13Cs/p1IUaTUKApMZ+5XzdZy8b9aJ9HhpZ9SLhJNOJ4cChMGdQzzdGCPSoI1GxmCsKTGK8QDZBLSJsOpKz2VW8tzcWdTmCfNk6p7Xj29O6vUroqESmAfHIBj4IILUAM3oA4aAINH8AxewKv1ZL1Z79bHpHXBKmb2wBSsrx/HuKZK</latexit> @L @p <latexit sha1_base64="/J5Xk+dXiOlf6omGGiJXLYgOMI8=">AAACBXicdVDLSsNAFJ34rPVVdelmsAiuQlLb2u6KblxWsA9IY5lMJu3QmUmYmQgldO0XuNUvcCdu/Q4/wP8waSNY0QMXDufcy733eBGjSlvWh7Gyura+sVnYKm7v7O7tlw4OuyqMJSYdHLJQ9j2kCKOCdDTVjPQjSRD3GOl5k6vM790TqWgobvU0Ii5HI0EDipFOJWegYn6X+Eij2bBUtsxmo9asNqBlWnNkpFJv1mxo50oZ5GgPS58DP8QxJ0JjhpRybCvSboKkppiRWXEQKxIhPEEj4qRUIE6Um8xPnsHTVPFhEMq0hIZz9edEgrhSU+6lnRzpsfrtZeJfnhProOEmVESxJgIvFgUxgzqE2f/Qp5JgzaYpQVjS9FaIx0girNOUlrb4Kjsty+X7efg/6VZMu26e31TLrcs8oQI4BifgDNjgArTANWiDDsAgBI/gCTwbD8aL8Wq8LVpXjHzmCCzBeP8CCxKaQA==</latexit> data X <latexit sha1_base64="OIzM9hBXAwa2CsqFH+Q4ck6rUCg=">AAACBXicdVDLSgMxFM3UV62vqks3wSK4Gma01C6LblxWsA+YjiWTybShmWRIMkIZuvYL3OoXuBO3focf4H+YaUewogcCh3Pu5Z6cIGFUacf5sEorq2vrG+XNytb2zu5edf+gq0QqMelgwYTsB0gRRjnpaKoZ6SeSoDhgpBdMrnK/d0+kooLf6mlC/BiNOI0oRtpI3kCl8V0WIo1mw2rNtZ05oGM36nXnvGlIoXxbNVCgPax+DkKB05hwjRlSynOdRPsZkppiRmaVQapIgvAEjYhnKEcxUX42jzyDJ0YJYSSkeVzDufpzI0OxUtM4MJMx0mP128vFvzwv1VHTzyhPUk04XhyKUga1gPn/YUglwZpNDUFYUpMV4jGSCGvT0tKVUOXRlnr5n3TPbLdhn9/Ua63LoqEyOALH4BS44AK0wDVogw7AQIBH8ASerQfrxXq13hajJavYOQRLsN6/AM2Bmhg=</latexit> data X 2 2 1 四則演算や初等関数の微分は内部で定義されている それらを連鎖させれば行列積で勾配が計算できる 後ろからかければ全て行列ベクトル積になる 画像ごとにこれが行われ最後に和をとる <latexit sha1_base64="60kCDCJfdCUFlI7azaDnN8WmG14=">AAACHXicdVDLSsNAFJ3UV62vqEs3o0VwVRJRbHdFNy4r2Ae0IUwmk3boZBJmJkIJWfsZfoFb/QJ34lb8AP/DSRuxFT0wcObc9/FiRqWyrA+jtLS8srpWXq9sbG5t75i7ex0ZJQKTNo5YJHoekoRRTtqKKkZ6sSAo9BjpeuOrPN69I0LSiN+qSUycEA05DShGSkuueTgIBMLpIEZCUcTgyLWzn1/iWplrVq2aNQWcI41G3a43oF0oVVCg5ZqfAz/CSUi4wgxJ2betWDlp3hIzklUGiSQxwmM0JH1NOQqJdNLpKRk81ooPg0joxxWcqvMVKQqlnISezgyRGsnfsVz8K9ZPVFB3UsrjRBGOZ4OChEEVwdwX6FNBsGITTRAWVO8K8Qhpb5R2b2GKL/PVcl++j4f/k85pzT6vWTdn1eZl4VAZHIAjcAJscAGa4Bq0QBtgcA8ewRN4Nh6MF+PVeJulloyiZh8swHj/AiX8o0g=</latexit> @h1 @u0 <latexit sha1_base64="4dcnyKt/ee7kGXZ6S3uBkWEAkPc=">AAACHXicdVDLSsNAFJ3UV62vqEs3o0VwVRKt2mXRjcsK9gFNCJPJpB06mYSZiVBC136GX+BWv8CduBU/wP9w0ka0ohcGzj3nvub4CaNSWda7UVpYXFpeKa9W1tY3NrfM7Z2OjFOBSRvHLBY9H0nCKCdtRRUjvUQQFPmMdP3RZa53b4mQNOY3apwQN0IDTkOKkdKUZ+47oUA4cxIkFEUMpp49+c6GOvPMql2zpgGt2lm9bp00NCiYL6kKimh55ocTxDiNCFeYISn7tpUoN8tHYkYmFSeVJEF4hAakryFHEZFuNv3KBB5qJoBhLPTjCk7Znx0ZiqQcR76ujJAayt9aTv6l9VMVNtyM8iRVhOPZojBlUMUw9wUGVBCs2FgDhAXVt0I8RNobpd2b2xLI/LQ5X/4HneOafVqzruvV5kXhUBnsgQNwBGxwDprgCrRAG2BwBx7AI3gy7o1n48V4nZWWjKJnF8yF8fYJDnqjNw==</latexit> @u1 @h1 <latexit sha1_base64="CX39qY1yvYuKVy5jRO31RUVPsKU=">AAACG3icbVDLSsNAFJ34rPUVdenCwSK4CkmrVndFNy4r2Ac0IUwmk3bo5MHMRCihSz/DL3CrX+BO3LrwA/wPJ21QWz0wcDjnvuZ4CaNCmuaHtrC4tLyyWlorr29sbm3rO7ttEacckxaOWcy7HhKE0Yi0JJWMdBNOUOgx0vGGV7nfuSNc0Di6laOEOCHqRzSgGEklufqBHXCEMztBXFLEYDL+4alrjV29YhrmBNA0TutV86IGvxWrIBVQoOnqn7Yf4zQkkcQMCdGzzEQ6WT4SMzIu26kgCcJD1Cc9RSMUEuFkk4+M4ZFSfBjEXL1Iwon6uyNDoRCj0FOVIZIDMe/l4n9eL5XBuZPRKEklifB0UZAyKGOYpwJ9ygmWbKQIwpyqWyEeIJWMVNnNbPFFflqeizWfwl/SrhrWmVG7Oak0LouESmAfHIJjYIE6aIBr0AQtgME9eARP4Fl70F60V+1tWrqgFT17YAba+xfXKqKf</latexit> @p @u1 <latexit sha1_base64="rQepUHmc6aWrxxB1wV5j6ZVGC6k=">AAACHXicdVDLSsNAFJ3UV62vqEs3o0VwVRKt2mXRjcsK9gFNCJPJpB06mYSZiVBC136GX+BWv8CduBU/wP9w0ka0ohcGzj3nvub4CaNSWda7UVpYXFpeKa9W1tY3NrfM7Z2OjFOBSRvHLBY9H0nCKCdtRRUjvUQQFPmMdP3RZa53b4mQNOY3apwQN0IDTkOKkdKUZ+47oUA4cxIkFEUMpp49+c66OvPMql2zpgGt2lm9bp00NCiYL6kKimh55ocTxDiNCFeYISn7tpUoN8tHYkYmFSeVJEF4hAakryFHEZFuNv3KBB5qJoBhLPTjCk7Znx0ZiqQcR76ujJAayt9aTv6l9VMVNtyM8iRVhOPZojBlUMUw9wUGVBCs2FgDhAXVt0I8RNobpd2b2xLI/LQ5X/4HneOafVqzruvV5kXhUBnsgQNwBGxwDprgCrRAG2BwBx7AI3gy7o1n48V4nZWWjKJnF8yF8fYJ8zGjJg==</latexit> @u1 @W1 <latexit sha1_base64="2g++4FK2qtbTNVizFSiWmGPzmRk=">AAACHXicdVDLSsNAFJ34rPUVdelmtAiuQlJabXdFNy4r2Ac0IUwmk3bo5MHMRCihaz/DL3CrX+BO3Iof4H84aSNa0QMD555779x7j5cwKqRpvmtLyyura+uljfLm1vbOrr633xVxyjHp4JjFvO8hQRiNSEdSyUg/4QSFHiM9b3yZ53u3hAsaRzdykhAnRMOIBhQjqSRXP7IDjnBmJ4hLihhMXXP6HfVU5OoV02g26s1aA5qGOUNOqmfNugWtQqmAAm1X/7D9GKchiSRmSIiBZSbSyfIvMSPTsp0KkiA8RkMyUDRCIRFONjtlCk+U4sMg5upFEs7Unx0ZCoWYhJ6qDJEcid+5XPwrN0hl0HAyGiWpJBGeDwpSBmUMc1+gTznBkk0UQZhTtSvEI6S8kcq9hSm+yFfLffk6Hv5PulXDqhvmda3SuigcKoFDcAxOgQXOQQtcgTboAAzuwAN4BE/avfasvWiv89Ilreg5AAvQ3j4BLYSjTA==</latexit> @u0 @W0 <latexit sha1_base64="RlrtYxiGwNDm/OSOojM6YjHJMWs=">AAACI3icbVC7TsMwFHV4lvIKMLJYVAimKgEEjBUsDAxFog+piSrHdVqrjmPZDlIV5RP4DL6AFb6ADbEwMPIfOG0kaMuRLB2d+zo+gWBUacf5tBYWl5ZXVktr5fWNza1te2e3qeJEYtLAMYtlO0CKMMpJQ1PNSFtIgqKAkVYwvM7rrQciFY35vR4J4keoz2lIMdJG6tpHXigRTj2BpKaIQS9CeoARS2+z7FcVWdeuOFVnDDhP3IJUQIF61/72ejFOIsI1ZkipjusI7af5QsxIVvYSRQTCQ9QnHUM5iojy0/GHMnholB4MY2ke13Cs/p1IUaTUKApMZ+5XzdZy8b9aJ9HhpZ9SLhJNOJ4cChMGdQzzdGCPSoI1GxmCsKTGK8QDZBLSJsOpKz2VW8tzcWdTmCfNk6p7Xj29O6vUroqESmAfHIBj4IILUAM3oA4aAINH8AxewKv1ZL1Z79bHpHXBKmb2wBSsrx/HuKZK</latexit> @L @p Forward propagation Backward propagation Cross entropy loss 確率的勾配降下法 (SGD) ：ラベル 誤差逆伝播法 <latexit sha1_base64="BpaiO7b9hbl/rfkDJL7cM+CMhk8=">AAACNHicbVDLSsNAFJ34rO+qSzeDRRDEkqhoN0LRjQsXFawtNCXcTCc6dPJg5kYoIb/iZ/gFbnUvuJNu/QYntYhVDwwczn2dOX4ihUbbfrWmpmdm5+ZLC4tLyyura+X1jRsdp4rxJotlrNo+aC5FxJsoUPJ2ojiEvuQtv39e1Fv3XGkRR9c4SHg3hNtIBIIBGskr11pehntOftrycN/lCG6ggGVuAgoFSDcEvGMgs8s8/xap6c29csWu2iPQv8QZkwoZo+GVh24vZmnII2QStO44doLdrFjJJM8X3VTzBFgfbnnH0AhCrrvZ6Ic53TFKjwaxMi9COlJ/TmQQaj0IfdNZONa/a4X4X62TYlDrZiJKUuQR+zoUpJJiTIu4aE8ozlAODAGmhPFK2R2YiNCEOnGlpwtrRS7O7xT+kpuDqnNcPbw6qtTPxgmVyBbZJrvEISekTi5IgzQJIw/kiTyTF+vRerPereFX65Q1ntkkE7A+PgFacq02</latexit> Wt+1 = Wt ⌘ @L @Wt
- 13. 0.9 0.1 0 Labradoodle Fried chicken 1 <latexit sha1_base64="Qrf0MYIwlAOrUIJFBxNweXaH96A=">AAAB/3icbVDLSsNAFL2pr1pfVZduBovgqiSi6EYounFZwbSFNpTJZNIOnUzCzEQsoQu/wK1+gTtx66f4Af6HkzYL23pg4HDOvdwzx084U9q2v63Syura+kZ5s7K1vbO7V90/aKk4lYS6JOax7PhYUc4EdTXTnHYSSXHkc9r2R7e5336kUrFYPOhxQr0IDwQLGcHaSO7T9bBv96s1u25PgZaJU5AaFGj2qz+9ICZpRIUmHCvVdexEexmWmhFOJ5VeqmiCyQgPaNdQgSOqvGwadoJOjBKgMJbmCY2m6t+NDEdKjSPfTEZYD9Wil4v/ed1Uh1dexkSSairI7FCYcqRjlP8cBUxSovnYEEwkM1kRGWKJiTb9zF0JVB5tYnpxFltYJq2zunNRt+/Pa42boqEyHMExnIIDl9CAO2iCCwQYvMArvFnP1rv1YX3ORktWsXMIc7C+fgH/kJbJ</latexit> x= h0 <latexit sha1_base64="oeS8g7Am64cZNl7f2teu7TnWjwI=">AAACBHicdVDLSgMxFM3UV62vqks3wSK4GjKl1XYhFN24rGAf2A5DJpO2oZnMkGSEUrr1C9zqF7gTt/6HH+B/mGlHsKIHLhzOuZd77/FjzpRG6MPKrayurW/kNwtb2zu7e8X9g7aKEkloi0Q8kl0fK8qZoC3NNKfdWFIc+px2/PFV6nfuqVQsErd6ElM3xEPBBoxgbaS7xEMXHQ+NPOQVS8iu16r1Sg0iG82RkvJZvepAJ1NKIEPTK372g4gkIRWacKxUz0GxdqdYakY4nRX6iaIxJmM8pD1DBQ6pcqfzi2fwxCgBHETSlNBwrv6cmOJQqUnom84Q65H67aXiX14v0YOaO2UiTjQVZLFokHCoI5i+DwMmKdF8YggmkplbIRlhiYk2IS1tCVR62szk8v08/J+0y7ZTtdFNpdS4zBLKgyNwDE6BA85BA1yDJmgBAgR4BE/g2XqwXqxX623RmrOymUOwBOv9C3UxmK8=</latexit> u0 = W0h0 <latexit sha1_base64="i98NF53nvMx1GTvIOlT02vBIGAA=">AAACBHicdVDLSsNAFL2pr1pfVZdugkVwFRKt2o1QdOOygn1gG8JkMmmHTiZhZiKU0K1f4Fa/wJ249T/8AP/DSVuhFT0wcDjnXu6Z4yeMSmXbn0ZhaXllda24XtrY3NreKe/utWScCkyaOGax6PhIEkY5aSqqGOkkgqDIZ6TtD69zv/1AhKQxv1OjhLgR6nMaUoyUlu5Tz7lse87Ac7xyxbHsCUzbOq9W7dOaJjPlx6rADA2v/NULYpxGhCvMkJRdx06UmyGhKGZkXOqlkiQID1GfdDXlKCLSzSaJx+aRVgIzjIV+XJkTdX4jQ5GUo8jXkxFSA/nby8W/vG6qwpqbUZ6kinA8PRSmzFSxmX/fDKggWLGRJggLqrOaeIAEwkqXtHAlkHm08Xwv/5PWieWcWfZttVK/mjVUhAM4hGNw4ALqcAMNaAIGDk/wDC/Go/FqvBnv09GCMdvZhwUYH988fpiK</latexit> u1 = W1h1 <latexit sha1_base64="4dcnyKt/ee7kGXZ6S3uBkWEAkPc=">AAACHXicdVDLSsNAFJ3UV62vqEs3o0VwVRKt2mXRjcsK9gFNCJPJpB06mYSZiVBC136GX+BWv8CduBU/wP9w0ka0ohcGzj3nvub4CaNSWda7UVpYXFpeKa9W1tY3NrfM7Z2OjFOBSRvHLBY9H0nCKCdtRRUjvUQQFPmMdP3RZa53b4mQNOY3apwQN0IDTkOKkdKUZ+47oUA4cxIkFEUMpp49+c6GOvPMql2zpgGt2lm9bp00NCiYL6kKimh55ocTxDiNCFeYISn7tpUoN8tHYkYmFSeVJEF4hAakryFHEZFuNv3KBB5qJoBhLPTjCk7Znx0ZiqQcR76ujJAayt9aTv6l9VMVNtyM8iRVhOPZojBlUMUw9wUGVBCs2FgDhAXVt0I8RNobpd2b2xLI/LQ5X/4HneOafVqzruvV5kXhUBnsgQNwBGxwDprgCrRAG2BwBx7AI3gy7o1n48V4nZWWjKJnF8yF8fYJDnqjNw==</latexit> @u1 @h1 <latexit sha1_base64="60kCDCJfdCUFlI7azaDnN8WmG14=">AAACHXicdVDLSsNAFJ3UV62vqEs3o0VwVRJRbHdFNy4r2Ae0IUwmk3boZBJmJkIJWfsZfoFb/QJ34lb8AP/DSRuxFT0wcObc9/FiRqWyrA+jtLS8srpWXq9sbG5t75i7ex0ZJQKTNo5YJHoekoRRTtqKKkZ6sSAo9BjpeuOrPN69I0LSiN+qSUycEA05DShGSkuueTgIBMLpIEZCUcTgyLWzn1/iWplrVq2aNQWcI41G3a43oF0oVVCg5ZqfAz/CSUi4wgxJ2betWDlp3hIzklUGiSQxwmM0JH1NOQqJdNLpKRk81ooPg0joxxWcqvMVKQqlnISezgyRGsnfsVz8K9ZPVFB3UsrjRBGOZ4OChEEVwdwX6FNBsGITTRAWVO8K8Qhpb5R2b2GKL/PVcl++j4f/k85pzT6vWTdn1eZl4VAZHIAjcAJscAGa4Bq0QBtgcA8ewRN4Nh6MF+PVeJulloyiZh8swHj/AiX8o0g=</latexit> @h1 @u0 <latexit sha1_base64="2g++4FK2qtbTNVizFSiWmGPzmRk=">AAACHXicdVDLSsNAFJ34rPUVdelmtAiuQlJabXdFNy4r2Ac0IUwmk3bo5MHMRCihaz/DL3CrX+BO3Iof4H84aSNa0QMD555779x7j5cwKqRpvmtLyyura+uljfLm1vbOrr633xVxyjHp4JjFvO8hQRiNSEdSyUg/4QSFHiM9b3yZ53u3hAsaRzdykhAnRMOIBhQjqSRXP7IDjnBmJ4hLihhMXXP6HfVU5OoV02g26s1aA5qGOUNOqmfNugWtQqmAAm1X/7D9GKchiSRmSIiBZSbSyfIvMSPTsp0KkiA8RkMyUDRCIRFONjtlCk+U4sMg5upFEs7Unx0ZCoWYhJ6qDJEcid+5XPwrN0hl0HAyGiWpJBGeDwpSBmUMc1+gTznBkk0UQZhTtSvEI6S8kcq9hSm+yFfLffk6Hv5PulXDqhvmda3SuigcKoFDcAxOgQXOQQtcgTboAAzuwAN4BE/avfasvWiv89Ilreg5AAvQ3j4BLYSjTA==</latexit> @u0 @W0 <latexit sha1_base64="+jqY1jG3/sRBUYetLzlPwiYD6Ak=">AAACBnicdVDLSsNAFL3xWeur6tLNYBHqpiQq2i6EohuXFewD2hAmk0k7dPJgZiKU0L1f4Fa/wJ249Tf8AP/DSRuhFT0wcDjnXu6Z48acSWWan8bS8srq2npho7i5tb2zW9rbb8soEYS2SMQj0XWxpJyFtKWY4rQbC4oDl9OOO7rJ/M4DFZJF4b0ax9QO8CBkPiNYaak/dKwr3zEriWOeOKWyWTWnQHOkXq9ZtTqycqUMOZpO6avvRSQJaKgIx1L2LDNWdoqFYoTTSbGfSBpjMsID2tM0xAGVdjrNPEHHWvGQHwn9QoWm6vxGigMpx4GrJwOshvK3l4l/eb1E+TU7ZWGcKBqS2SE/4UhFKCsAeUxQovhYE0wE01kRGWKBidI1LVzxZBZtonv5+Tz6n7RPq9ZF9ezuvNy4zhsqwCEcQQUsuIQG3EITWkAghid4hhfj0Xg13oz32eiSke8cwAKMj289EpkS</latexit> h1 = f0(u0) <latexit sha1_base64="YsNX+xavKPnYRsukvqtfuULWxoM=">AAACBHicbVDLSsNAFJ3UV62vqks3g0Wom5C0anUhFN24rGAf2IYwmUzaoZNJmJkIpXTrF7jVL3Anbv0PP8D/cNIGsdUDA4dz7uWeOV7MqFSW9WnklpZXVtfy64WNza3tneLuXktGicCkiSMWiY6HJGGUk6aiipFOLAgKPUba3vA69dsPREga8Ts1iokToj6nAcVIaek+vgxcu5y49rFbLFmmNQW0zNNaxbqowh/FzkgJZGi4xa+eH+EkJFxhhqTs2lasnDESimJGJoVeIkmM8BD1SVdTjkIinfE08QQeacWHQST04wpO1d8bYxRKOQo9PRkiNZCLXir+53UTFZw7Y8rjRBGOZ4eChEEVwfT70KeCYMVGmiAsqM4K8QAJhJUuae6KL9NoE92LvdjCX9KqmPaZWb09KdWvsoby4AAcgjKwQQ3UwQ1ogCbAgIMn8AxejEfj1Xgz3mejOSPb2QdzMD6+Af4MmGY=</latexit> p = f1(u1) 二次最適化 <latexit sha1_base64="eGta8ATILI7rVM5edzp7/uMnFog=">AAAB+3icbVDLSsNAFJ3UV62vqks3g0VwVRIVdVl047IF+4A2lMnkph06mYSZiRBCvsCtfoE7cevH+AH+h5M2C1s9MHA4517umePFnClt219WZW19Y3Orul3b2d3bP6gfHvVUlEgKXRrxSA48ooAzAV3NNIdBLIGEHoe+N7sv/P4TSMUi8ajTGNyQTAQLGCXaSJ10XG/YTXsO/Jc4JWmgEu1x/XvkRzQJQWjKiVJDx461mxGpGeWQ10aJgpjQGZnA0FBBQlBuNg+a4zOj+DiIpHlC47n6eyMjoVJp6JnJkOipWvUK8T9vmOjg1s2YiBMNgi4OBQnHOsLFr7HPJFDNU0MIlcxkxXRKJKHadLN0xVdFtNz04qy28Jf0LprOdfOyc9Vo3ZUNVdEJOkXnyEE3qIUeUBt1EUWAntELerVy6816tz4WoxWr3DlGS7A+fwCA+ZVy</latexit> y Forward propagation Backward propagation Cross entropy loss 確率的勾配降下法 (SGD) ：ラベル ニュートン法 <latexit sha1_base64="BpaiO7b9hbl/rfkDJL7cM+CMhk8=">AAACNHicbVDLSsNAFJ34rO+qSzeDRRDEkqhoN0LRjQsXFawtNCXcTCc6dPJg5kYoIb/iZ/gFbnUvuJNu/QYntYhVDwwczn2dOX4ihUbbfrWmpmdm5+ZLC4tLyyura+X1jRsdp4rxJotlrNo+aC5FxJsoUPJ2ojiEvuQtv39e1Fv3XGkRR9c4SHg3hNtIBIIBGskr11pehntOftrycN/lCG6ggGVuAgoFSDcEvGMgs8s8/xap6c29csWu2iPQv8QZkwoZo+GVh24vZmnII2QStO44doLdrFjJJM8X3VTzBFgfbnnH0AhCrrvZ6Ic53TFKjwaxMi9COlJ/TmQQaj0IfdNZONa/a4X4X62TYlDrZiJKUuQR+zoUpJJiTIu4aE8ozlAODAGmhPFK2R2YiNCEOnGlpwtrRS7O7xT+kpuDqnNcPbw6qtTPxgmVyBbZJrvEISekTi5IgzQJIw/kiTyTF+vRerPereFX65Q1ntkkE7A+PgFacq02</latexit> Wt+1 = Wt ⌘ @L @Wt <latexit sha1_base64="n5rjZ5Qg/xL62RwL7jDIoRo0hhc=">AAACU3icbVDLSgMxFE3HV31Uqy7dBIvgpmVGRd0IRTcuXCjYB3RquZOkNTTzIMkIZZhP8zNcuBZ3+gVuzExHsNYLgXPPuYd7c7xIcKVt+7VkLSwuLa+UV9fWNyqbW9XtnbYKY0lYi4QilF0PFBM8YC3NtWDdSDLwPcE63vgq0ztPTCoeBvd6ErG+D6OADzkBbahBteP6oB8JiOQmvXBV7A8Smj4kFDSk05aYlghQKq1PTEdTV4QjHOVw6qDFfP1HsWk6qNbshp0XngdOAWqoqNtB9d2lIYl9Fuh8W8+xI91PQGpOBEvX3FixCMgYRqxnYAA+U/0kDyDFB4aheBhK8wKNc/a3IwFfqYnvmcnsu+qvlpH/ab1YD8/7CQ+iWLOATBcNY4F1iLM0MeWSES0mBgCR3NyKySNIINpkPrOFquy0LBfnbwrzoH3UcE4bx3cnteZlkVAZ7aF9dIgcdIaa6BrdohYi6Bm9oQ/0WXopfVmWtTgdtUqFZxfNlFX5BvKjuGo=</latexit> L = data X d class X c ycd log pcd = data X d log p0d 0番目のクラス以外はyが0 <latexit sha1_base64="jJqM+7iD0K9mFlT0b4KMrfuNcuY=">AAACSHicbVBdS9xAFJ2s36u1qz72ZXApKOKStEX7IohCaWEfLHRdYbMNN7M3OjiZhJmbwhLyp/wZ/gJ9VPC9b6VvnawLrdoDA4dzv86cOFfSku/feo2Z2bn5hcWl5vLKq9XXrbX1U5sVRmBPZCozZzFYVFJjjyQpPMsNQhor7MeXx3W9/wONlZn+RuMchymca5lIAeSkqNXtRyXtBNVBP6LdEAm2Pu2EmFupMs2/bH8vd4MqTAyIMszBkATFwxToQoAqu1X1V3XzVdRq+x1/Av6SBFPSZlOcRK2HcJSJIkVNQoG1g8DPaVjWK4XCqhkWFnMQl3COA0c1pGiH5eTXFX/rlBFPMuOeJj5R/50oIbV2nMaus3Zsn9dq8X+1QUHJx2EpdV4QavF4KCkUp4zXEfKRNChIjR0BYaTzysUFuIzIBf3kysjW1upcgucpvCSn7zrBXuf91w/tw6NpQovsDdtkWyxg++yQfWYnrMcEu2I37I7de9feT++X9/uxteFNZzbYEzQafwAK6rNU</latexit> Wt+1 = Wt ⌘(F + ✏I) 1 @L @Wt <latexit sha1_base64="UewtE/fDOnWMzvPBtTmXUd0cysU=">AAACSHicbVDLShxBFK0e38bHmCyzKRwEgzh0m6BuAhI3Ci4UHEeYHpvbNbe1sLq6qbotDE3/lJ+RL0iWEbJ3J9mlehzweaDgcO7r1IlzJS35/m+vMTE5NT0zOzf/YWFxabm58vHMZoUR2BGZysx5DBaV1NghSQrPc4OQxgq78fV+Xe/eoLEy06c0zLGfwqWWiRRAToqaR92opI2g+t6NaDNEgvWDjRBzK1Wm+eGXi3IzqMLEgCjDHAxJUDxMga4EqPKoqp5UN19FzZbf9kfgb0kwJi02xnHU/BsOMlGkqEkosLYX+Dn1y3qlUFjNh4XFHMQ1XGLPUQ0p2n45+nXF15wy4Elm3NPER+rziRJSa4dp7Dprx/Z1rRbfq/UKSnb7pdR5QajF46GkUJwyXkfIB9KgIDV0BISRzisXV+AyIhf0iysDW1urcwlep/CWnG21g+3215Nvrb0f44Rm2We2ytZZwHbYHjtgx6zDBLtlv9gfduf99O69B+/fY2vDG898Yi/QaPwHDoyzVg==</latexit> Wt+1 = Wt ⌘(H + ✏I) 1 @L @Wt 自然勾配法 (NGD) Hessian Matrix <latexit sha1_base64="J/eT61UPlf4/Zpervl7KdtRMB4E=">AAACTHicbVDLSsNAFJ3U9zvq0s1gEerCklRRN0LRTZcVrBWaNtxMJnVw8mBmIpSQ3/Iz3Atu6xe4E8FJWkSrFwbOnHPvnDvHSziTyrJejMrc/MLi0vLK6tr6xuaWub1zK+NUENohMY/FnQeSchbRjmKK07tEUAg9Trvew1Whdx+pkCyObtQoof0QhhELGAGlKddsty4cmYauP8h8UJCXFzLICAcp85GbET93AgEkcxIQigEfNGpHDo+HOCnFw/xbwd1BI3fNqlW3ysJ/gT0FVTSttmuOHT8maUgjVZr2bCtR/ax4knCarzqppAmQBxjSnoYRhFT2s/LnOT7QjI+DWOgTKVyyPycyCKUchZ7uDEHdy1mtIP/TeqkKzvsZi5JU0YhMjIKUYxXjIkbsM0GJ4iMNgAimd8XkHnROSof9y8WXxWpFLvZsCn/BbaNun9aPr0+qzctpQstoD+2jGrLRGWqiFmqjDiLoCb2iMXozno1348P4nLRWjOnMLvpVlcUvVVy2ew==</latexit> H = data X d class X c ycd @2 ( log pcd) @W2 Fisher Information Matrix <latexit sha1_base64="4+ok3dn+3WEzhQuWD4QWl/PV5n0=">AAACj3icnVHLTttAFB2bFtJQIMCyi46IKiWLRjYgYAOKWqlqdyARghQn1vV4nIwyfmjmGimy/BV8HR/Qj+iuYycSzxVXGunMOfelc4NMCo2O82DZax8+rm80PjU3P29t77R29250mivGByyVqboNQHMpEj5AgZLfZopDHEg+DOY/K314x5UWaXKNi4yPY5gmIhIM0FB+6/7Xuafz2A8nRQgIZf1hk4JJ0LrM/IKFpSd5hB0vUsAKLwOFAiTtfPdkOqXLjG75KAxLT4npDLuT63cW+q2203PqoK+BuwJtsopLv/XXC1OWxzzBeu+R62Q4Lqq+TPKy6eWaZ8DmMOUjAxOIuR4XtXkl/WaYkEapMi9BWrNPKwqItV7EgcmMAWf6pVaRb2mjHKOzcSGSLEeesOWgKJcUU1pdgoZCcYZyYQAwJcyulM3AeIXmXs+mhLparTS+uC9deA1uDnvuSe/o6rjd/7FyqEG+kAPSIS45JX3ym1ySAWHkn/XV6lhde9c+tS/s/jLVtlY1++RZ2H/+A5X9y0Y=</latexit> F = data X d class X c pcd ✓ @( log pcd) @W ◆T ✓ @( log pcd) @W ◆ <latexit sha1_base64="jb0lx9ewDeWcpdT4TYMCFgH2XhM=">AAACJnicbVDLSsNAFJ3UV62vqEs3g0UQFyVRUTdC0Y0LFxXsA5oQJtNJO3QyCTMToYT8g5/hF7jVL3An4s6N/+GkDWhbDwwczn2dOX7MqFSW9WmUFhaXllfKq5W19Y3NLXN7pyWjRGDSxBGLRMdHkjDKSVNRxUgnFgSFPiNtf3id19sPREga8Xs1iokboj6nAcVIackzjy6dQCCcOjESiiIGnRCpAUYsvc2yX7XtWZlnVq2aNQacJ3ZBqqBAwzO/nV6Ek5BwhRmSsmtbsXLTfCVmJKs4iSQxwkPUJ11NOQqJdNPxnzJ4oJUeDCKhH1dwrP6dSFEo5Sj0dWfuWM7WcvG/WjdRwYWbUh4ninA8ORQkDKoI5gHBHhUEKzbSBGFBtVeIB0hnpHSMU1d6MreW52LPpjBPWsc1+6x2cndarV8VCZXBHtgHh8AG56AObkADNAEGj+AZvIBX48l4M96Nj0lryShmdsEUjK8ffU2nGw==</latexit> = @L @W0 <latexit sha1_base64="cTNj8iNZAYwSNbFoDmIfybXzjCI=">AAACKnicbVDLSsNAFJ3UV62vqEs3g0VwFZIq6kYounHhooJ9QJOGyXTSDp08mJkIJeQv/Ay/wK1+gbviVv/DSRvEth4YOJz7OnO8mFEhTXOilVZW19Y3ypuVre2d3T19/6AlooRj0sQRi3jHQ4IwGpKmpJKRTswJCjxG2t7oNq+3nwgXNAof5TgmToAGIfUpRlJJrm5c2z5HOLVjxCVFrFeDdoDkECOW3mfZrw7brtmrZa5eNQ1zCrhMrIJUQYGGq3/b/QgnAQklZkiIrmXG0knzpZiRrGIngsQIj9CAdBUNUUCEk07/lcETpfShH3H1Qgmn6t+JFAVCjANPdeaexWItF/+rdRPpXzkpDeNEkhDPDvkJgzKCeUiwTznBko0VQZhT5RXiIVI5SRXl3JW+yK3luViLKSyTVs2wLoyzh/Nq/aZIqAyOwDE4BRa4BHVwBxqgCTB4Bq/gDbxrL9qHNtE+Z60lrZg5BHPQvn4AGkioYw==</latexit> = @2 L @W2 0 <latexit sha1_base64="y2oDuE0Z4ODbnn3qGECe4x7CyFE=">AAACJXicbVDLSsNAFJ3UV62vqEs3g0XQTUlatborunHhooJ9QBPKZDpph04mYWYilJBv8DP8Arf6Be5EcOXK/3DSFrXVAwOHc19njhcxKpVlvRu5hcWl5ZX8amFtfWNzy9zeacowFpg0cMhC0faQJIxy0lBUMdKOBEGBx0jLG15m9dYdEZKG/FaNIuIGqM+pTzFSWuqaR44vEE6cCAlFEYNOgNQAI5Zcp+mPGnfttGsWrZI1BrRKJ9WydV6B34o9JUUwRb1rfjq9EMcB4QozJGXHtiLlJtlKzEhacGJJIoSHqE86mnIUEOkm4y+l8EArPeiHQj+u4Fj9PZGgQMpR4OnOzLGcr2Xif7VOrPwzN6E8ihXheHLIjxlUIczygT0qCFZspAnCgmqvEA+QzkjpFGeu9GRmLcvFnk/hL2mWS/ZpqXJzXKxdTBPKgz2wDw6BDaqgBq5AHTQABvfgETyBZ+PBeDFejbdJa86YzuyCGRgfX0Zdpw0=</latexit> @L @u1 <latexit sha1_base64="AcONnGolPxuebPshgODpJbL+6Ck=">AAACIXicdVDLSgMxFM34rPU16tJNsBRclUxptd0V3bisYB/QTodMmmlDMw+SjFCGfoGf4Re41S9wJ+7Etf9hph3Fih4InJz7Pm7EmVQIvRkrq2vrG5u5rfz2zu7evnlw2JZhLAhtkZCHoutiSTkLaEsxxWk3EhT7Lqcdd3KZxju3VEgWBjdqGlHbx6OAeYxgpSXHLPY9gUnSj7BQDPNBGcYOmn3/YcdBg/LMMQuoVK9V65UaRCU0R0rKZ/WqBa1MKYAMTcf86A9DEvs0UIRjKXsWipSdpE0Jp7N8P5Y0wmSCR7SnaYB9Ku1kfs4MFrUyhF4o9AsUnKs/KxLsSzn1XZ3pYzWWv2Op+FesFyuvZicsiGJFA7IY5MUcqhCm3sAhE5QoPtUEE8H0rpCMsfZHaQeXpgxlulrqy9fx8H/SLpesagldVwqNi8yhHDgGJ+AUWOAcNMAVaIIWIOAOPIBH8GTcG8/Gi/G6SF0xspojsATj/RO9yKSU</latexit> @2 u0 @W2 0 <latexit sha1_base64="4dcnyKt/ee7kGXZ6S3uBkWEAkPc=">AAACHXicdVDLSsNAFJ3UV62vqEs3o0VwVRKt2mXRjcsK9gFNCJPJpB06mYSZiVBC136GX+BWv8CduBU/wP9w0ka0ohcGzj3nvub4CaNSWda7UVpYXFpeKa9W1tY3NrfM7Z2OjFOBSRvHLBY9H0nCKCdtRRUjvUQQFPmMdP3RZa53b4mQNOY3apwQN0IDTkOKkdKUZ+47oUA4cxIkFEUMpp49+c6GOvPMql2zpgGt2lm9bp00NCiYL6kKimh55ocTxDiNCFeYISn7tpUoN8tHYkYmFSeVJEF4hAakryFHEZFuNv3KBB5qJoBhLPTjCk7Znx0ZiqQcR76ujJAayt9aTv6l9VMVNtyM8iRVhOPZojBlUMUw9wUGVBCs2FgDhAXVt0I8RNobpd2b2xLI/LQ5X/4HneOafVqzruvV5kXhUBnsgQNwBGxwDprgCrRAG2BwBx7AI3gy7o1n48V4nZWWjKJnF8yF8fYJDnqjNw==</latexit> @u1 @h1 <latexit sha1_base64="60kCDCJfdCUFlI7azaDnN8WmG14=">AAACHXicdVDLSsNAFJ3UV62vqEs3o0VwVRJRbHdFNy4r2Ae0IUwmk3boZBJmJkIJWfsZfoFb/QJ34lb8AP/DSRuxFT0wcObc9/FiRqWyrA+jtLS8srpWXq9sbG5t75i7ex0ZJQKTNo5YJHoekoRRTtqKKkZ6sSAo9BjpeuOrPN69I0LSiN+qSUycEA05DShGSkuueTgIBMLpIEZCUcTgyLWzn1/iWplrVq2aNQWcI41G3a43oF0oVVCg5ZqfAz/CSUi4wgxJ2betWDlp3hIzklUGiSQxwmM0JH1NOQqJdNLpKRk81ooPg0joxxWcqvMVKQqlnISezgyRGsnfsVz8K9ZPVFB3UsrjRBGOZ4OChEEVwdwX6FNBsGITTRAWVO8K8Qhpb5R2b2GKL/PVcl++j4f/k85pzT6vWTdn1eZl4VAZHIAjcAJscAGa4Bq0QBtgcA8ewRN4Nh6MF+PVeJulloyiZh8swHj/AiX8o0g=</latexit> @h1 @u0 <latexit sha1_base64="y2oDuE0Z4ODbnn3qGECe4x7CyFE=">AAACJXicbVDLSsNAFJ3UV62vqEs3g0XQTUlatborunHhooJ9QBPKZDpph04mYWYilJBv8DP8Arf6Be5EcOXK/3DSFrXVAwOHc19njhcxKpVlvRu5hcWl5ZX8amFtfWNzy9zeacowFpg0cMhC0faQJIxy0lBUMdKOBEGBx0jLG15m9dYdEZKG/FaNIuIGqM+pTzFSWuqaR44vEE6cCAlFEYNOgNQAI5Zcp+mPGnfttGsWrZI1BrRKJ9WydV6B34o9JUUwRb1rfjq9EMcB4QozJGXHtiLlJtlKzEhacGJJIoSHqE86mnIUEOkm4y+l8EArPeiHQj+u4Fj9PZGgQMpR4OnOzLGcr2Xif7VOrPwzN6E8ihXheHLIjxlUIczygT0qCFZspAnCgmqvEA+QzkjpFGeu9GRmLcvFnk/hL2mWS/ZpqXJzXKxdTBPKgz2wDw6BDaqgBq5AHTQABvfgETyBZ+PBeDFejbdJa86YzuyCGRgfX0Zdpw0=</latexit> @L @u1 <latexit sha1_base64="IC3BZczJB1g7VpddVsQWCOb6l3U=">AAACLHicdVBLSgNBEO3xG/9Rl24ag6Cb0BMSTXZBNy4jGCNkxtDT6Ukaez501whhyDU8hidwqydwI+JSz2FPMuIHfdDw6lVVV9XzYik0EPJszczOzS8sFpaWV1bX1jeKm1sXOkoU420WyUhdelRzKULeBgGSX8aK08CTvONdn2T5zg1XWkThOYxi7gZ0EApfMApG6hWJI7kP+46vKEudmCoQVOKkR8ZfUcdEjhKDIRxcVXrFEik36rVGtY5JmUyQkcpho2ZjO1dKKEerV3xz+hFLAh4Ck1Trrk1icNPsbyb5eNlJNI8pu6YD3jU0pAHXbjq5bIz3jNLHfqTMCwFP1O8dKQ20HgWeqQwoDPXvXCb+lesm4NfdVIRxAjxk00F+IjFEOLMJ94XiDOTIEMqUMLtiNqTGJDBm/pjS19lqY+PL5/H4f3JRKdu1MjmrlprHuUMFtIN20T6y0RFqolPUQm3E0C26Rw/o0bqznqwX63VaOmPlPdvoB6z3D89bqUQ=</latexit> ✓ @u0 @W0 ◆2<latexit sha1_base64="TggLdRqJE1uP/xnJQDmFFzUtW9E=">AAACIXicdVDLSsNAFJ34rPUVdelmsBRclaQotruiG5cV7AOaNEwmk3bo5MHMRCghX+Bn+AVu9QvciTtx7X84aaO0ogcGzpz7Pm7MqJCG8a6trK6tb2yWtsrbO7t7+/rBYVdECcekgyMW8b6LBGE0JB1JJSP9mBMUuIz03MlVHu/dES5oFN7KaUzsAI1C6lOMpJIcvWr5HOHUihGXFLFhHY4dM/v5w8QxhvXM0StGzZgBLpBms2E2mtAslAoo0Hb0T8uLcBKQUGKGhBiYRiztNG+KGcnKViJIjPAEjchA0RAFRNjp7JwMVpXiQT/i6oUSztTFihQFQkwDV2UGSI7F71gu/hUbJNJv2CkN40SSEM8H+QmDMoK5N9CjnGDJpoogzKnaFeIxUv5I5eDSFE/kq+W+fB8P/yfdes08rxk3Z5XWZeFQCRyDE3AKTHABWuAatEEHYHAPHsETeNYetBftVXubp65oRc0RWIL28QW2ZKSQ</latexit> @2 h1 @u2 0 <latexit sha1_base64="y2oDuE0Z4ODbnn3qGECe4x7CyFE=">AAACJXicbVDLSsNAFJ3UV62vqEs3g0XQTUlatborunHhooJ9QBPKZDpph04mYWYilJBv8DP8Arf6Be5EcOXK/3DSFrXVAwOHc19njhcxKpVlvRu5hcWl5ZX8amFtfWNzy9zeacowFpg0cMhC0faQJIxy0lBUMdKOBEGBx0jLG15m9dYdEZKG/FaNIuIGqM+pTzFSWuqaR44vEE6cCAlFEYNOgNQAI5Zcp+mPGnfttGsWrZI1BrRKJ9WydV6B34o9JUUwRb1rfjq9EMcB4QozJGXHtiLlJtlKzEhacGJJIoSHqE86mnIUEOkm4y+l8EArPeiHQj+u4Fj9PZGgQMpR4OnOzLGcr2Xif7VOrPwzN6E8ihXheHLIjxlUIczygT0qCFZspAnCgmqvEA+QzkjpFGeu9GRmLcvFnk/hL2mWS/ZpqXJzXKxdTBPKgz2wDw6BDaqgBq5AHTQABvfgETyBZ+PBeDFejbdJa86YzuyCGRgfX0Zdpw0=</latexit> @L @u1 <latexit sha1_base64="4dcnyKt/ee7kGXZ6S3uBkWEAkPc=">AAACHXicdVDLSsNAFJ3UV62vqEs3o0VwVRKt2mXRjcsK9gFNCJPJpB06mYSZiVBC136GX+BWv8CduBU/wP9w0ka0ohcGzj3nvub4CaNSWda7UVpYXFpeKa9W1tY3NrfM7Z2OjFOBSRvHLBY9H0nCKCdtRRUjvUQQFPmMdP3RZa53b4mQNOY3apwQN0IDTkOKkdKUZ+47oUA4cxIkFEUMpp49+c6GOvPMql2zpgGt2lm9bp00NCiYL6kKimh55ocTxDiNCFeYISn7tpUoN8tHYkYmFSeVJEF4hAakryFHEZFuNv3KBB5qJoBhLPTjCk7Znx0ZiqQcR76ujJAayt9aTv6l9VMVNtyM8iRVhOPZojBlUMUw9wUGVBCs2FgDhAXVt0I8RNobpd2b2xLI/LQ5X/4HneOafVqzruvV5kXhUBnsgQNwBGxwDprgCrRAG2BwBx7AI3gy7o1n48V4nZWWjKJnF8yF8fYJDnqjNw==</latexit> @u1 @h1 <latexit sha1_base64="oOKKxXBkqsxtVKat4hEUiX6oSQ0=">AAACLHicdVDLTgIxFO3gC/GFunTTSExwQ2aIRtgR3bjERMCEwUmndKCh80h7x4RM5jf8DL/ArX6BG2Nc6nfYAYxg9CRNTs99HzcSXIFpvhq5peWV1bX8emFjc2t7p7i711ZhLClr0VCE8sYligkesBZwEOwmkoz4rmAdd3SRxTt3TCoeBtcwjljPJ4OAe5wS0JJTNG3BPCjbniQ0sSMigROBh46V/vxix0xtyQdDOL6tOsWSWTEnwHOkXq9ZtTq2ZkoJzdB0ih92P6SxzwKggijVtcwIeknWmwqWFuxYsYjQERmwrqYB8ZnqJZPLUnyklT72QqlfAHiizlckxFdq7Ls60ycwVL9jmfhXrBuDV+slPIhiYAGdDvJigSHEmU24zyWjIMaaECq53hXTIdEmgTZzYUpfZaul2pfv4/H/pF2tWKcV8+qk1DifOZRHB+gQlZGFzlADXaImaiGK7tEjekLPxoPxYrwZ79PUnDGr2UcLMD6/AMh1qUA=</latexit> ✓ @h1 @u0 ◆2 <latexit sha1_base64="0w0W4FA0ABw+m2yU+f0MApEOkPQ=">AAACIXicdVBLTsMwFHT4lvILsGRjUVViFSWlQJcVbFgWiX6kJo0cx2mtOh/ZDlIV9QQcgxOwhROwQ+wQa+6B0wZEEYxkaTzznt/zeAmjQprmm7a0vLK6tl7aKG9ube/s6nv7HRGnHJM2jlnMex4ShNGItCWVjPQSTlDoMdL1xpe5370lXNA4upGThDghGkY0oBhJJbl61Q44wpmdIC4pYoMaTF1r+n2HI9ca1KauXrEMcwZoGmf1unnSUKRQvqwKKNBy9Q/bj3EakkhihoToW2YinSx/FDMyLdupIAnCYzQkfUUjFBLhZLPvTGFVKT4MYq5OJOFM/dmRoVCISeipyhDJkfjt5eJfXj+VQcPJaJSkkkR4PihIGZQxzLOBPuUESzZRBGFO1a4Qj5DKR6oEF6b4Il9tIZf/SadmWKeGeV2vNC+KhErgEByBY2CBc9AEV6AF2gCDO/AAHsGTdq89ay/a67x0SSt6DsACtPdPnuSkfw==</latexit> @2 u1 @h2 1 <latexit sha1_base64="y2oDuE0Z4ODbnn3qGECe4x7CyFE=">AAACJXicbVDLSsNAFJ3UV62vqEs3g0XQTUlatborunHhooJ9QBPKZDpph04mYWYilJBv8DP8Arf6Be5EcOXK/3DSFrXVAwOHc19njhcxKpVlvRu5hcWl5ZX8amFtfWNzy9zeacowFpg0cMhC0faQJIxy0lBUMdKOBEGBx0jLG15m9dYdEZKG/FaNIuIGqM+pTzFSWuqaR44vEE6cCAlFEYNOgNQAI5Zcp+mPGnfttGsWrZI1BrRKJ9WydV6B34o9JUUwRb1rfjq9EMcB4QozJGXHtiLlJtlKzEhacGJJIoSHqE86mnIUEOkm4y+l8EArPeiHQj+u4Fj9PZGgQMpR4OnOzLGcr2Xif7VOrPwzN6E8ihXheHLIjxlUIczygT0qCFZspAnCgmqvEA+QzkjpFGeu9GRmLcvFnk/hL2mWS/ZpqXJzXKxdTBPKgz2wDw6BDaqgBq5AHTQABvfgETyBZ+PBeDFejbdJa86YzuyCGRgfX0Zdpw0=</latexit> @L @u1 <latexit sha1_base64="IC3BZczJB1g7VpddVsQWCOb6l3U=">AAACLHicdVBLSgNBEO3xG/9Rl24ag6Cb0BMSTXZBNy4jGCNkxtDT6Ukaez501whhyDU8hidwqydwI+JSz2FPMuIHfdDw6lVVV9XzYik0EPJszczOzS8sFpaWV1bX1jeKm1sXOkoU420WyUhdelRzKULeBgGSX8aK08CTvONdn2T5zg1XWkThOYxi7gZ0EApfMApG6hWJI7kP+46vKEudmCoQVOKkR8ZfUcdEjhKDIRxcVXrFEik36rVGtY5JmUyQkcpho2ZjO1dKKEerV3xz+hFLAh4Ck1Trrk1icNPsbyb5eNlJNI8pu6YD3jU0pAHXbjq5bIz3jNLHfqTMCwFP1O8dKQ20HgWeqQwoDPXvXCb+lesm4NfdVIRxAjxk00F+IjFEOLMJ94XiDOTIEMqUMLtiNqTGJDBm/pjS19lqY+PL5/H4f3JRKdu1MjmrlprHuUMFtIN20T6y0RFqolPUQm3E0C26Rw/o0bqznqwX63VaOmPlPdvoB6z3D89bqUQ=</latexit> ✓ @u0 @W0 ◆2 <latexit sha1_base64="bJ4mOY/Sx8VJdy264L1EG5wbsqs=">AAACLHicdVDLTgIxFO3gC/E16tJNIzHBDZlBVJZENy4xkUfCIOmUDjR0HmnvmJAJv+Fn+AVu9QvcGONSv8MOYASjN2ly7jn31eNGgiuwrFcjs7S8srqWXc9tbG5t75i7ew0VxpKyOg1FKFsuUUzwgNWBg2CtSDLiu4I13eFlqjfvmFQ8DG5gFLGOT/oB9zgloKmuaTmCeVBwPElo4kREAicCx117/JMNdOZI3h/A8W2pa+btojUJbBXPymXrpKLBjPmW8mgWta754fRCGvssACqIUm3biqCTpLOpYOOcEysWETokfdbWMCA+U51k8rMxPtJMD3uh1C8APGHnOxLiKzXyXV3pExio31pK/qW1Y/AqnYQHUQwsoNNFXiwwhDi1Cfe4ZBTESANCJde3Yjog2iTQZi5s6an0tPG8L/+DRqlonxat63K+ejFzKIsO0CEqIBudoyq6QjVURxTdo0f0hJ6NB+PFeDPep6UZY9azjxbC+PwCsPypLw==</latexit> ✓ @u1 @h1 ◆2 <latexit sha1_base64="W8GkI4+vVnwzYjJIuHqMZlXN0ks=">AAACKXicbVDLSsNAFJ3UV62vqEs3g0VwY0latborunHhooJ9QJOWyXTSDp08mJkIJeQr/Ay/wK1+gTt1K/6HkzYUWz0wcDj3deY4IaNCGsaHlltaXlldy68XNja3tnf03b2mCCKOSQMHLOBtBwnCqE8akkpG2iEnyHMYaTmj67TeeiBc0MC/l+OQ2B4a+NSlGEkl9fQTy+UIx1aIuKSIdcvQ8pAcYsTi2ySZ6TDqmd1y0tOLRsmYABqls2rZuKzAmWJmpAgy1Hv6t9UPcOQRX2KGhOiYRijtOF2KGUkKViRIiPAIDUhHUR95RNjx5FsJPFJKH7oBV8+XcKL+noiRJ8TYc1Rn6lks1lLxv1onku6FHVM/jCTx8fSQGzEoA5hmBPuUEyzZWBGEOVVeIR4ilZNUSc5d6YvUWpqLuZjCX9Isl8zzUuXutFi7yhLKgwNwCI6BCaqgBm5AHTQABo/gGbyAV+1Je9Petc9pa07LZvbBHLSvH+KLqFU=</latexit> @2 L @u2 1 <latexit sha1_base64="oOKKxXBkqsxtVKat4hEUiX6oSQ0=">AAACLHicdVDLTgIxFO3gC/GFunTTSExwQ2aIRtgR3bjERMCEwUmndKCh80h7x4RM5jf8DL/ArX6BG2Nc6nfYAYxg9CRNTs99HzcSXIFpvhq5peWV1bX8emFjc2t7p7i711ZhLClr0VCE8sYligkesBZwEOwmkoz4rmAdd3SRxTt3TCoeBtcwjljPJ4OAe5wS0JJTNG3BPCjbniQ0sSMigROBh46V/vxix0xtyQdDOL6tOsWSWTEnwHOkXq9ZtTq2ZkoJzdB0ih92P6SxzwKggijVtcwIeknWmwqWFuxYsYjQERmwrqYB8ZnqJZPLUnyklT72QqlfAHiizlckxFdq7Ls60ycwVL9jmfhXrBuDV+slPIhiYAGdDvJigSHEmU24zyWjIMaaECq53hXTIdEmgTZzYUpfZaul2pfv4/H/pF2tWKcV8+qk1DifOZRHB+gQlZGFzlADXaImaiGK7tEjekLPxoPxYrwZ79PUnDGr2UcLMD6/AMh1qUA=</latexit> ✓ @h1 @u0 ◆2 <latexit sha1_base64="IC3BZczJB1g7VpddVsQWCOb6l3U=">AAACLHicdVBLSgNBEO3xG/9Rl24ag6Cb0BMSTXZBNy4jGCNkxtDT6Ukaez501whhyDU8hidwqydwI+JSz2FPMuIHfdDw6lVVV9XzYik0EPJszczOzS8sFpaWV1bX1jeKm1sXOkoU420WyUhdelRzKULeBgGSX8aK08CTvONdn2T5zg1XWkThOYxi7gZ0EApfMApG6hWJI7kP+46vKEudmCoQVOKkR8ZfUcdEjhKDIRxcVXrFEik36rVGtY5JmUyQkcpho2ZjO1dKKEerV3xz+hFLAh4Ck1Trrk1icNPsbyb5eNlJNI8pu6YD3jU0pAHXbjq5bIz3jNLHfqTMCwFP1O8dKQ20HgWeqQwoDPXvXCb+lesm4NfdVIRxAjxk00F+IjFEOLMJ94XiDOTIEMqUMLtiNqTGJDBm/pjS19lqY+PL5/H4f3JRKdu1MjmrlprHuUMFtIN20T6y0RFqolPUQm3E0C26Rw/o0bqznqwX63VaOmPlPdvoB6z3D89bqUQ=</latexit> ✓ @u0 @W0 ◆2 0 0 0 <latexit sha1_base64="wtR5RRFqIRcgx8j2NFVe7h3VpUg=">AAAB+3icbVDLSsNAFJ3UV62vqks3g0UQhJKoqMuiG5ct2Ae0oUwmN+3QySTMTIQS8gVu9QvciVs/xg/wP5y0WdjWAwOHc+7lnjlezJnStv1tldbWNza3ytuVnd29/YPq4VFHRYmk0KYRj2TPIwo4E9DWTHPoxRJI6HHoepOH3O8+g1QsEk96GoMbkpFgAaNEG6l1MazW7Lo9A14lTkFqqEBzWP0Z+BFNQhCacqJU37Fj7aZEakY5ZJVBoiAmdEJG0DdUkBCUm86CZvjMKD4OImme0Him/t1ISajUNPTMZEj0WC17ufif1090cOemTMSJBkHnh4KEYx3h/NfYZxKo5lNDCJXMZMV0TCSh2nSzcMVXebTM9OIst7BKOpd156Z+1bquNe6LhsroBJ2ic+SgW9RAj6iJ2ogiQC/oFb1ZmfVufVif89GSVewcowVYX78E95Uk</latexit> + <latexit sha1_base64="wtR5RRFqIRcgx8j2NFVe7h3VpUg=">AAAB+3icbVDLSsNAFJ3UV62vqks3g0UQhJKoqMuiG5ct2Ae0oUwmN+3QySTMTIQS8gVu9QvciVs/xg/wP5y0WdjWAwOHc+7lnjlezJnStv1tldbWNza3ytuVnd29/YPq4VFHRYmk0KYRj2TPIwo4E9DWTHPoxRJI6HHoepOH3O8+g1QsEk96GoMbkpFgAaNEG6l1MazW7Lo9A14lTkFqqEBzWP0Z+BFNQhCacqJU37Fj7aZEakY5ZJVBoiAmdEJG0DdUkBCUm86CZvjMKD4OImme0Him/t1ISajUNPTMZEj0WC17ufif1090cOemTMSJBkHnh4KEYx3h/NfYZxKo5lNDCJXMZMV0TCSh2nSzcMVXebTM9OIst7BKOpd156Z+1bquNe6LhsroBJ2ic+SgW9RAj6iJ2ogiQC/oFb1ZmfVufVif89GSVewcowVYX78E95Uk</latexit> + <latexit sha1_base64="wtR5RRFqIRcgx8j2NFVe7h3VpUg=">AAAB+3icbVDLSsNAFJ3UV62vqks3g0UQhJKoqMuiG5ct2Ae0oUwmN+3QySTMTIQS8gVu9QvciVs/xg/wP5y0WdjWAwOHc+7lnjlezJnStv1tldbWNza3ytuVnd29/YPq4VFHRYmk0KYRj2TPIwo4E9DWTHPoxRJI6HHoepOH3O8+g1QsEk96GoMbkpFgAaNEG6l1MazW7Lo9A14lTkFqqEBzWP0Z+BFNQhCacqJU37Fj7aZEakY5ZJVBoiAmdEJG0DdUkBCUm86CZvjMKD4OImme0Him/t1ISajUNPTMZEj0WC17ufif1090cOemTMSJBkHnh4KEYx3h/NfYZxKo5lNDCJXMZMV0TCSh2nSzcMVXebTM9OIst7BKOpd156Z+1bquNe6LhsroBJ2ic+SgW9RAj6iJ2ogiQC/oFb1ZmfVufVif89GSVewcowVYX78E95Uk</latexit> + <latexit sha1_base64="aENufBL+EZ99+g5stLK5ADY7YDY=">AAACKnicbVDLSgMxFM3UV62vqks3wSLUTZkRRZdFNy4r2Ad0hiGTybShmQfJHaEM8xd+hl/gVr/AXXGr/2GmLWhbDwROzrk39+Z4ieAKTHNilNbWNza3ytuVnd29/YPq4VFHxamkrE1jEcueRxQTPGJt4CBYL5GMhJ5gXW90V/jdJyYVj6NHGCfMCckg4gGnBLTkVhu2YAHU7UASmtkJkcCJwKlr5b+3rmvmtuSDIZy71ZrZMKfAq8Sakxqao+VWv20/pmnIIqCCKNW3zAScrHiZCpZX7FSxhNARGbC+phEJmXKy6b9yfKYVHwex1CcCPFX/dmQkVGoceroyJDBUy14h/uf1UwhunIxHSQosorNBQSowxLgICftcMgpirAmhkutdMR0SHRHoKBem+KpYLde5WMsprJLORcO6apgPl7Xm7TyhMjpBp6iOLHSNmugetVAbUfSMXtEbejdejA9jYnzOSkvGvOcYLcD4+gEYwKhf</latexit> ✓ @u1 @W0 ◆ <latexit sha1_base64="p4V4N6uy+7pvXjmkyIvfobbKdMA=">AAACLHicbVDLSgMxFM3UV62vqks3wSLUTZkRRZdFNy4r9AWdOmTSTBuaeZDcEcowv+Fn+AVu9QvciLjU7zDTDmhbDwROzrk39+a4keAKTPPdKKysrq1vFDdLW9s7u3vl/YO2CmNJWYuGIpRdlygmeMBawEGwbiQZ8V3BOu74JvM7D0wqHgZNmESs75NhwD1OCWjJKZu2YB5UbU8SmtgRkcCJwLFjpb+3jmOmtuTDEZzeN51yxayZU+BlYuWkgnI0nPKXPQhp7LMAqCBK9Swzgn6SvU0FS0t2rFhE6JgMWU/TgPhM9ZPpz1J8opUB9kKpTwB4qv7tSIiv1MR3daVPYKQWvUz8z+vF4F31Ex5EMbCAzgZ5scAQ4iwmPOCSURATTQiVXO+K6YjokECHOTdloLLVUp2LtZjCMmmf1ayLmnl3Xqlf5wkV0RE6RlVkoUtUR7eogVqIokf0jF7Qq/FkvBkfxuestGDkPYdoDsb3D6H7qSU=</latexit> ✓ @u1 @W0 ◆T <latexit sha1_base64="W8GkI4+vVnwzYjJIuHqMZlXN0ks=">AAACKXicbVDLSsNAFJ3UV62vqEs3g0VwY0latborunHhooJ9QJOWyXTSDp08mJkIJeQr/Ay/wK1+gTt1K/6HkzYUWz0wcDj3deY4IaNCGsaHlltaXlldy68XNja3tnf03b2mCCKOSQMHLOBtBwnCqE8akkpG2iEnyHMYaTmj67TeeiBc0MC/l+OQ2B4a+NSlGEkl9fQTy+UIx1aIuKSIdcvQ8pAcYsTi2ySZ6TDqmd1y0tOLRsmYABqls2rZuKzAmWJmpAgy1Hv6t9UPcOQRX2KGhOiYRijtOF2KGUkKViRIiPAIDUhHUR95RNjx5FsJPFJKH7oBV8+XcKL+noiRJ8TYc1Rn6lks1lLxv1onku6FHVM/jCTx8fSQGzEoA5hmBPuUEyzZWBGEOVVeIR4ilZNUSc5d6YvUWpqLuZjCX9Isl8zzUuXutFi7yhLKgwNwCI6BCaqgBm5AHTQABo/gGbyAV+1Je9Petc9pa07LZvbBHLSvH+KLqFU=</latexit> @2 L @u2 1 <latexit sha1_base64="1FSvpn0z1UZm0wDi/lJJVgJG/14=">AAAB+3icbVDLSsNAFJ3UV62vqks3g0VwVRIVdSMU3bhswT6gDWUyuWmHTiZhZiKUkC9wq1/gTtz6MX6A/+GkzcK2Hhg4nHMv98zxYs6Utu1vq7S2vrG5Vd6u7Ozu7R9UD486KkokhTaNeCR7HlHAmYC2ZppDL5ZAQo9D15s85H73GaRikXjS0xjckIwECxgl2kitu2G1ZtftGfAqcQpSQwWaw+rPwI9oEoLQlBOl+o4dazclUjPKIasMEgUxoRMygr6hgoSg3HQWNMNnRvFxEEnzhMYz9e9GSkKlpqFnJkOix2rZy8X/vH6ig1s3ZSJONAg6PxQkHOsI57/GPpNANZ8aQqhkJiumYyIJ1aabhSu+yqNlphdnuYVV0rmoO9f1y9ZVrXFfNFRGJ+gUnSMH3aAGekRN1EYUAXpBr+jNyqx368P6nI+WrGLnGC3A+voFIZWVNg==</latexit> = Gauss-Newton Matrix
- 14. 2次最適化 Gradient Descent Natural GradientDescent <latexit sha1_base64="IOyR436oWLZbrKn8O0Lz+NybarM=">AAACMXicbVDLSsNAFJ34rPVVdekmWARFLInvjSC6ceGigrVCU8rNdGqHTiZh5kYoIV/iZ/gFbvUL3Ingyp9w0kaw1QvDnDnnXu6Z40eCa3ScN2ticmp6ZrYwV5xfWFxaLq2s3uowVpTVaChCdeeDZoJLVkOOgt1FikHgC1b3exeZXn9gSvNQ3mA/Ys0A7iXvcApoqFbp0MMuQ2gluOOmp/kDdz1zeRJ8AV4A2KUgkqt060febpXKTsUZlP0XuDkok7yqrdKn1w5pHDCJVIDWDdeJsJmAQk4FS4terFkEtAf3rGGghIDpZjL4XmpvGqZtd0JljkR7wP6eSCDQuh/4pjMzq8e1jPxPa8TYOWkmXEYxMkmHizqxsDG0s6zsNleMougbAFRx49WmXVBA0SQ6sqWtM2upycUdT+EvuN2ruEeV/euD8tl5nlCBrJMNskVcckzOyCWpkhqh5JE8kxfyaj1Zb9a79TFsnbDymTUyUtbXNx2yq3o=</latexit> ✓t+1 = ✓t ⌘rL(✓t) <latexit sha1_base64="GfVQ19YivDaRfAFZWatRqCWvb+U=">AAACRXicbVBNa9tAEF25X2nSD7c99rLEFByCjdSENpdCaKAkkEMKtR2wVDNaj+Ilq5XYHQWM0G/qz+gv6CGX9tRjbiHXZOWo0HwMLPv2zTzm7YtzJS35/qnXevDw0eMnS0+XV549f/Gy/er10GaFETgQmcrMYQwWldQ4IEkKD3ODkMYKR/HxTt0fnaCxMtPfaJ5jlMKRlokUQI6atPdCmiHBpKT1oPrUPKgXuot3v6yHmFupMs331r6XvaAKNcQKwhRoJkCV+1X3n2Rt0u74fX9R/C4IGtBhTR1M2n/DaSaKFDUJBdaOAz+nqARDUiislsPCYg7iGI5w7KCGFG1ULr5c8XeOmfIkM+5o4gv2f0UJqbXzNHaTtVl7u1eT9/XGBSVbUSl1XhBqcb0oKRSnjNf58ak0KEjNHQBhpPPKxQwMCHIp39gytbW1yuUS3E7hLhi+7wcf+htfNzvbn5uElthbtsq6LGAf2TbbZQdswAT7wX6x3+yP99M78869i+vRltdo3rAb5V1eAci3spc=</latexit> ✓t+1 = ✓t ⌘(F + ✏I) 1 rL(✓t)
- 15.
- 16.
- 17. 2層の全結合NN D_in=3 H=5 D_out=2 Data batch_size(BS)=2 x(BS,D_in) w1(D_in,H) w2(H,D_out) y_p(BS,D_out) h_r=f(x*w1) y=f(x) ReLU (RectifiedLinear Unit) y_p=h_r*w2 Back propagation @L @w2 = @L @yp @yp @w2 = 1 NO 2(yp y)hr <latexit sha1_base64="0rAPpB7aTBShnIAwO9pRmo0aRkQ=">AAACdnicfVHLSsNAFJ1ErbU+GnUpyGAp1oUlKYJuhKIbF6IV7APaEibTSTt08mBmosSQD3Xnxn9w6aQNaFvxwsDhnHvvuZxxQkaFNM13TV9b3yhsFrdK2zu7e2Vj/6Ajgohj0sYBC3jPQYIw6pO2pJKRXsgJ8hxGus70NtO7L4QLGvjPMg7J0ENjn7oUI6ko23gbuBzhZBAiLili8D79wa92I73+R4/tMF2SM2pxQSnfYKXJg/2YNmqq5Tw+m9jcNipm3ZwVXAVWDiogr5ZtfA5GAY484kvMkBB9ywzlMMnMMCNpaRAJEiI8RWPSV9BHHhHDZJZRCquKGUE34Or5Es7Y3xMJ8oSIPUd1ekhOxLKWkX9p/Ui6V8OE+mEkiY/nRm7EoAxgFjgcUU6wZLECCHOqboV4glQkUn3LgstIZKelKhdrOYVV0GnULbNuPV1Umjd5QkVwBE5ADVjgEjTBHWiBNsDgQytoZc3QvvRjvaqfzlt1LZ85BAulm99738GM</latexit> <latexit sha1_base64="0rAPpB7aTBShnIAwO9pRmo0aRkQ=">AAACdnicfVHLSsNAFJ1ErbU+GnUpyGAp1oUlKYJuhKIbF6IV7APaEibTSTt08mBmosSQD3Xnxn9w6aQNaFvxwsDhnHvvuZxxQkaFNM13TV9b3yhsFrdK2zu7e2Vj/6Ajgohj0sYBC3jPQYIw6pO2pJKRXsgJ8hxGus70NtO7L4QLGvjPMg7J0ENjn7oUI6ko23gbuBzhZBAiLili8D79wa92I73+R4/tMF2SM2pxQSnfYKXJg/2YNmqq5Tw+m9jcNipm3ZwVXAVWDiogr5ZtfA5GAY484kvMkBB9ywzlMMnMMCNpaRAJEiI8RWPSV9BHHhHDZJZRCquKGUE34Or5Es7Y3xMJ8oSIPUd1ekhOxLKWkX9p/Ui6V8OE+mEkiY/nRm7EoAxgFjgcUU6wZLECCHOqboV4glQkUn3LgstIZKelKhdrOYVV0GnULbNuPV1Umjd5QkVwBE5ADVjgEjTBHWiBNsDgQytoZc3QvvRjvaqfzlt1LZ85BAulm99738GM</latexit> <latexit sha1_base64="0rAPpB7aTBShnIAwO9pRmo0aRkQ=">AAACdnicfVHLSsNAFJ1ErbU+GnUpyGAp1oUlKYJuhKIbF6IV7APaEibTSTt08mBmosSQD3Xnxn9w6aQNaFvxwsDhnHvvuZxxQkaFNM13TV9b3yhsFrdK2zu7e2Vj/6Ajgohj0sYBC3jPQYIw6pO2pJKRXsgJ8hxGus70NtO7L4QLGvjPMg7J0ENjn7oUI6ko23gbuBzhZBAiLili8D79wa92I73+R4/tMF2SM2pxQSnfYKXJg/2YNmqq5Tw+m9jcNipm3ZwVXAVWDiogr5ZtfA5GAY484kvMkBB9ywzlMMnMMCNpaRAJEiI8RWPSV9BHHhHDZJZRCquKGUE34Or5Es7Y3xMJ8oSIPUd1ekhOxLKWkX9p/Ui6V8OE+mEkiY/nRm7EoAxgFjgcUU6wZLECCHOqboV4glQkUn3LgstIZKelKhdrOYVV0GnULbNuPV1Umjd5QkVwBE5ADVjgEjTBHWiBNsDgQytoZc3QvvRjvaqfzlt1LZ85BAulm99738GM</latexit> <latexit sha1_base64="KD+WkL8b4iPtxogxdCifBK+sduE=">AAAB7HicbVDLSgNBEOyNrxijxrOXwSB4Crte9Ch48RjBPCBZwuxsJxkyO7vM9AphyQ949Qu8iX/kB/gfziY5mMSCgaKqm66pKFPSku9/e5W9/YPDo+px7aReOz07b9S7Ns2NwI5IVWr6EbeopMYOSVLYzwzyJFLYi2aPpd97RWNlql9onmGY8ImWYyk4Oak9ajT9lr8E2yXBmjRhjVHjZxinIk9Qk1Dc2kHgZxQW3JAUChe1YW4x42LGJzhwVPMEbVgsYy7YtVNiNk6Ne5rYUv27UfDE2nkSucmE09Rue6X4nzfIaXwfFlJnOaEWq0PjXDFKWflnFkuDgtTcES6MdFmZmHLDBblmNq7Etoy2cLUE2yXsku5tK/BbwbMPVbiEK7iBAO7gAZ6gDR0QEMMbvHuF9+F9ruqreOseL2AD3tcvU8WSkg==</latexit> <latexit sha1_base64="BPl6LZUWEc7LnKT4OpXuCsHjQG0=">AAACa3icfZHNS8MwGMbT+jXndNWrIMExnAdHu4teBMGLB9EJ7gPWUtIs3cLSD5JUqaV/qDcv/g8eTbeCbhNfCDw8T/K+L794MaNCmua7pm9sbm3vVHare7X9g7pxWOuLKOGY9HDEIj70kCCMhqQnqWRkGHOCAo+RgTe7LfLBC+GCRuGzTGPiBGgSUp9iJJXlGm+2zxHO7BhxSRGD9/mPfnU7+fU/eerG+UpcWMsNqmUHK88e3Me801JXLtLzqctdo2G2zXnBdWGVogHK6rrGpz2OcBKQUGKGhBhZZiydrBiGGcmrdiJIjPAMTchIyRAFRDjZnFEOm8oZQz/i6oQSzt3fLzIUCJEGnroZIDkVq1lh/pWNEulfORkN40SSEC8G+QmDMoIFcDimnGDJUiUQ5lTtCvEUKSRSfcvSlLEoVssVF2uVwrrod9qW2baeTFABx+AUtIAFLsENuANd0AMYfGjbWl0ztC/9RG8uCOpaifIILJV+9g0Z5MCq</latexit> <latexit sha1_base64="BPl6LZUWEc7LnKT4OpXuCsHjQG0=">AAACa3icfZHNS8MwGMbT+jXndNWrIMExnAdHu4teBMGLB9EJ7gPWUtIs3cLSD5JUqaV/qDcv/g8eTbeCbhNfCDw8T/K+L794MaNCmua7pm9sbm3vVHare7X9g7pxWOuLKOGY9HDEIj70kCCMhqQnqWRkGHOCAo+RgTe7LfLBC+GCRuGzTGPiBGgSUp9iJJXlGm+2zxHO7BhxSRGD9/mPfnU7+fU/eerG+UpcWMsNqmUHK88e3Me801JXLtLzqctdo2G2zXnBdWGVogHK6rrGpz2OcBKQUGKGhBhZZiydrBiGGcmrdiJIjPAMTchIyRAFRDjZnFEOm8oZQz/i6oQSzt3fLzIUCJEGnroZIDkVq1lh/pWNEulfORkN40SSEC8G+QmDMoIFcDimnGDJUiUQ5lTtCvEUKSRSfcvSlLEoVssVF2uVwrrod9qW2baeTFABx+AUtIAFLsENuANd0AMYfGjbWl0ztC/9RG8uCOpaifIILJV+9g0Z5MCq</latexit> <latexit sha1_base64="9s6RO95OuhDQX1hzGQv1yWyzpFM=">AAACdnicfVHLSsNAFJ1ErbU+GnUpyGAp1oUl6UY3QtGNC9EK9gFNCZPpxA6dPJiZKDHkQ9258R9cOmkD2la8MHA45957LmfciFEhTfNd09fWN0qb5a3K9s7uXtXYP+iJMOaYdHHIQj5wkSCMBqQrqWRkEHGCfJeRvju9yfX+C+GChsGTTCIy8tFzQD2KkVSUY7zZHkc4tSPEJUUM3mU/+NVpZVf/6IkTZUtyTi0uqBQbrCy9dx6yVkO1nCdnE4c7Rs1smrOCq8AqQA0U1XGMT3sc4tgngcQMCTG0zEiO0twMM5JV7FiQCOEpeiZDBQPkEzFKZxllsK6YMfRCrl4g4Yz9PZEiX4jEd1Wnj+RELGs5+Zc2jKV3OUppEMWSBHhu5MUMyhDmgcMx5QRLliiAMKfqVognSEUi1bcsuIxFflqmcrGWU1gFvVbTMpvWo1lrXxcJlcEROAENYIEL0Aa3oAO6AIMPraRVNUP70o/1un46b9W1YuYQLJRufgN6n8GI</latexit> <latexit sha1_base64="0rAPpB7aTBShnIAwO9pRmo0aRkQ=">AAACdnicfVHLSsNAFJ1ErbU+GnUpyGAp1oUlKYJuhKIbF6IV7APaEibTSTt08mBmosSQD3Xnxn9w6aQNaFvxwsDhnHvvuZxxQkaFNM13TV9b3yhsFrdK2zu7e2Vj/6Ajgohj0sYBC3jPQYIw6pO2pJKRXsgJ8hxGus70NtO7L4QLGvjPMg7J0ENjn7oUI6ko23gbuBzhZBAiLili8D79wa92I73+R4/tMF2SM2pxQSnfYKXJg/2YNmqq5Tw+m9jcNipm3ZwVXAVWDiogr5ZtfA5GAY484kvMkBB9ywzlMMnMMCNpaRAJEiI8RWPSV9BHHhHDZJZRCquKGUE34Or5Es7Y3xMJ8oSIPUd1ekhOxLKWkX9p/Ui6V8OE+mEkiY/nRm7EoAxgFjgcUU6wZLECCHOqboV4glQkUn3LgstIZKelKhdrOYVV0GnULbNuPV1Umjd5QkVwBE5ADVjgEjTBHWiBNsDgQytoZc3QvvRjvaqfzlt1LZ85BAulm99738GM</latexit> <latexit sha1_base64="0rAPpB7aTBShnIAwO9pRmo0aRkQ=">AAACdnicfVHLSsNAFJ1ErbU+GnUpyGAp1oUlKYJuhKIbF6IV7APaEibTSTt08mBmosSQD3Xnxn9w6aQNaFvxwsDhnHvvuZxxQkaFNM13TV9b3yhsFrdK2zu7e2Vj/6Ajgohj0sYBC3jPQYIw6pO2pJKRXsgJ8hxGus70NtO7L4QLGvjPMg7J0ENjn7oUI6ko23gbuBzhZBAiLili8D79wa92I73+R4/tMF2SM2pxQSnfYKXJg/2YNmqq5Tw+m9jcNipm3ZwVXAVWDiogr5ZtfA5GAY484kvMkBB9ywzlMMnMMCNpaRAJEiI8RWPSV9BHHhHDZJZRCquKGUE34Or5Es7Y3xMJ8oSIPUd1ekhOxLKWkX9p/Ui6V8OE+mEkiY/nRm7EoAxgFjgcUU6wZLECCHOqboV4glQkUn3LgstIZKelKhdrOYVV0GnULbNuPV1Umjd5QkVwBE5ADVjgEjTBHWiBNsDgQytoZc3QvvRjvaqfzlt1LZ85BAulm99738GM</latexit> <latexit sha1_base64="0rAPpB7aTBShnIAwO9pRmo0aRkQ=">AAACdnicfVHLSsNAFJ1ErbU+GnUpyGAp1oUlKYJuhKIbF6IV7APaEibTSTt08mBmosSQD3Xnxn9w6aQNaFvxwsDhnHvvuZxxQkaFNM13TV9b3yhsFrdK2zu7e2Vj/6Ajgohj0sYBC3jPQYIw6pO2pJKRXsgJ8hxGus70NtO7L4QLGvjPMg7J0ENjn7oUI6ko23gbuBzhZBAiLili8D79wa92I73+R4/tMF2SM2pxQSnfYKXJg/2YNmqq5Tw+m9jcNipm3ZwVXAVWDiogr5ZtfA5GAY484kvMkBB9ywzlMMnMMCNpaRAJEiI8RWPSV9BHHhHDZJZRCquKGUE34Or5Es7Y3xMJ8oSIPUd1ekhOxLKWkX9p/Ui6V8OE+mEkiY/nRm7EoAxgFjgcUU6wZLECCHOqboV4glQkUn3LgstIZKelKhdrOYVV0GnULbNuPV1Umjd5QkVwBE5ADVjgEjTBHWiBNsDgQytoZc3QvvRjvaqfzlt1LZ85BAulm99738GM</latexit> <latexit sha1_base64="0rAPpB7aTBShnIAwO9pRmo0aRkQ=">AAACdnicfVHLSsNAFJ1ErbU+GnUpyGAp1oUlKYJuhKIbF6IV7APaEibTSTt08mBmosSQD3Xnxn9w6aQNaFvxwsDhnHvvuZxxQkaFNM13TV9b3yhsFrdK2zu7e2Vj/6Ajgohj0sYBC3jPQYIw6pO2pJKRXsgJ8hxGus70NtO7L4QLGvjPMg7J0ENjn7oUI6ko23gbuBzhZBAiLili8D79wa92I73+R4/tMF2SM2pxQSnfYKXJg/2YNmqq5Tw+m9jcNipm3ZwVXAVWDiogr5ZtfA5GAY484kvMkBB9ywzlMMnMMCNpaRAJEiI8RWPSV9BHHhHDZJZRCquKGUE34Or5Es7Y3xMJ8oSIPUd1ekhOxLKWkX9p/Ui6V8OE+mEkiY/nRm7EoAxgFjgcUU6wZLECCHOqboV4glQkUn3LgstIZKelKhdrOYVV0GnULbNuPV1Umjd5QkVwBE5ADVjgEjTBHWiBNsDgQytoZc3QvvRjvaqfzlt1LZ85BAulm99738GM</latexit> <latexit sha1_base64="0rAPpB7aTBShnIAwO9pRmo0aRkQ=">AAACdnicfVHLSsNAFJ1ErbU+GnUpyGAp1oUlKYJuhKIbF6IV7APaEibTSTt08mBmosSQD3Xnxn9w6aQNaFvxwsDhnHvvuZxxQkaFNM13TV9b3yhsFrdK2zu7e2Vj/6Ajgohj0sYBC3jPQYIw6pO2pJKRXsgJ8hxGus70NtO7L4QLGvjPMg7J0ENjn7oUI6ko23gbuBzhZBAiLili8D79wa92I73+R4/tMF2SM2pxQSnfYKXJg/2YNmqq5Tw+m9jcNipm3ZwVXAVWDiogr5ZtfA5GAY484kvMkBB9ywzlMMnMMCNpaRAJEiI8RWPSV9BHHhHDZJZRCquKGUE34Or5Es7Y3xMJ8oSIPUd1ekhOxLKWkX9p/Ui6V8OE+mEkiY/nRm7EoAxgFjgcUU6wZLECCHOqboV4glQkUn3LgstIZKelKhdrOYVV0GnULbNuPV1Umjd5QkVwBE5ADVjgEjTBHWiBNsDgQytoZc3QvvRjvaqfzlt1LZ85BAulm99738GM</latexit> <latexit sha1_base64="0rAPpB7aTBShnIAwO9pRmo0aRkQ=">AAACdnicfVHLSsNAFJ1ErbU+GnUpyGAp1oUlKYJuhKIbF6IV7APaEibTSTt08mBmosSQD3Xnxn9w6aQNaFvxwsDhnHvvuZxxQkaFNM13TV9b3yhsFrdK2zu7e2Vj/6Ajgohj0sYBC3jPQYIw6pO2pJKRXsgJ8hxGus70NtO7L4QLGvjPMg7J0ENjn7oUI6ko23gbuBzhZBAiLili8D79wa92I73+R4/tMF2SM2pxQSnfYKXJg/2YNmqq5Tw+m9jcNipm3ZwVXAVWDiogr5ZtfA5GAY484kvMkBB9ywzlMMnMMCNpaRAJEiI8RWPSV9BHHhHDZJZRCquKGUE34Or5Es7Y3xMJ8oSIPUd1ekhOxLKWkX9p/Ui6V8OE+mEkiY/nRm7EoAxgFjgcUU6wZLECCHOqboV4glQkUn3LgstIZKelKhdrOYVV0GnULbNuPV1Umjd5QkVwBE5ADVjgEjTBHWiBNsDgQytoZc3QvvRjvaqfzlt1LZ85BAulm99738GM</latexit> L = 1 NO X (yp y)2 <latexit sha1_base64="HuDWCNBZ3fiwLb58W34hfFqbE4Y=">AAACGHicbZDLSsNAFIYn9VbrLepSF8Ei1IUlKYJuhKIbF6IV7AWaGCaTSTt0cmFmIoSQjY/hE7jVJ3Anbt35AL6HkzYL2/rDwM9/zuGc+ZyIEi50/VspLSwuLa+UVytr6xubW+r2ToeHMUO4jUIasp4DOaYkwG1BBMW9iGHoOxR3ndFlXu8+YsZJGNyLJMKWDwcB8QiCQka2un99bnoMotTI0hv7NjN57NcSOzpOjh4atlrV6/pY2rwxClMFhVq2+mO6IYp9HAhEIed9Q4+ElUImCKI4q5gxxxFEIzjAfWkD6GNupeNfZNqhTFzNC5l8gdDG6d+JFPqcJ74jO30ohny2lof/1fqx8M6slARRLHCAJou8mGoi1HIkmksYRoIm0kDEiLxVQ0MooQgJbmqLy/PTMsnFmKUwbzqNuqHXjbuTavOiIFQGe+AA1IABTkETXIEWaAMEnsALeAVvyrPyrnwon5PWklLM7IIpKV+/xXGgTg==</latexit> <latexit sha1_base64="HuDWCNBZ3fiwLb58W34hfFqbE4Y=">AAACGHicbZDLSsNAFIYn9VbrLepSF8Ei1IUlKYJuhKIbF6IV7AWaGCaTSTt0cmFmIoSQjY/hE7jVJ3Anbt35AL6HkzYL2/rDwM9/zuGc+ZyIEi50/VspLSwuLa+UVytr6xubW+r2ToeHMUO4jUIasp4DOaYkwG1BBMW9iGHoOxR3ndFlXu8+YsZJGNyLJMKWDwcB8QiCQka2un99bnoMotTI0hv7NjN57NcSOzpOjh4atlrV6/pY2rwxClMFhVq2+mO6IYp9HAhEIed9Q4+ElUImCKI4q5gxxxFEIzjAfWkD6GNupeNfZNqhTFzNC5l8gdDG6d+JFPqcJ74jO30ohny2lof/1fqx8M6slARRLHCAJou8mGoi1HIkmksYRoIm0kDEiLxVQ0MooQgJbmqLy/PTMsnFmKUwbzqNuqHXjbuTavOiIFQGe+AA1IABTkETXIEWaAMEnsALeAVvyrPyrnwon5PWklLM7IIpKV+/xXGgTg==</latexit> <latexit sha1_base64="HuDWCNBZ3fiwLb58W34hfFqbE4Y=">AAACGHicbZDLSsNAFIYn9VbrLepSF8Ei1IUlKYJuhKIbF6IV7AWaGCaTSTt0cmFmIoSQjY/hE7jVJ3Anbt35AL6HkzYL2/rDwM9/zuGc+ZyIEi50/VspLSwuLa+UVytr6xubW+r2ToeHMUO4jUIasp4DOaYkwG1BBMW9iGHoOxR3ndFlXu8+YsZJGNyLJMKWDwcB8QiCQka2un99bnoMotTI0hv7NjN57NcSOzpOjh4atlrV6/pY2rwxClMFhVq2+mO6IYp9HAhEIed9Q4+ElUImCKI4q5gxxxFEIzjAfWkD6GNupeNfZNqhTFzNC5l8gdDG6d+JFPqcJ74jO30ohny2lof/1fqx8M6slARRLHCAJou8mGoi1HIkmksYRoIm0kDEiLxVQ0MooQgJbmqLy/PTMsnFmKUwbzqNuqHXjbuTavOiIFQGe+AA1IABTkETXIEWaAMEnsALeAVvyrPyrnwon5PWklLM7IIpKV+/xXGgTg==</latexit> <latexit sha1_base64="HuDWCNBZ3fiwLb58W34hfFqbE4Y=">AAACGHicbZDLSsNAFIYn9VbrLepSF8Ei1IUlKYJuhKIbF6IV7AWaGCaTSTt0cmFmIoSQjY/hE7jVJ3Anbt35AL6HkzYL2/rDwM9/zuGc+ZyIEi50/VspLSwuLa+UVytr6xubW+r2ToeHMUO4jUIasp4DOaYkwG1BBMW9iGHoOxR3ndFlXu8+YsZJGNyLJMKWDwcB8QiCQka2un99bnoMotTI0hv7NjN57NcSOzpOjh4atlrV6/pY2rwxClMFhVq2+mO6IYp9HAhEIed9Q4+ElUImCKI4q5gxxxFEIzjAfWkD6GNupeNfZNqhTFzNC5l8gdDG6d+JFPqcJ74jO30ohny2lof/1fqx8M6slARRLHCAJou8mGoi1HIkmksYRoIm0kDEiLxVQ0MooQgJbmqLy/PTMsnFmKUwbzqNuqHXjbuTavOiIFQGe+AA1IABTkETXIEWaAMEnsALeAVvyrPyrnwon5PWklLM7IIpKV+/xXGgTg==</latexit> @L @w1 = @L @yp @yp @hr @hr @w1 = 1 NO 2(yp y)w2x <latexit sha1_base64="V1OkoDW7pmfxcKKULrjvVELuV8s=">AAACmHicfVHdSsMwGE3r//ybeqc3wSHMC0dTBL0Rhl6oIDrFqbCNkmbpFkx/SFJnKX0TX8wH8D1Mt8JcFT8InJzzfTkfJ27EmVSW9WmYc/MLi0vLK5XVtfWNzerW9pMMY0Fom4Q8FC8ulpSzgLYVU5y+RIJi3+X02X29yPXnNyokC4NHlUS05+NBwDxGsNKUU/3oegKTtBthoRjm8Cab4pGDsrN/9MSJspKcU9Pb0BHlhpyadagUFihLb527zK7rN46Sw5FjvzvVmtWwxgV/A1SAGiiq5VS/uv2QxD4NFOFYyg6yItVLczfCaVbpxpJGmLziAe1oGGCfyl46TjGDB5rpQy8U+gQKjtmfEyn2pUx8V3f6WA1lWcvJv7ROrLzTXsqCKFY0IBMjL+ZQhTD/EthnghLFEw0wEUzvCskQ60yU/rgZl77MV8t0Lqicwm/wZDeQ1UD3x7XmeZHQMtgD+6AOEDgBTXAFWqANiGEadQMZtrlrNs1L83rSahrFzA6YKfPhG92OzlM=</latexit> <latexit sha1_base64="V1OkoDW7pmfxcKKULrjvVELuV8s=">AAACmHicfVHdSsMwGE3r//ybeqc3wSHMC0dTBL0Rhl6oIDrFqbCNkmbpFkx/SFJnKX0TX8wH8D1Mt8JcFT8InJzzfTkfJ27EmVSW9WmYc/MLi0vLK5XVtfWNzerW9pMMY0Fom4Q8FC8ulpSzgLYVU5y+RIJi3+X02X29yPXnNyokC4NHlUS05+NBwDxGsNKUU/3oegKTtBthoRjm8Cab4pGDsrN/9MSJspKcU9Pb0BHlhpyadagUFihLb527zK7rN46Sw5FjvzvVmtWwxgV/A1SAGiiq5VS/uv2QxD4NFOFYyg6yItVLczfCaVbpxpJGmLziAe1oGGCfyl46TjGDB5rpQy8U+gQKjtmfEyn2pUx8V3f6WA1lWcvJv7ROrLzTXsqCKFY0IBMjL+ZQhTD/EthnghLFEw0wEUzvCskQ60yU/rgZl77MV8t0Lqicwm/wZDeQ1UD3x7XmeZHQMtgD+6AOEDgBTXAFWqANiGEadQMZtrlrNs1L83rSahrFzA6YKfPhG92OzlM=</latexit> <latexit sha1_base64="V1OkoDW7pmfxcKKULrjvVELuV8s=">AAACmHicfVHdSsMwGE3r//ybeqc3wSHMC0dTBL0Rhl6oIDrFqbCNkmbpFkx/SFJnKX0TX8wH8D1Mt8JcFT8InJzzfTkfJ27EmVSW9WmYc/MLi0vLK5XVtfWNzerW9pMMY0Fom4Q8FC8ulpSzgLYVU5y+RIJi3+X02X29yPXnNyokC4NHlUS05+NBwDxGsNKUU/3oegKTtBthoRjm8Cab4pGDsrN/9MSJspKcU9Pb0BHlhpyadagUFihLb527zK7rN46Sw5FjvzvVmtWwxgV/A1SAGiiq5VS/uv2QxD4NFOFYyg6yItVLczfCaVbpxpJGmLziAe1oGGCfyl46TjGDB5rpQy8U+gQKjtmfEyn2pUx8V3f6WA1lWcvJv7ROrLzTXsqCKFY0IBMjL+ZQhTD/EthnghLFEw0wEUzvCskQ60yU/rgZl77MV8t0Lqicwm/wZDeQ1UD3x7XmeZHQMtgD+6AOEDgBTXAFWqANiGEadQMZtrlrNs1L83rSahrFzA6YKfPhG92OzlM=</latexit> <latexit sha1_base64="V1OkoDW7pmfxcKKULrjvVELuV8s=">AAACmHicfVHdSsMwGE3r//ybeqc3wSHMC0dTBL0Rhl6oIDrFqbCNkmbpFkx/SFJnKX0TX8wH8D1Mt8JcFT8InJzzfTkfJ27EmVSW9WmYc/MLi0vLK5XVtfWNzerW9pMMY0Fom4Q8FC8ulpSzgLYVU5y+RIJi3+X02X29yPXnNyokC4NHlUS05+NBwDxGsNKUU/3oegKTtBthoRjm8Cab4pGDsrN/9MSJspKcU9Pb0BHlhpyadagUFihLb527zK7rN46Sw5FjvzvVmtWwxgV/A1SAGiiq5VS/uv2QxD4NFOFYyg6yItVLczfCaVbpxpJGmLziAe1oGGCfyl46TjGDB5rpQy8U+gQKjtmfEyn2pUx8V3f6WA1lWcvJv7ROrLzTXsqCKFY0IBMjL+ZQhTD/EthnghLFEw0wEUzvCskQ60yU/rgZl77MV8t0Lqicwm/wZDeQ1UD3x7XmeZHQMtgD+6AOEDgBTXAFWqANiGEadQMZtrlrNs1L83rSahrFzA6YKfPhG92OzlM=</latexit> w1 w1 ⌘ @L @w1 <latexit sha1_base64="kN3sQo8OP8glKG68w4PsUtC/f3k=">AAACMXicbVDLSgNBEJyN73fUo5fBIHgx7IqiR9GLBw8RjArZEHonvcmQ2QczvYaw5Ev8DL/Aq35BbiJ48iecjQGfDQM1VdXTPRWkShpy3ZFTmpqemZ2bX1hcWl5ZXSuvb1ybJNMC6yJRib4NwKCSMdZJksLbVCNEgcKboHdW6Dd3qI1M4isapNiMoBPLUAogS7XKh/2W5ysMCbRO+tze9nwk8EMNIvdT0CRB8YvhF7aWYatccavuuPhf4E1AhU2q1iq/+e1EZBHGJBQY0/DclJp58aRQOFz0M4MpiB50sGFhDBGaZj7+3pDvWKbNw0TbExMfs987coiMGUSBdUZAXfNbK8j/tEZG4XEzl3GaEcbic1CYKU4JL7LibalRkBpYAEJLuysXXbDJkE30x5S2KVYrcvF+p/AXXO9XPbfqXR5UTk4nCc2zLbbNdpnHjtgJO2c1VmeC3bNH9sSenQdn5Lw4r5/WkjPp2WQ/ynn/AO/Rq1o=</latexit> <latexit sha1_base64="kN3sQo8OP8glKG68w4PsUtC/f3k=">AAACMXicbVDLSgNBEJyN73fUo5fBIHgx7IqiR9GLBw8RjArZEHonvcmQ2QczvYaw5Ev8DL/Aq35BbiJ48iecjQGfDQM1VdXTPRWkShpy3ZFTmpqemZ2bX1hcWl5ZXSuvb1ybJNMC6yJRib4NwKCSMdZJksLbVCNEgcKboHdW6Dd3qI1M4isapNiMoBPLUAogS7XKh/2W5ysMCbRO+tze9nwk8EMNIvdT0CRB8YvhF7aWYatccavuuPhf4E1AhU2q1iq/+e1EZBHGJBQY0/DclJp58aRQOFz0M4MpiB50sGFhDBGaZj7+3pDvWKbNw0TbExMfs987coiMGUSBdUZAXfNbK8j/tEZG4XEzl3GaEcbic1CYKU4JL7LibalRkBpYAEJLuysXXbDJkE30x5S2KVYrcvF+p/AXXO9XPbfqXR5UTk4nCc2zLbbNdpnHjtgJO2c1VmeC3bNH9sSenQdn5Lw4r5/WkjPp2WQ/ynn/AO/Rq1o=</latexit> <latexit sha1_base64="kN3sQo8OP8glKG68w4PsUtC/f3k=">AAACMXicbVDLSgNBEJyN73fUo5fBIHgx7IqiR9GLBw8RjArZEHonvcmQ2QczvYaw5Ev8DL/Aq35BbiJ48iecjQGfDQM1VdXTPRWkShpy3ZFTmpqemZ2bX1hcWl5ZXSuvb1ybJNMC6yJRib4NwKCSMdZJksLbVCNEgcKboHdW6Dd3qI1M4isapNiMoBPLUAogS7XKh/2W5ysMCbRO+tze9nwk8EMNIvdT0CRB8YvhF7aWYatccavuuPhf4E1AhU2q1iq/+e1EZBHGJBQY0/DclJp58aRQOFz0M4MpiB50sGFhDBGaZj7+3pDvWKbNw0TbExMfs987coiMGUSBdUZAXfNbK8j/tEZG4XEzl3GaEcbic1CYKU4JL7LibalRkBpYAEJLuysXXbDJkE30x5S2KVYrcvF+p/AXXO9XPbfqXR5UTk4nCc2zLbbNdpnHjtgJO2c1VmeC3bNH9sSenQdn5Lw4r5/WkjPp2WQ/ynn/AO/Rq1o=</latexit> <latexit sha1_base64="kN3sQo8OP8glKG68w4PsUtC/f3k=">AAACMXicbVDLSgNBEJyN73fUo5fBIHgx7IqiR9GLBw8RjArZEHonvcmQ2QczvYaw5Ev8DL/Aq35BbiJ48iecjQGfDQM1VdXTPRWkShpy3ZFTmpqemZ2bX1hcWl5ZXSuvb1ybJNMC6yJRib4NwKCSMdZJksLbVCNEgcKboHdW6Dd3qI1M4isapNiMoBPLUAogS7XKh/2W5ysMCbRO+tze9nwk8EMNIvdT0CRB8YvhF7aWYatccavuuPhf4E1AhU2q1iq/+e1EZBHGJBQY0/DclJp58aRQOFz0M4MpiB50sGFhDBGaZj7+3pDvWKbNw0TbExMfs987coiMGUSBdUZAXfNbK8j/tEZG4XEzl3GaEcbic1CYKU4JL7LibalRkBpYAEJLuysXXbDJkE30x5S2KVYrcvF+p/AXXO9XPbfqXR5UTk4nCc2zLbbNdpnHjtgJO2c1VmeC3bNH9sSenQdn5Lw4r5/WkjPp2WQ/ynn/AO/Rq1o=</latexit> w2 w2 ⌘ @L @w2 <latexit sha1_base64="XDVYNpwB7UZogd7iSX6oklwpHpw=">AAACMXicbVDLSgNBEJz1bXxFPXoZDIIXw64oehS9ePAQwZhANoTeSa8Ozj6Y6TWEJV/iZ/gFXvULchPBkz/hbAz4iA0DNVXV0z0VpEoact2hMzU9Mzs3v7BYWlpeWV0rr29cmyTTAusiUYluBmBQyRjrJElhM9UIUaCwEdydFXrjHrWRSXxF/RTbEdzEMpQCyFKd8mGvs+8rDAm0Tnrc3vZ8JPBDDSL3U9AkQfGLwTe2lkGnXHGr7qj4JPDGoMLGVeuU3/1uIrIIYxIKjGl5bkrtvHhSKByU/MxgCuIObrBlYQwRmnY++t6A71imy8NE2xMTH7E/O3KIjOlHgXVGQLfmr1aQ/2mtjMLjdi7jNCOMxdegMFOcEl5kxbtSoyDVtwCElnZXLm7BJkM20V9TuqZYrcjF+5vCJLjer3pu1bs8qJycjhNaYFtsm+0yjx2xE3bOaqzOBHtgT+yZvTiPztB5dd6+rFPOuGeT/Srn4xP07atd</latexit> <latexit sha1_base64="XDVYNpwB7UZogd7iSX6oklwpHpw=">AAACMXicbVDLSgNBEJz1bXxFPXoZDIIXw64oehS9ePAQwZhANoTeSa8Ozj6Y6TWEJV/iZ/gFXvULchPBkz/hbAz4iA0DNVXV0z0VpEoact2hMzU9Mzs3v7BYWlpeWV0rr29cmyTTAusiUYluBmBQyRjrJElhM9UIUaCwEdydFXrjHrWRSXxF/RTbEdzEMpQCyFKd8mGvs+8rDAm0Tnrc3vZ8JPBDDSL3U9AkQfGLwTe2lkGnXHGr7qj4JPDGoMLGVeuU3/1uIrIIYxIKjGl5bkrtvHhSKByU/MxgCuIObrBlYQwRmnY++t6A71imy8NE2xMTH7E/O3KIjOlHgXVGQLfmr1aQ/2mtjMLjdi7jNCOMxdegMFOcEl5kxbtSoyDVtwCElnZXLm7BJkM20V9TuqZYrcjF+5vCJLjer3pu1bs8qJycjhNaYFtsm+0yjx2xE3bOaqzOBHtgT+yZvTiPztB5dd6+rFPOuGeT/Srn4xP07atd</latexit> <latexit sha1_base64="XDVYNpwB7UZogd7iSX6oklwpHpw=">AAACMXicbVDLSgNBEJz1bXxFPXoZDIIXw64oehS9ePAQwZhANoTeSa8Ozj6Y6TWEJV/iZ/gFXvULchPBkz/hbAz4iA0DNVXV0z0VpEoact2hMzU9Mzs3v7BYWlpeWV0rr29cmyTTAusiUYluBmBQyRjrJElhM9UIUaCwEdydFXrjHrWRSXxF/RTbEdzEMpQCyFKd8mGvs+8rDAm0Tnrc3vZ8JPBDDSL3U9AkQfGLwTe2lkGnXHGr7qj4JPDGoMLGVeuU3/1uIrIIYxIKjGl5bkrtvHhSKByU/MxgCuIObrBlYQwRmnY++t6A71imy8NE2xMTH7E/O3KIjOlHgXVGQLfmr1aQ/2mtjMLjdi7jNCOMxdegMFOcEl5kxbtSoyDVtwCElnZXLm7BJkM20V9TuqZYrcjF+5vCJLjer3pu1bs8qJycjhNaYFtsm+0yjx2xE3bOaqzOBHtgT+yZvTiPztB5dd6+rFPOuGeT/Srn4xP07atd</latexit> <latexit sha1_base64="XDVYNpwB7UZogd7iSX6oklwpHpw=">AAACMXicbVDLSgNBEJz1bXxFPXoZDIIXw64oehS9ePAQwZhANoTeSa8Ozj6Y6TWEJV/iZ/gFXvULchPBkz/hbAz4iA0DNVXV0z0VpEoact2hMzU9Mzs3v7BYWlpeWV0rr29cmyTTAusiUYluBmBQyRjrJElhM9UIUaCwEdydFXrjHrWRSXxF/RTbEdzEMpQCyFKd8mGvs+8rDAm0Tnrc3vZ8JPBDDSL3U9AkQfGLwTe2lkGnXHGr7qj4JPDGoMLGVeuU3/1uIrIIYxIKjGl5bkrtvHhSKByU/MxgCuIObrBlYQwRmnY++t6A71imy8NE2xMTH7E/O3KIjOlHgXVGQLfmr1aQ/2mtjMLjdi7jNCOMxdegMFOcEl5kxbtSoyDVtwCElnZXLm7BJkM20V9TuqZYrcjF+5vCJLjer3pu1bs8qJycjhNaYFtsm+0yjx2xE3bOaqzOBHtgT+yZvTiPztB5dd6+rFPOuGeT/Srn4xP07atd</latexit> hr > 0 <latexit sha1_base64="cIbhX78ttMoC8DFM/VOzOTz2skU=">AAAB/3icbVDLSsNAFL2pr1pfUZduBovgqiQi6EqKblxWMG2hDWUymbRDJ5MwMxFK6MIvcKtf4E7c+il+gP/hpM3Cth4YOJxzL/fMCVLOlHacb6uytr6xuVXdru3s7u0f2IdHbZVkklCPJDyR3QArypmgnmaa024qKY4DTjvB+K7wO09UKpaIRz1JqR/joWARI1gbyRsN5I0zsOtOw5kBrRK3JHUo0RrYP/0wIVlMhSYcK9VznVT7OZaaEU6ntX6maIrJGA9pz1CBY6r8fBZ2is6MEqIokeYJjWbq340cx0pN4sBMxliP1LJXiP95vUxH137ORJppKsj8UJRxpBNU/ByFTFKi+cQQTCQzWREZYYmJNv0sXAlVEW1qenGXW1gl7YuG6zTch8t687ZsqAoncArn4MIVNOEeWuABAQYv8Apv1rP1bn1Yn/PRilXuHMMCrK9f9mSWwA==</latexit> <latexit sha1_base64="cIbhX78ttMoC8DFM/VOzOTz2skU=">AAAB/3icbVDLSsNAFL2pr1pfUZduBovgqiQi6EqKblxWMG2hDWUymbRDJ5MwMxFK6MIvcKtf4E7c+il+gP/hpM3Cth4YOJxzL/fMCVLOlHacb6uytr6xuVXdru3s7u0f2IdHbZVkklCPJDyR3QArypmgnmaa024qKY4DTjvB+K7wO09UKpaIRz1JqR/joWARI1gbyRsN5I0zsOtOw5kBrRK3JHUo0RrYP/0wIVlMhSYcK9VznVT7OZaaEU6ntX6maIrJGA9pz1CBY6r8fBZ2is6MEqIokeYJjWbq340cx0pN4sBMxliP1LJXiP95vUxH137ORJppKsj8UJRxpBNU/ByFTFKi+cQQTCQzWREZYYmJNv0sXAlVEW1qenGXW1gl7YuG6zTch8t687ZsqAoncArn4MIVNOEeWuABAQYv8Apv1rP1bn1Yn/PRilXuHMMCrK9f9mSWwA==</latexit> <latexit sha1_base64="cIbhX78ttMoC8DFM/VOzOTz2skU=">AAAB/3icbVDLSsNAFL2pr1pfUZduBovgqiQi6EqKblxWMG2hDWUymbRDJ5MwMxFK6MIvcKtf4E7c+il+gP/hpM3Cth4YOJxzL/fMCVLOlHacb6uytr6xuVXdru3s7u0f2IdHbZVkklCPJDyR3QArypmgnmaa024qKY4DTjvB+K7wO09UKpaIRz1JqR/joWARI1gbyRsN5I0zsOtOw5kBrRK3JHUo0RrYP/0wIVlMhSYcK9VznVT7OZaaEU6ntX6maIrJGA9pz1CBY6r8fBZ2is6MEqIokeYJjWbq340cx0pN4sBMxliP1LJXiP95vUxH137ORJppKsj8UJRxpBNU/ByFTFKi+cQQTCQzWREZYYmJNv0sXAlVEW1qenGXW1gl7YuG6zTch8t687ZsqAoncArn4MIVNOEeWuABAQYv8Apv1rP1bn1Yn/PRilXuHMMCrK9f9mSWwA==</latexit> <latexit sha1_base64="cIbhX78ttMoC8DFM/VOzOTz2skU=">AAAB/3icbVDLSsNAFL2pr1pfUZduBovgqiQi6EqKblxWMG2hDWUymbRDJ5MwMxFK6MIvcKtf4E7c+il+gP/hpM3Cth4YOJxzL/fMCVLOlHacb6uytr6xuVXdru3s7u0f2IdHbZVkklCPJDyR3QArypmgnmaa024qKY4DTjvB+K7wO09UKpaIRz1JqR/joWARI1gbyRsN5I0zsOtOw5kBrRK3JHUo0RrYP/0wIVlMhSYcK9VznVT7OZaaEU6ntX6maIrJGA9pz1CBY6r8fBZ2is6MEqIokeYJjWbq340cx0pN4sBMxliP1LJXiP95vUxH137ORJppKsj8UJRxpBNU/ByFTFKi+cQQTCQzWREZYYmJNv0sXAlVEW1qenGXW1gl7YuG6zTch8t687ZsqAoncArn4MIVNOEeWuABAQYv8Apv1rP1bn1Yn/PRilXuHMMCrK9f9mSWwA==</latexit>
- 18. NumPyだけによる実装 import numpy asnp epochs = 300 batch_size = 32 D_in = 784 H = 100 D_out = 10 learning_rate = 1.0e-06 # create random input and output data x = np.random.randn(batch_size, D_in) y = np.random.randn(batch_size, D_out) # randomly initialize weights w1 = np.random.randn(D_in, H) w2 = np.random.randn(H, D_out) for epoch in range(epochs): # forward pass h = x.dot(w1) # h = x * w1 h_r = np.maximum(h, 0) # h_r = ReLU(h) y_p = h_r.dot(w2) # y_p = h_r * w2 # compute mean squared error and print loss loss = np.square(y_p - y).sum() print(epoch, loss) # backward pass: compute gradients of loss with respect to w2 grad_y_p = 2.0 * (y_p - y) grad_w2 = h_r.T.dot(grad_y_p) # backward pass: compute gradients of loss with respect to w1 grad_h_r = grad_y_p.dot(w2.T) grad_h = grad_h_r.copy() grad_h[h < 0] = 0 grad_w1 = x.T.dot(grad_h) # update weights w1 -= learning_rate * grad_w1 w2 -= learning_rate * grad_w2 w1 w1 ⌘ @L @w1 <latexit sha1_base64="kN3sQo8OP8glKG68w4PsUtC/f3k=">AAACMXicbVDLSgNBEJyN73fUo5fBIHgx7IqiR9GLBw8RjArZEHonvcmQ2QczvYaw5Ev8DL/Aq35BbiJ48iecjQGfDQM1VdXTPRWkShpy3ZFTmpqemZ2bX1hcWl5ZXSuvb1ybJNMC6yJRib4NwKCSMdZJksLbVCNEgcKboHdW6Dd3qI1M4isapNiMoBPLUAogS7XKh/2W5ysMCbRO+tze9nwk8EMNIvdT0CRB8YvhF7aWYatccavuuPhf4E1AhU2q1iq/+e1EZBHGJBQY0/DclJp58aRQOFz0M4MpiB50sGFhDBGaZj7+3pDvWKbNw0TbExMfs987coiMGUSBdUZAXfNbK8j/tEZG4XEzl3GaEcbic1CYKU4JL7LibalRkBpYAEJLuysXXbDJkE30x5S2KVYrcvF+p/AXXO9XPbfqXR5UTk4nCc2zLbbNdpnHjtgJO2c1VmeC3bNH9sSenQdn5Lw4r5/WkjPp2WQ/ynn/AO/Rq1o=</latexit> <latexit sha1_base64="kN3sQo8OP8glKG68w4PsUtC/f3k=">AAACMXicbVDLSgNBEJyN73fUo5fBIHgx7IqiR9GLBw8RjArZEHonvcmQ2QczvYaw5Ev8DL/Aq35BbiJ48iecjQGfDQM1VdXTPRWkShpy3ZFTmpqemZ2bX1hcWl5ZXSuvb1ybJNMC6yJRib4NwKCSMdZJksLbVCNEgcKboHdW6Dd3qI1M4isapNiMoBPLUAogS7XKh/2W5ysMCbRO+tze9nwk8EMNIvdT0CRB8YvhF7aWYatccavuuPhf4E1AhU2q1iq/+e1EZBHGJBQY0/DclJp58aRQOFz0M4MpiB50sGFhDBGaZj7+3pDvWKbNw0TbExMfs987coiMGUSBdUZAXfNbK8j/tEZG4XEzl3GaEcbic1CYKU4JL7LibalRkBpYAEJLuysXXbDJkE30x5S2KVYrcvF+p/AXXO9XPbfqXR5UTk4nCc2zLbbNdpnHjtgJO2c1VmeC3bNH9sSenQdn5Lw4r5/WkjPp2WQ/ynn/AO/Rq1o=</latexit> <latexit sha1_base64="kN3sQo8OP8glKG68w4PsUtC/f3k=">AAACMXicbVDLSgNBEJyN73fUo5fBIHgx7IqiR9GLBw8RjArZEHonvcmQ2QczvYaw5Ev8DL/Aq35BbiJ48iecjQGfDQM1VdXTPRWkShpy3ZFTmpqemZ2bX1hcWl5ZXSuvb1ybJNMC6yJRib4NwKCSMdZJksLbVCNEgcKboHdW6Dd3qI1M4isapNiMoBPLUAogS7XKh/2W5ysMCbRO+tze9nwk8EMNIvdT0CRB8YvhF7aWYatccavuuPhf4E1AhU2q1iq/+e1EZBHGJBQY0/DclJp58aRQOFz0M4MpiB50sGFhDBGaZj7+3pDvWKbNw0TbExMfs987coiMGUSBdUZAXfNbK8j/tEZG4XEzl3GaEcbic1CYKU4JL7LibalRkBpYAEJLuysXXbDJkE30x5S2KVYrcvF+p/AXXO9XPbfqXR5UTk4nCc2zLbbNdpnHjtgJO2c1VmeC3bNH9sSenQdn5Lw4r5/WkjPp2WQ/ynn/AO/Rq1o=</latexit> <latexit sha1_base64="kN3sQo8OP8glKG68w4PsUtC/f3k=">AAACMXicbVDLSgNBEJyN73fUo5fBIHgx7IqiR9GLBw8RjArZEHonvcmQ2QczvYaw5Ev8DL/Aq35BbiJ48iecjQGfDQM1VdXTPRWkShpy3ZFTmpqemZ2bX1hcWl5ZXSuvb1ybJNMC6yJRib4NwKCSMdZJksLbVCNEgcKboHdW6Dd3qI1M4isapNiMoBPLUAogS7XKh/2W5ysMCbRO+tze9nwk8EMNIvdT0CRB8YvhF7aWYatccavuuPhf4E1AhU2q1iq/+e1EZBHGJBQY0/DclJp58aRQOFz0M4MpiB50sGFhDBGaZj7+3pDvWKbNw0TbExMfs987coiMGUSBdUZAXfNbK8j/tEZG4XEzl3GaEcbic1CYKU4JL7LibalRkBpYAEJLuysXXbDJkE30x5S2KVYrcvF+p/AXXO9XPbfqXR5UTk4nCc2zLbbNdpnHjtgJO2c1VmeC3bNH9sSenQdn5Lw4r5/WkjPp2WQ/ynn/AO/Rq1o=</latexit> w2 w2 ⌘ @L @w2 <latexit sha1_base64="XDVYNpwB7UZogd7iSX6oklwpHpw=">AAACMXicbVDLSgNBEJz1bXxFPXoZDIIXw64oehS9ePAQwZhANoTeSa8Ozj6Y6TWEJV/iZ/gFXvULchPBkz/hbAz4iA0DNVXV0z0VpEoact2hMzU9Mzs3v7BYWlpeWV0rr29cmyTTAusiUYluBmBQyRjrJElhM9UIUaCwEdydFXrjHrWRSXxF/RTbEdzEMpQCyFKd8mGvs+8rDAm0Tnrc3vZ8JPBDDSL3U9AkQfGLwTe2lkGnXHGr7qj4JPDGoMLGVeuU3/1uIrIIYxIKjGl5bkrtvHhSKByU/MxgCuIObrBlYQwRmnY++t6A71imy8NE2xMTH7E/O3KIjOlHgXVGQLfmr1aQ/2mtjMLjdi7jNCOMxdegMFOcEl5kxbtSoyDVtwCElnZXLm7BJkM20V9TuqZYrcjF+5vCJLjer3pu1bs8qJycjhNaYFtsm+0yjx2xE3bOaqzOBHtgT+yZvTiPztB5dd6+rFPOuGeT/Srn4xP07atd</latexit> <latexit sha1_base64="XDVYNpwB7UZogd7iSX6oklwpHpw=">AAACMXicbVDLSgNBEJz1bXxFPXoZDIIXw64oehS9ePAQwZhANoTeSa8Ozj6Y6TWEJV/iZ/gFXvULchPBkz/hbAz4iA0DNVXV0z0VpEoact2hMzU9Mzs3v7BYWlpeWV0rr29cmyTTAusiUYluBmBQyRjrJElhM9UIUaCwEdydFXrjHrWRSXxF/RTbEdzEMpQCyFKd8mGvs+8rDAm0Tnrc3vZ8JPBDDSL3U9AkQfGLwTe2lkGnXHGr7qj4JPDGoMLGVeuU3/1uIrIIYxIKjGl5bkrtvHhSKByU/MxgCuIObrBlYQwRmnY++t6A71imy8NE2xMTH7E/O3KIjOlHgXVGQLfmr1aQ/2mtjMLjdi7jNCOMxdegMFOcEl5kxbtSoyDVtwCElnZXLm7BJkM20V9TuqZYrcjF+5vCJLjer3pu1bs8qJycjhNaYFtsm+0yjx2xE3bOaqzOBHtgT+yZvTiPztB5dd6+rFPOuGeT/Srn4xP07atd</latexit> <latexit sha1_base64="XDVYNpwB7UZogd7iSX6oklwpHpw=">AAACMXicbVDLSgNBEJz1bXxFPXoZDIIXw64oehS9ePAQwZhANoTeSa8Ozj6Y6TWEJV/iZ/gFXvULchPBkz/hbAz4iA0DNVXV0z0VpEoact2hMzU9Mzs3v7BYWlpeWV0rr29cmyTTAusiUYluBmBQyRjrJElhM9UIUaCwEdydFXrjHrWRSXxF/RTbEdzEMpQCyFKd8mGvs+8rDAm0Tnrc3vZ8JPBDDSL3U9AkQfGLwTe2lkGnXHGr7qj4JPDGoMLGVeuU3/1uIrIIYxIKjGl5bkrtvHhSKByU/MxgCuIObrBlYQwRmnY++t6A71imy8NE2xMTH7E/O3KIjOlHgXVGQLfmr1aQ/2mtjMLjdi7jNCOMxdegMFOcEl5kxbtSoyDVtwCElnZXLm7BJkM20V9TuqZYrcjF+5vCJLjer3pu1bs8qJycjhNaYFtsm+0yjx2xE3bOaqzOBHtgT+yZvTiPztB5dd6+rFPOuGeT/Srn4xP07atd</latexit> <latexit sha1_base64="XDVYNpwB7UZogd7iSX6oklwpHpw=">AAACMXicbVDLSgNBEJz1bXxFPXoZDIIXw64oehS9ePAQwZhANoTeSa8Ozj6Y6TWEJV/iZ/gFXvULchPBkz/hbAz4iA0DNVXV0z0VpEoact2hMzU9Mzs3v7BYWlpeWV0rr29cmyTTAusiUYluBmBQyRjrJElhM9UIUaCwEdydFXrjHrWRSXxF/RTbEdzEMpQCyFKd8mGvs+8rDAm0Tnrc3vZ8JPBDDSL3U9AkQfGLwTe2lkGnXHGr7qj4JPDGoMLGVeuU3/1uIrIIYxIKjGl5bkrtvHhSKByU/MxgCuIObrBlYQwRmnY++t6A71imy8NE2xMTH7E/O3KIjOlHgXVGQLfmr1aQ/2mtjMLjdi7jNCOMxdegMFOcEl5kxbtSoyDVtwCElnZXLm7BJkM20V9TuqZYrcjF+5vCJLjer3pu1bs8qJycjhNaYFtsm+0yjx2xE3bOaqzOBHtgT+yZvTiPztB5dd6+rFPOuGeT/Srn4xP07atd</latexit> @L @w2 = @L @yp @yp @w2 = 1 NO 2 (yp y) hr @L @w1 = @L @yp @yp @hr @hr @w1 = 1 NO 2 (yp y) w2x L = 1 NO X (yp y) 2 00_numpy.py
- 19. PyTorch の導入 import torch epochs= 300 batch_size = 32 D_in = 784 H = 100 D_out = 10 learning_rate = 1.0e-06 # create random input and output data x = torch.randn(batch_size, D_in) y = torch.randn(batch_size, D_out) # randomly initialize weights w1 = torch.randn(D_in, H) w2 = torch.randn(H, D_out) for epoch in range(epochs): # forward pass: compute predicted y h = x.mm(w1) h_r = h.clamp(min=0) y_p = h_r.mm(w2) # compute and print loss loss = (y_p - y).pow(2).sum().item() print(t, loss) # backward pass: compute gradients of loss with respect to w2 grad_y_p = 2.0 * (y_p - y) grad_w2 = h_r.t().mm(grad_y_p) # backward pass: compute gradients of loss with respect to w1 grad_h_r = grad_y_p.mm(w2.t()) grad_h = grad_h_r.clone() grad_h[h < 0] = 0 grad_w1 = x.t().mm(grad_h) # update weights w1 -= learning_rate * grad_w1 w2 -= learning_rate * grad_w2 np.random torch np torch x.dot(w1) x.mm(w1) np.maximum(h, 0) h.clamp(min=0) np.square(y_p-y) (y_p-y).pow(2) copy() clone() 01_tensors.py
- 20. 自動微分の導入 # randomly initializeweights w1 = torch.randn(D_in, H) w2 = torch.randn(H, D_out) for epoch in range(epochs): # forward pass: compute predicted y h = x.mm(w1) h_r = h.clamp(min=0) y_p = h_r.mm(w2) # compute and print loss loss = (y_p - y).pow(2).sum().item() print(t, loss) # backward pass: compute gradients of loss with respect to w2 grad_y_p = 2.0 * (y_p - y) grad_w2 = h_r.t().mm(grad_y_p) # backward pass: compute gradients of loss with respect to w1 grad_h_r = grad_y_p.mm(w2.t()) grad_h = grad_h_r.clone() grad_h[h < 0] = 0 grad_w1 = x.t().mm(grad_h) # update weights w1 -= learning_rate * grad_w1 w2 -= learning_rate * grad_w2 01_tensor.py 02_autograd.py # randomly initialize weights w1 = torch.randn(D_in, H, requires_grad=True) w2 = torch.randn(H, D_out, requires_grad=True) for epoch in range(epochs): # forward pass: compute predicted y h = x.mm(w1) h_r = h.clamp(min=0) y_p = h_r.mm(w2) # compute and print loss loss = (y_p - y).pow(2).sum() print(t, loss.item()) # backward pass loss.backward() with torch.no_grad(): # update weights w1 -= learning_rate * w1.grad w2 -= learning_rate * w2.grad # initialize weights w1.grad.zero_() w2.grad.zero_() @L @w1 = @L @yp @yp @hr @hr @w1 = 1 NO 2(yp y)w2x <latexit sha1_base64="V1OkoDW7pmfxcKKULrjvVELuV8s=">AAACmHicfVHdSsMwGE3r//ybeqc3wSHMC0dTBL0Rhl6oIDrFqbCNkmbpFkx/SFJnKX0TX8wH8D1Mt8JcFT8InJzzfTkfJ27EmVSW9WmYc/MLi0vLK5XVtfWNzerW9pMMY0Fom4Q8FC8ulpSzgLYVU5y+RIJi3+X02X29yPXnNyokC4NHlUS05+NBwDxGsNKUU/3oegKTtBthoRjm8Cab4pGDsrN/9MSJspKcU9Pb0BHlhpyadagUFihLb527zK7rN46Sw5FjvzvVmtWwxgV/A1SAGiiq5VS/uv2QxD4NFOFYyg6yItVLczfCaVbpxpJGmLziAe1oGGCfyl46TjGDB5rpQy8U+gQKjtmfEyn2pUx8V3f6WA1lWcvJv7ROrLzTXsqCKFY0IBMjL+ZQhTD/EthnghLFEw0wEUzvCskQ60yU/rgZl77MV8t0Lqicwm/wZDeQ1UD3x7XmeZHQMtgD+6AOEDgBTXAFWqANiGEadQMZtrlrNs1L83rSahrFzA6YKfPhG92OzlM=</latexit> <latexit sha1_base64="V1OkoDW7pmfxcKKULrjvVELuV8s=">AAACmHicfVHdSsMwGE3r//ybeqc3wSHMC0dTBL0Rhl6oIDrFqbCNkmbpFkx/SFJnKX0TX8wH8D1Mt8JcFT8InJzzfTkfJ27EmVSW9WmYc/MLi0vLK5XVtfWNzerW9pMMY0Fom4Q8FC8ulpSzgLYVU5y+RIJi3+X02X29yPXnNyokC4NHlUS05+NBwDxGsNKUU/3oegKTtBthoRjm8Cab4pGDsrN/9MSJspKcU9Pb0BHlhpyadagUFihLb527zK7rN46Sw5FjvzvVmtWwxgV/A1SAGiiq5VS/uv2QxD4NFOFYyg6yItVLczfCaVbpxpJGmLziAe1oGGCfyl46TjGDB5rpQy8U+gQKjtmfEyn2pUx8V3f6WA1lWcvJv7ROrLzTXsqCKFY0IBMjL+ZQhTD/EthnghLFEw0wEUzvCskQ60yU/rgZl77MV8t0Lqicwm/wZDeQ1UD3x7XmeZHQMtgD+6AOEDgBTXAFWqANiGEadQMZtrlrNs1L83rSahrFzA6YKfPhG92OzlM=</latexit> <latexit sha1_base64="V1OkoDW7pmfxcKKULrjvVELuV8s=">AAACmHicfVHdSsMwGE3r//ybeqc3wSHMC0dTBL0Rhl6oIDrFqbCNkmbpFkx/SFJnKX0TX8wH8D1Mt8JcFT8InJzzfTkfJ27EmVSW9WmYc/MLi0vLK5XVtfWNzerW9pMMY0Fom4Q8FC8ulpSzgLYVU5y+RIJi3+X02X29yPXnNyokC4NHlUS05+NBwDxGsNKUU/3oegKTtBthoRjm8Cab4pGDsrN/9MSJspKcU9Pb0BHlhpyadagUFihLb527zK7rN46Sw5FjvzvVmtWwxgV/A1SAGiiq5VS/uv2QxD4NFOFYyg6yItVLczfCaVbpxpJGmLziAe1oGGCfyl46TjGDB5rpQy8U+gQKjtmfEyn2pUx8V3f6WA1lWcvJv7ROrLzTXsqCKFY0IBMjL+ZQhTD/EthnghLFEw0wEUzvCskQ60yU/rgZl77MV8t0Lqicwm/wZDeQ1UD3x7XmeZHQMtgD+6AOEDgBTXAFWqANiGEadQMZtrlrNs1L83rSahrFzA6YKfPhG92OzlM=</latexit> <latexit sha1_base64="V1OkoDW7pmfxcKKULrjvVELuV8s=">AAACmHicfVHdSsMwGE3r//ybeqc3wSHMC0dTBL0Rhl6oIDrFqbCNkmbpFkx/SFJnKX0TX8wH8D1Mt8JcFT8InJzzfTkfJ27EmVSW9WmYc/MLi0vLK5XVtfWNzerW9pMMY0Fom4Q8FC8ulpSzgLYVU5y+RIJi3+X02X29yPXnNyokC4NHlUS05+NBwDxGsNKUU/3oegKTtBthoRjm8Cab4pGDsrN/9MSJspKcU9Pb0BHlhpyadagUFihLb527zK7rN46Sw5FjvzvVmtWwxgV/A1SAGiiq5VS/uv2QxD4NFOFYyg6yItVLczfCaVbpxpJGmLziAe1oGGCfyl46TjGDB5rpQy8U+gQKjtmfEyn2pUx8V3f6WA1lWcvJv7ROrLzTXsqCKFY0IBMjL+ZQhTD/EthnghLFEw0wEUzvCskQ60yU/rgZl77MV8t0Lqicwm/wZDeQ1UD3x7XmeZHQMtgD+6AOEDgBTXAFWqANiGEadQMZtrlrNs1L83rSahrFzA6YKfPhG92OzlM=</latexit> 微分を自動的に計算してくれる
- 21. 活性化関数の自作 03_function.py import torch forepoch in range(epochs): # forward pass: compute predicted y h = x.mm(w1) h_r = h.clamp(min=0) y_p = h_r.mm(w2) 02_autograd.py import torch class ReLU(torch.autograd.Function): @staticmethod def forward(ctx, input): ctx.save_for_backward(input) return input.clamp(min=0) @staticmethod def backward(ctx, grad_output): input, = ctx.saved_tensors grad_input = grad_output.clone() grad_input[input<0] = 0 return grad_input for epoch in range(epochs): # forward pass: compute predicted y relu = ReLU.apply h = x.mm(w1) h_r = relu(h) y_p = h_r.mm(w2) . . . . . . y=f(x) ReLU (Rectified Linear Unit)
- 22. torch.nnの利用 04_nn_module.py # createrandom input and output data x = torch.randn(batch_size, D_in) y = torch.randn(batch_size, D_out) # randomly initialize weights w1 = torch.randn(D_in, H, requires_grad=True) w2 = torch.randn(H, D_out, requires_grad=True) for epoch in range(epochs): # forward pass: compute predicted y h = x.mm(w1) h_r = h.clamp(min=0) y_p = h_r.mm(w2) # compute and print loss loss = (y_p - y).pow(2).sum() print(t, loss.item()) # backward pass loss.backward() with torch.no_grad(): # update weights w1 -= learning_rate * w1.grad w2 -= learning_rate * w2.grad # initialize weights w1.grad.zero_() w2.grad.zero_() 02_autograd.py # create random input and output data x = torch.randn(batch_size, D_in) y = torch.randn(batch_size, D_out) # define model model = torch.nn.Sequential( torch.nn.Linear(D_in, H), torch.nn.ReLU(), torch.nn.Linear(H, D_out), ) # define loss function criterion = torch.nn.MSELoss(reduction='sum') for epoch in range(epochs): # forward pass: compute predicted y y_p = model(x) # compute and print loss loss = criterion(y_p, y) print(t, loss.item()) # backward pass model.zero_grad() loss.backward() with torch.no_grad(): # update weights for param in model.parameters(): param -= learning_rate * param.grad
- 23. 最適化関数の呼び出し 05_optimizer.py 04_nn_module.py # defineloss function criterion = torch.nn.MSELoss(reduction='sum') for t in range(epochs): # forward pass: compute predicted y y_p = model(x) # compute and print loss loss = criterion(y_p, y) print(t, loss.item()) # backward pass model.zero_grad() loss.backward() with torch.no_grad(): # update weights for param in model.parameters(): param -= learning_rate * param.grad # define loss function criterion = torch.nn.MSELoss(reduction='sum') # define optimizer optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) for epoch in range(epochs): # forward pass: compute predicted y y_p = model(x) # compute and print loss loss = criterion(y_p, y) print(t, loss.item()) # backward pass optimizer.zero_grad() loss.backward() # update weights optimizer.step()
- 24. モデルを自作 06_mm_module.py 05_optimizer.py # createrandom input and output data x = torch.randn(batch_size, D_in) y = torch.randn(batch_size, D_out) # define model model = torch.nn.Sequential( torch.nn.Linear(D_in, H), torch.nn.ReLU(), torch.nn.Linear(H, D_out), ) # define loss function criterion = torch.nn.MSELoss(reduction='sum') import torch.nn as nn import torch.nn.functional as F class TwoLayerNet(nn.Module): def __init__(self, D_in, H, D_out): super(TwoLayerNet, self).__init__() self.fc1 = nn.Linear(D_in, H) self.fc2 = nn.Linear(H, D_out) def forward(self, x): h = self.fc1(x) h_r = F.relu(h) y_p = self.fc2(h_r) return y_p # create random input and output data x = torch.randn(batch_size, D_in) y = torch.randn(batch_size, D_out) # define model model = TwoLayerNet(D_in, H, D_out) # define loss function criterion = nn.MSELoss(reduction='sum') . . . 学習時に不変
- 25. MNIST Datasetのロード 07_mnist.py 06_mm_module.py importtorch.nn as nn import torch.nn.functional as F from torchvision import datasets, transforms # read input data and labels train_dataset = datasets.MNIST('./data', train=True, download=True, transform=transforms.ToTensor()) train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True) for epoch in range(epochs): # Set model to training mode model.train() # Loop over each batch from the training set for batch_idx, (x, y) in enumerate(train_loader): # forward pass: compute predicted y y_p = model(x) . . . import torch.nn as nn import torch.nn.functional as F # create random input and output data x = torch.randn(batch_size, D_in) y = torch.randn(batch_size, D_out) for t in range(epochs): # forward pass: compute predicted y y_p = model(x) . . . . . .
- 26. Validationデータによる検証 08_validate.py def validate(): model.eval() val_loss, val_acc= 0, 0 for data, target in val_loader: output = model(data) loss = criterion(output, target) val_loss += loss.item() pred = output.data.max(1)[1] val_acc += 100. * pred.eq(target.data).cpu().sum() / target.size(0) val_loss /= len(val_loader) val_acc /= len(val_loader) print('nValidation set: Average loss: {:.4f}, Accuracy: {:.1f}%n'.format( val_loss, val_acc)) 学習時に使うデータ ハイパラやモデル を変えて試すとき に使うデータ 最終的な精度の評価 に使うデータ Validation dataのloss 予測クラスがラベルと一致しているか？ パーセンテージに変換 sum()はGPUでやると遅いのでCPUで
- 27. train(), main()関数の形で書く 09_train.py def train(train_loader,model,criterion,optimizer,epoch): model.train() t= time.perf_counter() for batch_idx, (data, target) in enumerate(train_loader): output = model(data) loss = criterion(output, target) optimizer.zero_grad() loss.backward() optimizer.step() if batch_idx % 200 == 0: print('Train Epoch: {} [{:>5}/{} ({:.0%})]tLoss: {:.6f}t Time:{:.4f}'.format( epoch, batch_idx * len(data), len(train_loader.dataset), batch_idx / len(train_loader), loss.data.item(), time.perf_counter() - t)) t = time.perf_counter() def main(): epochs = 10 batch_size = 32 learning_rate = 1.0e-02 train_dataset = datasets.MNIST('./data', train=True, download=True, transform=transforms.ToTensor()) val_dataset = datasets.MNIST('./data', train=False, transform=transforms.ToTensor()) train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True) val_loader = torch.utils.data.DataLoader(dataset=validation_dataset, batch_size=batch_size, shuffle=False) model = TwoLayerNet(D_in, H, D_out) criterion = nn.CrossEntropyLoss() optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) for epoch in range(epochs): model.train() train(train_loader,model,criterion,optimizer,epoch) validate(val_loader,model,criterion)
- 28. 畳み込みNNモデル 10_cnn.py 09_train.py class CNN(nn.Module): def__init__(self): super(CNN, self).__init__() self.conv1 = nn.Conv2d(1, 32, 3, 1) self.conv2 = nn.Conv2d(32, 64, 3, 1) self.dropout1 = nn.Dropout2d(0.25) self.dropout2 = nn.Dropout2d(0.5) self.fc1 = nn.Linear(9216, 128) self.fc2 = nn.Linear(128, 10) def forward(self, x): x = self.conv1(x) x = F.relu(x) x = self.conv2(x) x = F.relu(x) x = F.max_pool2d(x, 2) x = self.dropout1(x) x = torch.flatten(x, 1) x = self.fc1(x) x = F.relu(x) x = self.dropout2(x) x = self.fc2(x) output = F.log_softmax(x, dim=1) return output class TwoLayerNet(nn.Module): def __init__(self, D_in, H, D_out): super(TwoLayerNet, self).__init__() self.fc1 = nn.Linear(D_in, H) self.fc2 = nn.Linear(H, D_out) def forward(self, x): x = x.view(-1, D_in) h = self.fc1(x) h_r = F.relu(h) y_p = self.fc2(h_r) return F.log_softmax(y_p, dim=1)
- 29. GPUを利用 11_gpu.py device = torch.device('cuda') model= CNN().to(device) def train(train_loader,model,criterion,optimizer,epoch): model.train() t = time.perf_counter() for batch_idx, (data, target) in enumerate(train_loader): data = data.to(device) target = target.to(device) def validate(loss_vector, accuracy_vector): model.eval() val_loss, correct = 0, 0 for data, target in validation_loader: data = data.to(device) target = target.to(device) . . . . . . . . . PyTorchは裏でcuDNNを呼んでいる 1. torch.device(‘cuda’)でデバイスを指定 2. data, targetをデバイスに送る 3. 計算は全て自動的にGPUを用いて行われる
- 30. 分散並列 12_distributed.py import os import torch importtorch.distributed as dist master_addr = os.getenv("MASTER_ADDR", default="localhost") master_port = os.getenv('MASTER_PORT', default='8888') method = "tcp://{}:{}".format(master_addr, master_port) rank = int(os.getenv('OMPI_COMM_WORLD_RANK', '0')) world_size = int(os.getenv('OMPI_COMM_WORLD_SIZE', '1')) dist.init_process_group("nccl", init_method=method, rank=rank, world_size=world_size) print('Rank: {}, Size: {}'.format(dist.get_rank(),dist.get_world_size())) ngpus = 4 device = rank % ngpus x = torch.randn(1).to(device) print('rank {}: {}'.format(rank, x)) dist.broadcast(x, src=0) print('rank {}: {}'.format(rank, x)) 通信に用いるホストアドレスとポート番号を指定 OpenMPI環境変数からrankとsizeを取得 PyTorchにこれらを設定 PyTorchによる集団通信 .bashrcに以下を記入 if [ -f "$SGE_JOB_SPOOL_DIR/pe_hostfile" ]; then export MASTER_ADDR=`head -n 1 $SGE_JOB_SPOOL_DIR/pe_hostfile | cut -d " " -f 1` fi mpirun -np 4 python 12_distributed.py
- 31. 分散並列MNIST 13_ddp.py def print0(message): if torch.distributed.is_initialized(): iftorch.distributed.get_rank() == 0: print(message, flush=True) else: print(message, flush=True) train_sampler = torch.utils.data.distributed.DistributedSampler( train_dataset, num_replicas=torch.distributed.get_world_size(), rank=torch.distributed.get_rank()) model = DDP(model, device_ids=[rank]) . . . . . . 全プロセスがprintすると見づらいので１プロセスだけprintするようなprint関数を定義 train dataの読み込みで異なるプロセスが異なるデータを読むようにする モデルをDDP()に通すことで分散並列計算を行う
- 32. Argparse 14_args.py import argparse import torch importtorch.distributed as dist import torch.nn as nn parser = argparse.ArgumentParser(description='PyTorch MNIST Example') parser.add_argument('--batch-size', type=int, default=32, metavar='N', help='input batch size for training (default: 32)') parser.add_argument('--epochs', type=int, default=10, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=1.0e-02, metavar='LR', help='learning rate (default: 1.0e-02)') args = parser.parse_args() epochs = args.epochs batch_size = args.batch_size learning_rate = args.lr * world_size 直接数字を入れていたところをargsの変数を入れられる https://docs.python.org/ja/3/library/argparse.html#action
- 33. AverageMeter 15_meter.py def train(train_loader,model,criterion,optimizer,epoch,device): batch_time =AverageMeter('Time', ':.4f') train_loss = AverageMeter('Loss', ':.6f') class AverageMeter(object): def __init__(self, name, fmt=':f'): self.name = name self.fmt = fmt self.reset() def reset(self): self.val = 0 self.avg = 0 self.sum = 0 self.count = 0 def update(self, val, n=1): self.val = val self.sum += val * n self.count += n self.avg = self.sum / self.count def __str__(self): fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' return fmtstr.format(**self.__dict__) valが既にn個の平均の場合 値 平均 和 個数 出力形式
- 34. ProgressMeter 15_meter.py def train(train_loader,model,criterion,optimizer,epoch,device): batch_time =AverageMeter('Time', ':.4f') train_loss = AverageMeter('Loss', ‘:.6f') progress = ProgressMeter( len(train_loader), [train_loss, batch_time], prefix="Epoch: [{}]".format(epoch)) class ProgressMeter(object): def __init__(self, num_batches, meters, prefix="", postfix=""): self.batch_fmtstr = self._get_batch_fmtstr(num_batches) self.meters = meters self.prefix = prefix self.postfix = postfix def display(self, batch): entries = [self.prefix + self.batch_fmtstr.format(batch)] entries += [str(meter) for meter in self.meters] entries += self.postfix print0('t'.join(entries)) def _get_batch_fmtstr(self, num_batches): num_digits = len(str(num_batches // 1)) fmt = '{:' + str(num_digits) + 'd}' return '[' + fmt + '/' + fmt.format(num_batches) + ']' 前にprintするもの 後にprintするもの printしたい変数 printしたいものを連結 [ 今のbatch / 全batch数 ] のような表示をしたい
- 35. Weights and Biases pipinstall wandb wandb login import wandb os.environ['MASTER_ADDR'] = 'localhost' os.environ['MASTER_PORT'] = '8888' rank = int(os.getenv('OMPI_COMM_WORLD_RANK', '0')) world_size = int(os.getenv('OMPI_COMM_WORLD_SIZE', '1')) dist.init_process_group("nccl", rank=rank, world_size=world_size) device = torch.device('cuda',rank) if torch.distributed.get_rank() == 0: wandb.init(project="example-project") wandb.config.update(args) epochs = args.epochs batch_size = args.batch_size learning_rate = args.lr * world_size for epoch in range(epochs): model.train() train_loss, train_acc = train(train_loader,model,criterion,optimizer,epoch,device) val_loss, val_acc = validate(val_loader,model,criterion,device) if torch.distributed.get_rank() == 0: wandb.log({ 'train_loss': train_loss, 'train_acc': train_acc, 'val_loss': val_loss, 'val_acc': val_acc }) wandbで記録したい変数 trainとvalidateがlossとaccuracyを返すようにする wandbの初期化 argsを渡すと実験条件を勝手に記録してくれる 16_wandb.py
- 36. train_dataset = datasets.CIFAR10('./data', train=True, download=True, transform=transforms.ToTensor()) val_dataset= datasets.CIFAR10('./data', train=False, download=True, transform=transforms.ToTensor()) CIFAR10 17_cifar10.py model = VGG('VGG19').to(device) model = DDP(model, device_ids=[rank % 4]) criterion = nn.CrossEntropyLoss() optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) データセットの変更 モデルの変更
- 37. transform_train = transforms.Compose([ transforms.RandomCrop(32,padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_val = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) データ拡張 18_augmentation.py 輝度値の正規化
- 38. parser.add_argument('--momentum', type=float, default=0.9,metavar='M', help='momentum (default: 0.9)') parser.add_argument('--wd', '--weight_decay', type=float, default=5.0e-04, metavar='W', help='learning rate (default: 5.0e-04)') 正則化 19_regularization.py optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.wd) <latexit sha1_base64="GBuDE2XllGcgcaw34c+lteFIv9Q=">AAACJnicbVDLSsNAFJ34rO+oSzeDRRCFkhRFN0LRjcsKVoWmlsnkxg5OHszcCCXNP/gZfoFb/QJ3Iu7c+B9OHwtrPTBwOOce7p3jp1JodJxPa2p6ZnZuvrSwuLS8srpmr29c6SRTHBo8kYm68ZkGKWJooEAJN6kCFvkSrv37s75//QBKiyS+xG4KrYjdxSIUnKGR2vaeh0IGkMviRO57oWI896SJB6zIq0Wv52EHkNFe77batstOxRmAThJ3RMpkhHrb/vaChGcRxMgl07rpOim2cqZQcAnFopdpSBm/Z3fQNDRmEehWPvhTQXeMEtAwUebFSAfq70TOIq27kW8mI4Yd/dfri/95zQzD41Yu4jRDiPlwUZhJigntF0QDoYCj7BrCuBLmVso7zBSDpsaxLYHun1aYXty/LUySq2rFPaw4Fwfl2umooRLZIttkl7jkiNTIOamTBuHkkTyTF/JqPVlv1rv1MRydskaZTTIG6+sH2qSnTw==</latexit> ˜ l = l + 2 ||✓||2 <latexit sha1_base64="aTi3bmxUDeM1ohqP26MgoCugRRA=">AAACI3icbVDLSgMxFM34tr6qLt0EiygIZUYU3QhFNy4VrAqdUu5kbm0wkxmSO0IZ+gl+hl/gVr/Anbhx4dL/MH0stHogcHLOvbk3J8qUtOT7H97E5NT0zOzcfGlhcWl5pby6dmXT3Aisi1Sl5iYCi0pqrJMkhTeZQUgihdfR3Wnfv75HY2WqL6mbYTOBWy3bUgA5qVXeDjVECkKSKsZC9Y6Hd652Q+VeiYGH1EGCVrniV/0B+F8SjEiFjXDeKn+FcSryBDUJBdY2Aj+jZgGGpFDYK4W5xQzEHdxiw1ENCdpmMfhQj285Jebt1LijiQ/Unx0FJNZ2k8hVJkAdO+71xf+8Rk7to2YhdZYTajEc1M4Vp5T30+GxNChIdR0BYaTblYsOGBDkMvw1Jbb91Xoul2A8hb/kaq8aHFT9i/1K7WSU0BzbYJtshwXskNXYGTtndSbYA3tiz+zFe/RevTfvfVg64Y161tkveJ/fX8Glaw==</latexit> r˜ l = rl + ✓ <latexit sha1_base64="+SsjknKlypA8fgkVhF8vJQYEV3M=">AAACPnicbVDLSgMxFM34rO+qSzfBIlSkZUYU3QhFNy4VrAqdUu5k0jaYyQzJHaEM/R8/wy9wq36A7sStS9M6gm29EHJyzn3lBIkUBl331Zmanpmdmy8sLC4tr6yuFdc3rk2casbrLJaxvg3AcCkUr6NAyW8TzSEKJL8J7s4G+s0910bE6gp7CW9G0FGiLRigpVrFUx+7HKGV4Z7XP8kfWPHt5SsIJFBZ/mV3aYUOBWn7h/BLt4olt+oOg04CLwclksdFq/jmhzFLI66QSTCm4bkJNjPQKJjk/UU/NTwBdgcd3rBQQcRNMxv+tU93LBPSdqztUUiH7N+KDCJjelFgMyPArhnXBuR/WiPF9nEzEypJkSv2M6idSooxHRhHQ6E5Q9mzAJgWdlfKuqCBobV3ZEpoBqv1rS/euAuT4Hq/6h1W3cuDUu00d6hAtsg2KROPHJEaOScXpE4YeSBP5Jm8OI/Ou/PhfP6kTjl5zSYZCefrG44ksA8=</latexit> ✓t+1 = ✓t ⌘rl(✓t) ⌘ ✓t Momentum L2 正則化
- 39. Sweep sweep.yaml program: wrapper.py method: grid metric: goal:minimize name: val_loss parameters: epochs: values: [100] batch_size: values: [32] learning_rate: values: [0.005, 0.01, 0.02, 0.05, 0.1] momentum: values: [0.85, 0.9, 0.95] weight_decay: values: [1.0e-4, 2.0e-4, 5.0e-4, 1.0e-3, 2.0e-3] wandb sweep sweep.yaml
- 40. Models 19_regularization.py model = VGG('VGG19').to(device) #model = ResNet18().to(device) # model = PreActResNet18().to(device) # model = GoogLeNet().to(device) # model = DenseNet121().to(device) # model = ResNeXt29_2x64d().to(device) # model = MobileNet().to(device) # model = MobileNetV2().to(device) # model = DPN92().to(device) # model = ShuffleNetG2().to(device) # model = SENet18().to(device) # model = ShuffleNetV2(1).to(device) # model = EfficientNetB0().to(device) # model = RegNetX_200MF().to(device) 今はこれを使っている 他のモデルも試して見ましょう