Recommended
More Related Content
What's hot
What's hot (12)
これからの強化学習 1章
- 2. • • • • •
- 3. • • • • •
- 4. • •
- 5. • •
- 6. • • • • •
- 7. •
- 8. • • • •
- 9. • •
- 10. • • •
- 11. • • •
- 12. 𝐾 𝑖
- 13. 𝐾 𝑅 𝑖(𝑖 = 1, … , 𝐾) 𝑝+
- 14. • 𝑝+ 𝑖 𝑅𝑝+ 𝑅𝑝+ 𝑖
- 15. • • n • µ. = i i • µ.
- 16. • 𝑛 𝑛 • n • µ. = i i • µ.
- 17. • 𝑛 𝑛 • n • µ. = i i • µ.
- 19. • • • • • µ.
- 20. • • • •
- 21. • 𝐾 • r345 • r345 K µ. 7 = i + Kr345 i + K • µ.
- 22. • 𝑖 • R • • i µ. = i i • i U. = R 2 ln( ) i • x. = µ. + U. i
- 23. • 𝜇+ 𝑈+ 𝑖 • R • • i µ. = i i • i U. = R 2 ln( ) i • x. = µ. + U. i
- 24. •
- 25. •
- 26. • ε-greedy のグラフ
- 27. •
- 28. • • • • •
- 29. • 𝒮 = {𝑠D, 𝑠E, … , 𝑠F} • 𝑡 𝑆J • 𝑆J 𝑠D, … , 𝑠F • 𝒜(𝑠) 𝑠 • 𝑠 𝒜 𝑠 = {𝑎D, 𝑎E, … , 𝑎M} • 𝑆J 𝐴J • 𝐴J 𝑎D, … , 𝑎M • ℛ 𝑆J, 𝐴J, 𝑆JPD 𝑅JPD
- 30. • 𝑆Q ~ 𝑃Q 𝑠 𝑆Q 𝑃Q 𝑆Q • • 𝑠 𝑎 𝑠′ 𝑃(𝑠7|𝑠, 𝑎) • 𝑡 + 1 𝑆JPD 𝑆J 𝐴J 𝑆JPD ~ 𝑃(𝑠7|𝑆J, 𝐴J)
- 31. 𝑆JPD ~ 𝑃(𝑠7|𝑆J, 𝐴J) • 𝑆JPD 𝑆J, 𝐴J 𝑆JVD, 𝐴JVD • 𝑅JPD 𝑆J 𝐴J 𝑆JPD 𝑅JPD = 𝑟 𝑆J, 𝐴J, 𝑆JPD • 𝜋 𝑠 𝑎 𝜋(𝑎|𝑠)
- 32. 3Z
- 33. • • • • • • • 𝑃Q 𝑠 = [ 1 (𝑠 = 𝑠D) 0 (otherwise)
- 34. • •
- 36. • 𝐺J = d 𝑅JPDPe fVD egQ • 𝐺J = lim f→j 1 𝑇 d 𝑅JPDPe fVD egQ • 𝐺J = d 𝛾e 𝑅JPDPe j egQ = 𝑅JPD + 𝛾𝑅JPE + 𝛾E 𝑅JPm + ⋯ 0 ≤ 𝛾 ≤ 1
- 37. • • 𝑉q 𝑠 = 𝔼q[𝐺J|𝑆J = 𝑠] 𝑡 𝑠 𝜋
- 38. • 𝐺J = 𝑅JPD 𝑆J 𝑎 𝜋(𝑎|𝑆J) 𝑡 + 1 𝑠′ 𝑃 𝑆JPD = 𝑠7, 𝐴J = 𝑎 𝑆J = 𝑠 = 𝑃 𝑆JPD = 𝑠7 𝑆J = 𝑠, 𝐴J = 𝑎 𝜋(𝑎|𝑠) 𝑉q 𝑠 = 𝔼q 𝐺J 𝑆J = 𝑠 = d d 𝑃 𝑆JPD = 𝑠7, 𝐴J = 𝑎 𝑆J = 𝑠 𝑟 𝑠, 𝑎, 𝑠7 u∈𝒜 wwx∈𝒮 = d d 𝑃 𝑆JPD = 𝑠7 𝑆J = 𝑠, 𝐴J = 𝑎 𝜋 𝑎 𝑠 𝑟 𝑠, 𝑎, 𝑠7 u∈𝒜 wwx∈𝒮
- 39. • 𝑉q 𝑠 𝜋 • 𝜋∗ • 𝑉q∗ (𝑠) ∀𝑠 ∈ 𝒮, 𝑉∗ 𝑠 = 𝑉q∗ 𝑠 = max q 𝑉q(𝑠) • 𝑄q(𝑠, 𝑎) 𝑠 𝑎 𝑄q 𝑠, 𝑎 = 𝔼q[𝐺JPD|𝑆J = 𝑠, 𝐴J = 𝑎] • 𝑄∗(𝑠, 𝑎) 𝑄∗ 𝑠, 𝑎 = 𝑄q∗ 𝑠, 𝑎 = max q 𝑄q(𝑠, 𝑎)
- 40. • 𝜋 𝑎 𝑠 = } 1 (𝑎 = arg max u 𝑄(𝑠, 𝑎) 0 otherwise • 𝜋 𝑎 𝑠 = 1 − 𝜖 + 𝜖 |𝒜(𝑠)| (𝑎 = arg max u 𝑄(𝑠, 𝑎) 𝜖 |𝒜(𝑠)| otherwise • 𝜋 𝑎 𝑠 = exp(𝑄(𝑠, 𝑎)/𝑇) ∑ exp(𝑄(𝑠, 𝑏)/𝑇)…∈† • • • 𝑄
- 41. • • • • •
- 42. 𝜋 𝑉q(𝑠) 𝑉q 𝑠 = 𝔼q[𝐺J|𝑆J = 𝑠] 𝐺J = d 𝛾e 𝑅JPDPe j egQ = 𝑅JPD + 𝛾𝑅JPE + 𝛾E 𝑅JPm + ⋯ 𝑉q 𝑠 = 𝔼q d 𝛾e 𝑅JPDPe j egQ |𝑆J = 𝑠 = 𝔼q 𝑅JPD + 𝛾𝑅JPE + 𝛾E 𝑅JPm + ⋯ 𝑆J = 𝑠
- 43. 𝑉q 𝑠 = 𝔼q 𝐺J 𝑆J = 𝑠 = 𝔼q 𝑅JPD 𝑆J = 𝑠 + 𝔼q 𝛾𝑅JPE + 𝛾E 𝑅JPm + ⋯ 𝑆J = 𝑠 = 𝔼q 𝑅JPD 𝑆J = 𝑠 + 𝛾𝔼q 𝑅JPE + 𝛾𝑅JPm + ⋯ 𝑆J = 𝑠 𝔼q 𝑅JPD 𝑆J = 𝑠 = d 𝜋(𝑎|𝑠) d 𝑃 𝑠7 𝑠, 𝑎 𝑟(𝑠, 𝑎, 𝑠7) wx∈𝒮u∈𝒜(w) 𝔼q 𝑅JPE + 𝛾𝑅JPm + ⋯ 𝑆J = 𝑠 = d 𝜋 𝑎 𝑠 d 𝑃 𝑠7 𝑠, 𝑎 𝔼q 𝑅JPE + 𝛾𝑅JPm + ⋯ 𝑆JPD = 𝑠7 wx∈𝒮u∈𝒜 w = d 𝜋 𝑎 𝑠 d 𝑃 𝑠7 𝑠, 𝑎 𝑉q(𝑠7) wx∈𝒮u∈𝒜 w
- 44. 𝑉q 𝑠 = d 𝜋(𝑎|𝑠) d 𝑃 𝑠7 𝑠, 𝑎 {𝑟 𝑠, 𝑎, 𝑠7 + 𝛾𝑉q 𝑠7 } wx∈𝒮u∈𝒜(w) 𝑄q 𝑠, 𝑎 = d 𝑃 𝑠7 𝑠, 𝑎 𝑟 𝑠, 𝑎, 𝑠7 + 𝛾𝑉q 𝑠 wx∈𝒮 𝑉q 𝑠 = d 𝜋 𝑎 𝑠 𝑄q 𝑠, 𝑎 u∈𝒜 w 𝑄q 𝑠, 𝑎 = d 𝑃 𝑠7 𝑠, 𝑎 𝑟 𝑠, 𝑎, 𝑠7 + d 𝛾𝜋 𝑎7 𝑠7 𝑄q(𝑠7, 𝑎7) u∈𝒜(w7)wx∈𝒮
- 45. 𝑃 𝑉q 𝑠 = d 𝜋(𝑎|𝑠) d 𝑃 𝑠7 𝑠, 𝑎 {𝑟 𝑠, 𝑎, 𝑠7 + 𝛾𝑉q 𝑠7 } wx∈𝒮u∈𝒜(w) 𝑄q 𝑠, 𝑎 = d 𝑃 𝑠7 𝑠, 𝑎 𝑟 𝑠, 𝑎, 𝑠7 + d 𝛾𝜋 𝑎7 𝑠7 𝑄q(𝑠7, 𝑎7) u∈𝒜(w7)wx∈𝒮
- 46. 𝑆J 𝐴J 𝑅JPD 𝑆JPD 𝐴JPD 𝑄(𝑆J, 𝐴J) 𝑄 𝑆J, 𝐴J ← 1 − 𝛼 𝑄 𝑆J, 𝐴J + 𝛼 𝑅JPD + 𝛾𝑄 𝑆JPD, 𝐴JPD • 𝛼 •
- 47. 𝑄(𝑠, 𝑎) 𝑠 𝜋 𝑎 𝑎 𝑟 𝑠′ 𝑠′ 𝑎′ 𝑄 𝑆J, 𝐴J ← 1 − 𝛼 𝑄 𝑆J, 𝐴J + 𝛼 𝑅JPD + 𝛾𝑄 𝑆JPD, 𝐴JPD 𝑠 ← 𝑠7, 𝑎 ← 𝑎′ http://stlab.ssi.ist.hokudai.ac.jp/yuhyama/lecture/OLD/softcomputing/softcomputing-b-4up.pdf
- 48. Q s, a ← 1 − α Q s, a + α r′ + γQ s′, a′ ⇕ Q s, a ← Q s, a + α r7 + γQ s7 , a7 − Q(s, a) •
- 49. • 𝜋 • 𝑉∗ 𝑠 = max u∈𝒜 𝑄∗(𝑠, 𝑎) = max u∈𝒜 d 𝑃 𝑠7 𝑠, 𝑎 (𝑟 𝑠, 𝑎, 𝑠7 + 𝛾𝑉∗ 𝑠7 ) wx∈𝒮 𝑄∗ 𝑠, 𝑎 = d 𝑃(𝑠7|𝑠, 𝑎)(𝑟 𝑠, 𝑎, 𝑠7 + 𝛾 max ux∈𝒜(wx) 𝑄∗ 𝑠7, 𝑎7 ) wx∈𝒮 • 𝜋(𝑎|𝑠) • 𝜋∗ 𝑉q 𝑠 = d 𝜋 𝑎 𝑠 𝑄q 𝑠, 𝑎 u∈𝒜 w 𝑄q 𝑠, 𝑎 = d 𝑃 𝑠7 𝑠, 𝑎 𝑟 𝑠, 𝑎, 𝑠7 + d 𝛾𝜋 𝑎7 𝑠7 𝑄q(𝑠7, 𝑎7) u∈𝒜(w7)wx∈𝒮
- 50. • 𝑉∗ 𝑠 = max u∈𝒜 𝑄∗(𝑠, 𝑎) = max u∈𝒜 d 𝑃 𝑠7 𝑠, 𝑎 (𝑟 𝑠, 𝑎, 𝑠7 + 𝛾𝑉∗ 𝑠7 ) wx∈𝒮 𝑄∗ 𝑠, 𝑎 = d 𝑃(𝑠7|𝑠, 𝑎)(𝑟 𝑠, 𝑎, 𝑠7 + 𝛾 max ux∈𝒜(wx) 𝑄∗ 𝑠7, 𝑎7 ) wx∈𝒮 • • 𝑃
- 51. • 𝑄 𝑆J, 𝐴J ← 1 − 𝛼 𝑄 𝑆J, 𝐴J + 𝛼(𝑅JPD + 𝛾 max ux∈𝒜 •Ž•• 𝑄 𝑆JPD, 𝑎7 ) • 𝛼 •
- 52. • 𝑄 𝑆J, 𝐴J ← 1 − 𝛼 𝑄 𝑆J, 𝐴J + 𝛼(𝑅JPD + 𝛾 max ux∈𝒜 •Ž•• 𝑄 𝑆JPD, 𝑎7 ) • 𝑄 𝑆J, 𝐴J ← 1 − 𝛼 𝑄 𝑆J, 𝐴J + 𝛼 𝑅JPD + 𝛾𝑄 𝑆JPD, 𝐴JPD • • • •
- 53. Q s, a ← 1 − α Q s, a + α r′ + γ max ‘x∈𝒜(3x) Q s′, a′ ⇕ Q s, a ← Q s, a + α r7 + γ max ‘x∈𝒜(3x) Q s′, a′ − Q(s, a) • π Qを更新する時の状態 Qを更新する時の状態
- 54. Q s, a ← 1 − α Q s, a + α r′ + γQ s′, a′ ⇕ Q s, a ← Q s, a + α r7 + γQ s7 , a7 − Q(s, a) •
- 55. ▼ + 0 Q(s1, a1) Q(s1, a2) Q(s1, a2) Q(s1, a1) -50 3 5000 5000 報酬小 リスク小 報酬大 リスク大
- 56. • • • • •
- 58. • •
- 59. • •
- 60. • • 𝑠 = {𝑠D, 𝑠E} 𝑎 𝑎 = 𝑓 𝑠 = 𝜃D 𝑠D + 𝜃E 𝑠E 𝑎 𝑓(𝑠) 𝜃
- 61. • •
- 62. • 𝐽(𝜃) 𝜃 𝜃 𝜋” • 𝜃 ← 𝜃 + 𝛿𝜃 𝜃 ← 𝜃 + 𝜂𝛻” 𝐽(𝜃) 𝛻” 𝜃
- 63. • 𝜋” 𝜋” 𝜋”
- 64. 𝑝(𝑎|𝑠) • • 𝑝 𝑎 𝑠 = 𝜋” 𝑎 𝑠 • 𝜋” 𝜃 • 𝜋” 𝑎 𝑠 = exp 𝜃wu ∑ exp(𝜃w…)…∈𝒜 𝜃wu 𝜃 = 𝜃DD, … , 𝜃wu, … , 𝜃 • † f
- 65. • 𝜙 𝜋” 𝑎 𝑠 = exp 𝜃› 𝜙 𝑠, 𝑎 ∑ exp 𝜃› 𝜙 𝑠, 𝑏…∈𝒜 𝜙 𝑠, 𝑎 𝜙 𝜃
- 66. • 𝑑w 𝑠 ∈ ℝžŸ 𝑑u 𝑎 ∈ ℝž 𝜋” 𝑎 𝑠 = 1 2𝜋 ž /E 𝐶 D/E exp − 1 2 𝑎 − 𝑊𝑠 › 𝐶VD(𝑎 − 𝑊𝑠) 𝜋” 𝑎 𝑠 𝑊 ∈ ℝž מŸ 𝑑u×𝑑w 𝐶 𝜃 𝑊, 𝐶
- 67. • •
- 68. 𝜋” • 𝜃 •
- 69. 𝜋” • 𝑆Q = 𝑠Q • 𝐺J = lim f→j 𝑇VD ∑ 𝑅JPDPe fVD egQ 𝐽 𝜃; 𝑠Q = 𝔼 𝐺Q|𝑆Q = 𝑠Q = lim f→j 1 𝑇 𝔼 d 𝑅J f JgD |𝑆Q = 𝑠Q • 𝐺J = ∑ 𝛾e 𝑅JPDPe j egQ 𝐽 𝜃; 𝑠Q = 𝔼 𝐺Q 𝑆Q = 𝑠Q = 𝔼 d 𝛾JVD 𝑅J j JgD |𝑆Q = 𝑠Q
- 70. 𝜋” • 𝜃 𝜃JPD = 𝜃J + 𝜂𝛻” 𝐽(𝜃) 𝜂 𝛻” 𝐽(𝜃) 𝐽(𝜃) 𝜃 𝛻” 𝐽 𝜃 = 𝜕𝐽 𝜃 𝜕𝜃D , … , 𝜕𝐽 𝜃 𝜕𝜃ž › • 𝑄(𝑠, 𝑎) 𝛻” 𝐽 𝜃 = 𝔼q¦ 𝜕𝜋” 𝑎 𝑠 𝜕𝜃 1 𝜋”(𝑎|𝑠) 𝑄q 𝑠, 𝑎 𝛻” 𝐽 𝜃 = 𝔼q¦ 𝛻” log 𝜋”(𝑎|𝑠) 𝑄q 𝑠, 𝑎
- 71. 𝜋” 𝛻” 𝐽 𝜃 = 𝔼q¦ 𝛻” log 𝜋”(𝑎|𝑠) 𝑄q 𝑠, 𝑎 • 𝛻” 𝐽 𝜃 ≈ 1 𝑀 d 1 𝑇 d 𝛻” log 𝜋” 𝑎J © 𝑠J © 𝑄(𝑠J © , 𝑎J © ) f JgD M ©gD 𝑎J © , 𝑠J © • 𝑄q (𝑠, 𝑎) 𝑄q 𝑠J, 𝑎J ≈ 𝑅J 𝑎JVD 𝑠J
- 72. 𝜋” • 𝑄q (𝑠, 𝑎) 𝑤 𝑄« (𝑠, 𝑎) 𝑄« 𝑠, 𝑎 = 𝑤› 𝜙(𝑠, 𝑎) 𝜙(𝑠, 𝑎) 𝜙(𝑠, 𝑎) 𝑄« 𝑠, 𝑎 = 𝑤› 𝜙 𝑠, 𝑎 = 𝑤› 𝛻” log 𝜋”(𝑎|𝑠) 𝑤 𝑤∗ 𝛻” 𝐽 𝜃 = 𝔼q¦ 𝛻” log 𝜋” 𝑎 𝑠 𝑄«∗ (𝑠, 𝑎)
- 73. 𝜋” • 𝛻” 𝐽 𝜃 = 𝜕𝐽 𝜃 𝜕𝜃D , … , 𝜕𝐽 𝜃 𝜕𝜃ž › 𝜃 • 𝛻¬” 𝐽(𝜃) 𝛻¬” 𝐽 𝜃 = 𝐹VD 𝜃 𝛻” 𝐽(𝜃) 𝐹(𝜃) 𝐹 𝜃 = 𝔼 𝛻” log 𝜋” 𝑎 𝑠 𝛻” log 𝜋” 𝑎 𝑠 ›
- 74. • • • • • • •
- 75. 𝛻” 𝐽 𝜃 ≈ 1 𝑀 d 1 𝑇 d 𝛻” log 𝜋” 𝑎J © 𝑠J © 𝑄(𝑠J © , 𝑎J © ) f JgD M ©gD • 𝑄q(𝑠J, 𝑎J) 𝑅J 𝛻” 𝐽 𝜃 ≈ 1 𝑀 d 1 𝑇 d 𝛻” log 𝜋” 𝑎J © 𝑠J © 𝑅J © f JgD M ©gD 𝑅J © • 𝑏 𝛻” ®¯ 𝐽 𝜃 = d d 𝑅J © − 𝑏° 𝛻”log 𝜋”(𝑎J © |𝑠J © ) f JgD M ©gD 𝑏° = D Mf ∑ ∑ 𝑅J ©f JgD M ©gD
- 76. • 𝑄q(𝑠J, 𝑎J) 𝑅J • 𝑏q(𝑠) 𝛻” 𝐽 𝜃 = 𝔼q¦ 𝛻” log 𝜋”(𝑎|𝑠)(𝑄q 𝑠, 𝑎 − 𝑏q 𝑠 ) • 𝑉q(𝑠) 𝛻” 𝐽 𝜃 = 𝔼q¦ 𝛻” log 𝜋” 𝑎 𝑠 𝑄q 𝑠, 𝑎 − 𝑉q 𝑠 = 𝔼q¦ 𝛻” log 𝜋” 𝑎 𝑠 𝐴q 𝑠, 𝑎
- 77. • 𝑉q(𝑠) 𝛻” 𝐽 𝜃 = 𝔼q¦ 𝛻” log 𝜋” 𝑎 𝑠 𝑄q 𝑠, 𝑎 − 𝑉q 𝑠 = 𝔼q¦ 𝛻” log 𝜋” 𝑎 𝑠 𝐴q 𝑠, 𝑎 𝐴q 𝑠, 𝑎 = 𝑤› 𝛻” log 𝜋”(𝑎|𝑠) 𝛻¬” 𝐽 𝜃 = 𝐹VD 𝜃 𝛻” 𝐽 𝜃 = 𝐹VD 𝜃 𝐹 𝜃 𝑤 = 𝑤
- 78. 𝛻¬” 𝐽 𝜃 = 𝐹VD 𝜃 𝛻” 𝐽 𝜃 = 𝐹VD 𝜃 𝐹 𝜃 𝑤 = 𝑤 𝑤 • 𝑤 •
- 79. • •
- 80. • • • • •
- 81. • • •
- 82. • (𝒮, 𝒜, 𝑇, 𝑅, Ω, 𝑂) 𝑇 𝑠, 𝑎, 𝑠7 = 𝑃(𝑠7|𝑠, 𝑎) 𝑅(𝑠, 𝑎) 𝑂 𝑠7, 𝑎, 𝑜 = 𝑃(𝑜|𝑎, 𝑠7) •
- 83. 本章では これ
- 85. • • 𝑏 𝑠 = 𝑃(𝑠|ℎ) • ∀𝑠 𝑏 𝑠 = 0,1 d 𝑏 𝑠 = 1 w∈𝒮
- 87. • • 𝑥µ 𝑠µ 𝑥¶ 𝑠¶
- 88. • 𝑥µ 𝑠µ 𝑥¶ 𝑠¶ • 𝑏(𝑠µ, 𝑠¶) = (0.5,0.5) • 𝑥µ 𝑏(𝑠µ, 𝑠¶) = (0.85,0.15)
- 89. • 𝑥µ 𝑠µ 𝑥¶ 𝑠¶ • 𝑏(𝑠µ, 𝑠¶) = (0.5,0.5) • 𝑥µ 𝑏(𝑠µ, 𝑠¶) = (0.85,0.15)
- 90. b(s_l) + b(s_r) = 1上 にb(s)は存在
- 92. ℬ 𝜏 𝑏, 𝑎, 𝑏7 𝑅ℬ(𝑏, 𝑎)
- 93. 𝑉∗ 𝑠 = max u∈𝒜 d 𝑃 𝑠7 𝑠, 𝑎 𝑟 𝑠, 𝑎, 𝑠7 + 𝛾𝑉∗ 𝑠7 wx∈𝒮 = max u∈𝒜 𝑅 𝑠, 𝑎 + 𝛾 d 𝑇 𝑠, 𝑎, 𝑠7 𝑉∗ 𝑠7 wx∈𝒮 𝑉 𝑠 ← max u∈𝒜 𝑅 𝑠, 𝑎 + 𝛾 d 𝑇 𝑠, 𝑎, 𝑠7 𝑉 𝑠7 wx∈𝒮 T s, a, s7 = P(s7|s, a) R(s, a)
- 94. 𝑉(𝑏) 𝑉 𝑏 = max ¿∈À 𝑏 Á 𝛼 = max ¿∈À d 𝑏 𝑠 𝛼(𝑠) w∈𝒮 𝑅 𝑠, 𝑎 = 𝑟u 𝑠 = 𝛼(𝑠) 𝑟µÂÃJ (𝑠¶, 𝑠µ) = 𝑅 𝑠¶, 𝑙𝑒𝑓𝑡 , 𝑅 𝑠µ, 𝑙𝑒𝑓𝑡 = (−100, 10) 𝑏 = (0.5,0.5) 𝑏 Á 𝛼 = −45
- 95. • 𝑉 𝑏 = max u∈𝒜 d ∑ 𝑅 𝑠, 𝑎 𝑏(𝑠)w∈𝒮 Ω + 𝛾𝑃 𝑜 𝑏, 𝑎 𝑉(𝑏u,Ç) Ç∈È • 𝛼 Γ Γ Γ ← prune Ë Γu u∈𝒜 Γu = ⨁ Ç∈È Γu,Ç Γu,Ç = 1 Ω 𝑟u + 𝛾𝛼u,Ç|𝛼 ∈ Γ prune:価値関数に 寄与しない𝛼を除去
- 96. • • • • • • • •