paper reading : [NIPS 2016] Causal Bandits: Learning Good Interventions via Causal Inference https://papers.nips.cc/paper/6195-causal-bandits-learning-good-interventions-via-causal-inference.pdf

4. X1 X4
X2
X3

7. 𝒴 = {yx=0, yx=1}
p (yx=0) p (yx=1)
p (y|do (x = 0)) p (y|do (x = 1))
ATE := 𝔼 (yx=1) − 𝔼 (yx=0)
ATE := 𝔼 (y|do (x = 1)) − 𝔼 (y|do (x = 0))

8. Pr (X1, X2, X3, X4) = Pr (X1 |X2, X3) Pr (X2 |X3) Pr (X3) Pr (X4 |X1, X2) =
4
∏
i=1
Pr (Xi | 𝒫Xi)
X1 X4
X2
X3
do (X1 = 1)
X1
X2 X3 X1
X1 X4
X2
X3

16. G
𝒳 = {X1, . . . , XN}
Xi 𝒫Xi
𝒜
a = do (X = x) ∈ 𝒜
x = {x1, . . . , xn} X = {X1, . . . , Xn}
do () ∈ 𝒜
a = do (X = x) μa = 𝔼[Y|do(X = x)]
μ* := max
a∈𝒜
μa

17. T
t at = do(Xt = xt) ∈ 𝒜
Xc
t ∼ Pr (Xc
t |do (Xt = xt)) Xc
t = 𝒳 − Xt
Yt ∈ {0,1}
T
̂a*T
∈ 𝒜
RT
RT = μ* − μ ̂a*T

18. G = (𝒳, E)
𝒳
E
𝒜
T
̂a*T
∈ 𝒜 T

19. X K
𝒜 = {do (X = k)} : k ∈ {1,...,K}
Y ∈ {0,1} Pr{Y = 1|X}

21. N X = {X1, . . . , XN}
Y ∈ {0,1}
𝒜 = {do()} ∪ {do(Xi = j) : 1 ≤ i ≤ N, j ∈ {0,1}}
Xi ∼ Bernoulli (qi)
q = (q1, . . . , qN) ∈ [0,1]
N
Pr (Y = 1|X)

22. do()
Pr (Y|do (Xi = j)) = Pr (Y|Xi = j)
Pr (Xi = j) (Xi = j)
do (Xi = j)
Pr (Xi = j) (Xi = j)

23. q
qi (1 − qi)
τ ∈ [2...N] Iτ =
{
i : min{qi, 1 − qi} <
1
τ }
τ |Iτ |
m (q) : ℝN
→ ℝ
m (q) = min{τ : |Iτ | ≤ τ}

24. m (q)
q =
(
1
2
,
1
2
, …
1
2)
I2 =
{
i : min{qi, 1 − qi} <
1
2}
= {}, Iτ:τ>2 = {}
m (q) = min{τ : |Iτ | ≤ τ} = 2
q = (0,0,…,0)
I2 =
{
i : min{qi, 1 − qi} <
1
2}
= {1,…, N}, Iτ:τ>2 = {1,…, N}
m (q) = min{τ : |Iτ | ≤ τ} = N

26. RT ∈ 𝒪
m (q)
T
log
(
NT
m )
T q
RT ∈ Ω
m (q)
T
2N
RT ∈ Ω
(
N
T )
N > m

27. Pr{Y|Xi = j} ≠ Pr{Y|do (Xi = j)}
Pr{X1, . . . XN |do (Xi = j)}
Pr{X1, . . . XN |do (Xi = j)} =
∏
k≠i
Pr{Xk | 𝒫Xk
}δ (Xi − j)
Pr {Y|a}

28. Y = XN, Xk = Xk−1
Pr(X1) = 0
do(Xi = 1) do(X1 = 1)
do()
Pr (Y|do (X2 = j)) =
∑
X1
Pr (X1) Pr (Y|X1, X2 = j)
X1 = X2 Pr(X2 = 1)
(X1 = 0,X2 = 1)
Pr (Y|do (X2 = 1))

29. ∀a ∈ 𝒜 Pr ( 𝒫Y |a) Pr (Y|a)
η ∈ [0,1]
|𝒜|
∑
a∈𝒜
ηa = 1
Q :=
∑
a∈𝒜
ηaPr { 𝒫Y |a}
η

30. η μa
∀a ∈ 𝒜
a
Ra (X) =
Pr{𝒫Y(X)|a}
Q{𝒫Y(X)}
=
Pr{𝒫Y(X)|a}
∑a∈𝒜
ηaPr { 𝒫Y(X)|a}
𝒫Y(X)
a
̂μa =
1
T
T
∑
t=1
Yt Ra(Xt) 1 {Ra(Xt) ≤ Ba}
Ba ≥ 0

31. η

32. m(η)
m(η) = max
a∈𝒜
𝔼a [
Pr{𝒫Y(X)|a}
Q{𝒫Y(X)} ]
B ∈ ℝ|𝒜|
Ba =
m (η) T
log (2T| 𝒜|)
RT ∈ 𝒪
m (η)
T
log (2T| 𝒜|)

33. Q
Q η
m (η)
η* = arg min
η
m (η) = arg min
η
max
a∈𝒜
𝔼a [
Pr{𝒫Y(X)|a}
Q{𝒫Y(X)} ]
= arg min
η
max
a∈𝒜
𝔼a
[
Pr{𝒫Y(X)|a}
∑b∈𝒜
ηbPr{𝒫Y(X)} ]
m (η) η
η*

35. Yt ∼ Bernouli
(
1
2
+ ϵ
)
if X1 = 1
Yt ∼ Bernouli
(
1
2
− ϵ′
)
if X1 = 0
𝔼[Y|do(X1 = 1)] =
1
2
+ ϵ
𝔼[Y|do(X1 = 0)] =
1
2
− ϵ′
a ∈ 𝒜{do(X1 = 1), do (X1 = 0)} 𝔼[Y|a] =
1
2
qi = 0 i ≤ m qi =
1
2
Xi ∼ Bernouli (qi)
ϵ′ =
q1ϵ
1 − q1
Xi:i>1
X1

36. q

37. ϵ