#abtesting@csaintamant #abtesting@csaintamant

#abtesting@csaintamant
we can do better

#abtesting@csaintamant
brainstorm. test.brainstorm. test. measure.

#abtesting@csaintamant
data driven
product development

#abtesting@csaintamant
continuous
improvement

vs ba

a b

A
B
C
D
E
F
G
A
B
C
D
E
F
G
A
B
C
D
E
F
G
A
B
C
D
E
F
G
A
B
C
D
E
F
G
A
B
C
D
E
F
G
A
B
C
D
E
F
G
Test 1 Test 2 Test 3 Test 4 Test 5 Test 6 Test 7

#abtesting@csaintamant
it gets
messy

#abtesting@csaintamant
▶ design
▶ define
▶ deliver
#abtesting@csaintamant

#abtesting@csaintamant
(testable)
hypothesis

#abtesting@csaintamant
individual profiles in a multi-
user household will create a
more personalized experience
and result in higher retention

#abtesting@csaintamant
test
variables

#abtesting@csaintamant
test cells

Variables Gate
Gate w/
Remember
Simple Setup Detailed Setup
Cell 1 (control)
Cell 2 ✓
Cell 3 ✓ ✓
Cell 4 ✓ ✓
Cell 5 ✓ ✓

#abtesting@csaintamant
be mindful

#abtesting@csaintamant
iterate later

#abtesting@csaintamant
▶ design
▶ define
▶ deliver
#abtesting@csaintamant

#abtesting@csaintamant
test identifiers

#abtesting@csaintamant
test # 1234
test cells 1 - 5

var tests = user.getTests();
if (tests.in(1234, 2) || tests.in(1234, 3) ||
tests.in(1234, 4) || tests.in(1234, 5)) {
// profiles enabled?
if (tests.in(1234, 3) || tests.in(1234, 4) ||
tests.in(1234, 5)) {
// gate?
if (tests.in(1234, 4)) {
// gate with remember?
}
}
}
else {
// control behavior
}

#abtesting@csaintamant
externalize config

{
meta: {
id: 1234,
name: 'profiles',
},
cells: {
'1': { control: true, profilesEnabled: false },
'2': { profilesEnabled: true, simpleSetup: true },
'3': { profilesEnabled: true, simpleSetup: true, gate: true },
'4': { profilesEnabled: true, simpleSetup: true, gate: true,
gateRemember: true },
'5': { profilesEnabled: true, detailedSetup: true, gate: true }
}
}

var profilesTest = user.getTests().get('profiles');
if (profilesTest.attr('profilesEnabled')) {
// profiles enabled?
if (profilesTest.attr('gate')) {
// gate?
if (profilesTest.attr('gateRemember')) {
// gate with remember?
}
}
}
else {
// control behavior
}

#abtesting@csaintamant
feature flags?
yep.

profilesEnabled =
new Rule('profilesEnabled', function(context, params) {
// check test flag
var profilesTest = context.getTests().get('profiles');
return profilesTest.attr('profilesEnabled');
});
rules

profilesEnabled =
new Rule('profilesEnabled', function(context, params) {
// check app config
return context.getAppConfig().profilesEnabled;
});
rules

var tests = user.getTests();
if (tests.in(1234, 3) ||
tests.in(1234, 4) ||
tests.in(1234, 5)) {
// gate?
}
before after
if (rules.profilesGate) {
// gate!
}

#abtesting@csaintamant
▶ design
▶ define
▶ deliver
#abtesting@csaintamant

#abtesting@csaintamant
UI assembly / delivery
HTML + (CSS + JS)

#abtesting@csaintamant
templating

profiles-setup.template
<div id="profiles-form">
<h1>Who will be watching Netflix?</h1>
<input id="profile1" /><label for-"profile1">That's you!</label>
<input id="profile2" />
<input id="profile3" />
<input id="profile4" />
<input id="profile5" />
<button>Continue</button>
</div>
<div id="profiles-promo">
<ul>
<li>Up to 5 people</li>
<li>No extra fees</li>
<li>Each person gets suggestions based on their own viewing and tastes</li>
<li>Great for kids</li>
</ul>
</div>

<div id="profiles-form">
<h1>Who will be watching Netflix?</h1>
<input id="profile1" /><label for-"profile1">That's you!</label>
<input id="profile2" />
<input id="profile3" />
<input id="profile4" />
<input id="profile5" />
<button>Continue</button>
</div>
<div id="profiles-promo">
{#control}
<ul>
<li>Up to 5 people</li>
<li>No extra fees</li>
<li>Each person gets suggestions based on their own viewing and tastes</li>
<li>Great for kids</li>
</ul>
{/control}
{#promo1}
<ul>
<li>Up to 5 people</li>
<li>No extra fees</li>
<li>Each person gets suggestions based on their own viewing and tastes</li>
<li>Kids under 12 get a safe area with kid-friendly movies and TV shows</li>
</ul>
{/promo1}
{#promo2}
<ul>
<li>Up to 5 people</li>
<li>No extra fees</li>
<li>Each person gets suggestions based on their own viewing and tastes</li>
<li>Kids under 12 get a safe area with kid-friendly movies and TV shows</li>
</ul>
{/promo2}
{#promo3}
<img src="images/profiles-setup.png" />
<h2>Each person added will get suggestions based on what they like to watch</h2>
<ul>
<li>Up to 5 people</li>
<li>No extra fees</li>
</ul>
{/promo3}
{#promo4}
<img src="images/profiles-setup.png" />
<h2>Have up to 5 profiles at no extra cost</h2>
<ul>
<li>Each person will get suggestions based on their own viewing and tastes</li>
<li>Great for kids</li>
</ul>
{/promo4}
</div>

profiles-setup.tmpl
if ifif if if
Control Promo 1 Promo 2 Promo 3 Promo 4

profiles-setup.tmpl
promo.tmpl promo2.tmplpromo1.tmpl promo3.tmpl promo4.tmpl
if ifif if if

<div id="profiles-form">
<h1>Who will be watching Netflix?</h1>
[...]
</div>
<div id="profiles-promo">
{> promo /}
</div>

promo.json
profiles-setup.tmpl
?
promo.tmpl promo2.tmplpromo1.tmpl promo3.tmpl promo4.tmpl

promo.json
{
"rules": [],
"templateName": "promo"
},
{
"rules": ["profilesPromo(1)"],
"templateName": "promo1"
},
{
"rules": ["profilesPromo(2)"],
"templateName": "promo2"
},
{
"rules": ["profilesPromo(3)"],
"templateName": "promo3"
},
{
"rules": ["profilesPromo(4)"],
"templateName": "promo4"
}

profilesPromo =
new Rule('profilesPromo', function(context, params) {
// check test membership
var test = context.getTests().get('profilesSetup');
return test && test.cell(params.id-1);
});
rules

#abtesting@csaintamant
how does it work?

profiles-setup.tmpl template engine partial

resolver promo.json
(mappings)
rules

rules
promo.tmpl
promo1.tmpl
promo2.tmpl

profiles-setup.tmpl template engine resolver

#abtesting@csaintamant
template loading
simplified

#abtesting@csaintamant
client-side
packaging

app.js
import backbone from 'backbone';
import search from 'search';
import profiles from 'profiles';
export ...

#abtesting@csaintamant
search
app.js
profiles
dep1 dep2 dep3 dep4 dep5
sub-dep sub-depsub-dep sub-dep sub-dep sub-dep

#abtesting@csaintamant
search
app.js
profiles
dep1 dep2 dep3 dep4 dep5
sub-dep sub-depsub-dep sub-dep sub-dep sub-dep

search
app.js
profiles
dep1 dep2 dep3 dep4 dep5
sub-dep sub-depsub-dep sub-dep sub-dep sub-dep

#abtesting@csaintamant
100s of files..?
Millions of combinations..?

#abtesting@csaintamant
conditional
dependencies

search
app.js
profiles
dep1 dep2 dep3 dep4 dep5
sub-dep sub-depsub-dep sub-dep sub-dep sub-dep

rules again!
profilesEnabled =
new Rule('profilesEnabled', function(context, params) {
// check test flag
var profilesTest = context.getTests().get('profiles');
return profilesTest.attr('profilesEnabled');
});

/*
* @includewhen rule.profilesEnabled
*/
profiles.js

/*
* @includewhen rule.searchEnabled
*/
search.js

#abtesting@csaintamant
build time:
static analysis

#abtesting@csaintamant
minify / compress
build time:

profiles.js
backbone
search.js
app.js
registry

"app.js": {
"deps": [
"backbone",
"search.js",
"profiles.js",
],
"depsFull": [
"react",
"searchDep2.js",
"searchDep1.js",
"search.js",
"profilesDep2.js",
"profilesDep1.js",
"profiles.js"
]
}

"profiles.js": {
"rule": "profilesEnabled",
"deps": [
"backbone",
"profilesDep2.js",
"profilesDep1.js",
],
"depsFull": [
"backbone",
"profilesSubDep3.js",
"profilesSubDep2.js"
"profilesSubDep1.js"
"profilesDep2.js",
"profilesDep1.js"
]
}
rule!

javascript
registry
build

#abtesting@csaintamant
request time:
evaluate rules

#abtesting@csaintamant
assemble package
request time:

registry
rulespackager
request

#abtesting@csaintamant
packaging
simplified

#abtesting@csaintamant
everything is
a module

#abtesting@csaintamant
define dependencies

#abtesting@csaintamant
only load
what you need
…per user

#abtesting@csaintamant
isolate complexity

#abtesting@csaintamant
▶ design
▶ define
▶ deliver
#abtesting@csaintamant

#abtesting@csaintamant
multivariate testing
is powerful…

#abtesting@csaintamant
…but complex

#abtesting@csaintamant
plan for complexity

“Simplicity is the
ultimate sophistication”
- Leonardo da Vinci

go experiment!

questions?
Chris Saint-Amant
@csaintamant
#abtesting