Scaling A/B testing on Netflix.com with

_________Alex Liu @stinkydofu

data driven product development

A

B

C

D

E

F

G

A

B

C

D

E

F

G

A

B

C

D

E

F

G

A

B

C

D

E

F

G

A

B

C

D

E

F

G

A

B

C

D

E

F

G

A

B

C

D

E

F

G

Test 1 Test 2 Test 3 Test 4 Test 5 Test 6 Test 7

2,097,152 unique experiences across seven tests

hundreds of new A/B tests per year

433518929550349486086117218185493567650…72061153709996

2105 566 685templates CSS JS

2.5M unique packages every week

<html/> <link/> <script/>

problem: conditional dependencies

▶ Templating ▶ Packaging ▶ Bonus Round

Templating

payment.dust<div id="payments"> <input id="first-name"><input id="last-name"> <span class="payment-logos"> <img src="logo1.png"><img src="logo2.png"> <img src="logo3.png"><img src="logo4.png"> </span> <input id="card-number"><input id="security-code"> <select name="month"></select><select name="year"></select> <checkbox id="agree-to-terms"/> <button>Start Your Trial</button> </div>

<input id=“first-name"><input id=“last-name"> {@inTest id="10" cell=“2a"} <div class="payment-types"> <div id="CC"> <span class="payment-logos"> <img src="logo1.png"><img src="logo2.png"> <img src="logo3.png"><img src="logo4.png"> </span> <input id=“card-number"><input id=“security-code”> </div> <div id="DD"> <span class=“payment-logos"> <img src="logo1.png"><img src="logo2.png"> <img src="logo3.png"><img src="logo4.png"> </span> <input id="CPF"><input id="bank-name"> <input id="branch-number"><input id="account-number"> <input id="digit"> </div> </div> {/@inTest} {@inTest id="10" cell=“3"} <div class="payment-types"> <div id="CC"> <span class="payment-logos"> <img src="logo1.png"><img src="logo2.png"> <img src="logo3.png"><img src="logo4.png"> </span> <input id=“card-number"><input id=“security-code”> </div> <div id="DD"> <span class="payment-logos"> <img src="logo1.png"><img src="logo2.png"> <img src="logo3.png"><img src="logo4.png"> </span> <input id="CPF"><input id="bank-name"> <input id="branch-number"><input id="account-number"> <input id="digit"> </div> </div> {/@inTest} {@inTest id="10" cell=“4"} <div class="payment-types"> <div id="CC"> <span class="payment-logos"> <img src="logo1.png"><img src="logo2.png"> <img src="logo3.png"><img src="logo4.png"> </span> <input id=“card-number"><input id=“security-code”> </div> <div id="DD"> <span class="payment-logos"> <img src="logo1.png"><img src="logo2.png"> <img src="logo3.png"><img src="logo4.png"> </span> <input id="CPF"><input id="bank-name"> <input id="branch-number"><input id="account-number"> <input id="digit"> </div> </div> {/@inTest} {@inTest id="10" cell=“5"} <div class="payment-types"> <div id="CC"> <span class="payment-logos"> <img src="logo1.png"><img src="logo2.png"> <img src="logo3.png"><img src="logo4.png"> </span> <input id=“card-number"><input id=“security-code”> </div> <div id="DD"> <span class="payment-logos"> <img src="logo1.png"><img src="logo2.png"> <img src="logo3.png"><img src="logo4.png"> </span> <input id="CPF"><input id="bank-name"> <input id="branch-number"><input id="account-number"> <input id="digit"> </div> </div> {/@inTest} <checkbox id="agree-to-terms"></checkbox> <button>Start Your Trial</button> </div>

payment.dust

if ifif if if

Control Cell 2 Cell 3 Cell 4 Cell 5

<div class="payment-types"> <div id="CC"> {> payment_type_cc /} </div> <div id="DD"> {> payment_type_dd /} </div> </div>

payment.dust

Control Cell 2 Cell 3 Cell 4 Cell 5

payment_type_cc.dust payment_type_dd.dust

if ifif if if

payment.dust

Control.dust

payment_type_cc.dust payment_type_dd.dust

Cell3.dustCell2.dust Cell4.dust Cell5.dust

if ifif if if

<div id="payments"> <input id="first-name"> <input id="last-name">

{> payment_method /}

<input type="checkbox" id="terms"> <button>Start Your Trial</button> </div>

payment.json

payment.dust

?

Control.dust Cell3.dustCell2.dust Cell4.dust Cell5.dust

payment_type_cc.dust payment_type_dd.dust

payment.json{ "rules": [], "templateName": "control" }, { "rules": ["PaymentTest(2)"], "templateName": "payment_cell2" }, { "rules": ["PaymentTest(3)"], "templateName": "payment_cell3" }, { "rules": ["PaymentTest(4)"], "templateName": "payment_cell4" }, { "rules": ["PaymentTest(5)"], "templateName": "payment_cell5" }

require('nf-rule-infrastructure')

var Rule = require('nf-rule-infrastructure'), PaymentTest;

PaymentTest = new Rule('PaymentTest', function(context, params, cb) { var test = context.abtests.get(10); cb(test && test.cell(params.id)); });

module.exports = PaymentTest;

anatomy of a rule

require('nf-template-resolver')

payment.dust dustjs partial

resolver payment.json (mappings)

rule

rules

control.dust

cell2.dust

cell3.dust

payment.dust dustjs resolver

<input id=“first-name"><input id=“last-name"> {@inTest id="10" cell=“2a"} <div class="payment-types"> <div id="CC"> <span class="payment-logos"> <img src="logo1.png"><img src="logo2.png"> <img src="logo3.png"><img src="logo4.png"> </span> <input id=“card-number"><input id=“security-code”> </div> <div id="DD"> <span class=“payment-logos"> <img src="logo1.png"><img src="logo2.png"> <img src="logo3.png"><img src="logo4.png"> </span> <input id="CPF"><input id="bank-name"> <input id="branch-number"><input id="account-number"> <input id="digit"> </div> </div> {/@inTest} {@inTest id="10" cell=“3"} <div class="payment-types"> <div id="CC"> <span class="payment-logos"> <img src="logo1.png"><img src="logo2.png"> <img src="logo3.png"><img src="logo4.png"> </span> <input id=“card-number"><input id=“security-code”> </div> <div id="DD"> <span class="payment-logos"> <img src="logo1.png"><img src="logo2.png"> <img src="logo3.png"><img src="logo4.png"> </span> <input id="CPF"><input id="bank-name"> <input id="branch-number"><input id="account-number"> <input id="digit"> </div> </div> {/@inTest} {@inTest id="10" cell=“4"} <div class="payment-types"> <div id="CC"> <span class="payment-logos"> <img src="logo1.png"><img src="logo2.png"> <img src="logo3.png"><img src="logo4.png"> </span> <input id=“card-number"><input id=“security-code”> </div> <div id="DD"> <span class="payment-logos"> <img src="logo1.png"><img src="logo2.png"> <img src="logo3.png"><img src="logo4.png"> </span> <input id="CPF"><input id="bank-name"> <input id="branch-number"><input id="account-number"> <input id="digit"> </div> </div> {/@inTest} {@inTest id="10" cell=“5"} <div class="payment-types"> <div id="CC"> <span class="payment-logos"> <img src="logo1.png"><img src="logo2.png"> <img src="logo3.png"><img src="logo4.png"> </span> <input id=“card-number"><input id=“security-code”> </div> <div id="DD"> <span class="payment-logos"> <img src="logo1.png"><img src="logo2.png"> <img src="logo3.png"><img src="logo4.png"> </span> <input id="CPF"><input id="bank-name"> <input id="branch-number"><input id="account-number"> <input id="digit"> </div> </div> {/@inTest} <checkbox id="agree-to-terms"></checkbox> <button>Start Your Trial</button> </div>

<div id="payments"> <input id="first-name"> <input id="last-name">

{> payment_method /}

<input type="checkbox" id="terms"> <button>Start Your Trial</button> </div>

Wins▶ combine rules ▶ improve template legibility ▶ increase template reuse

▶ Templating ▶ Packaging ▶ Bonus Round

Packaging

everything is a module

oldSearch

app.js

newSearch

dep1 dep2 dep3 dep4 dep5

sub-dep sub-depsub-dep sub-dep sub-dep sub-dep

oldSearch

app.js

newSearch

dep1 dep2 dep3 dep4 dep5

sub-dep sub-depsub-dep sub-dep sub-dep sub-dep

app.js

import jquery from 'jquery'; import oldSearch from 'oldSearch'; import newSearch from 'newSearch';

export ...

oldSearch

app.js

newSearch

dep1 dep2 dep3 dep4 dep5

sub-dep sub-depsub-dep sub-dep sub-dep sub-dep

685 files…?

2.5M packages…?

oldSearch

app.js

newSearch

dep1 dep2 dep3 dep4 dep5

sub-dep sub-depsub-dep sub-dep sub-dep sub-dep

problem: conditional dependencies

requestbuild

require('nf-include-when')

/* * @includewhen rule.notInNewSearch */

oldSearch.js

/* * @includewhen rule.inNewSearch */

newSearch.js

var Rule = require('nf-rule-infrastructure'), inNewSearch;

inNewSearch = new Rule('inNewSearch', function(context, cb) { var test = context.abtests.get(1534); cb(test && test.cell(1)); });

module.exports = inNewSearch;

anatomy of a rule

require('nf-asset-registry')

import jquery from 'jquery'; import oldSearch from 'oldSearch'; import newSearch from 'newSearch';

export ...

app.js

newSearch.js

jquery

oldSearch.js

app.js

registry

"app.js": { "deps": [ "jquery", "oldSearch.js", "newSearch.js", ], "depsFull": [ "jquery", "oldSearchDep2.js", "oldSearchDep1.js", "oldSearch.js", "newSearchDep2.js", "newSearchDep1.js", "newSearch.js" ] }

"newSearch.js": { "rule": "inNewSearch", "deps": [ "jquery", "newSearchDep2.js", "newSearchDep1.js", ], "depsFull": [ "jquery", "newSearchSubDep3.js", "newSearchSubDep2.js" "newSearchSubDep1.js" "newSearchDep2.js", "newSearchDep1.js" ] }

nf-include-when

require('nf-packager')

var packager = require('nf-packager'), includeWhen = require('nf-include-when'), registries = require('nf-asset-registry');

function getScriptUrl() return packager.getPackageDefinition('app.js', registries, includeWhen); }

"app.js": { "deps": [ "jquery", "oldSearch.js", "newSearch.js", ], "depsFull": [ "jquery", "oldSearchDep2.js", "oldSearchDep1.js", "oldSearch.js", "newSearchDep2.js", "newSearchDep1.js", "newSearch.js" ], "fileSize": "4.41 kB", "fileSizeFull": "120.52 kB" }

Step 1: Get the full dependency tree for the requested package from the registry.

[ "jquery", /* no rule */ "oldSearchDep2.js", /* no rule */ "oldSearchDep1.js", /* no rule */ "oldSearch.js", /* rules.notInNewSearch */ "newSearchDep2.js", /* no rule */ "newSearchDep1.js”, /* no rule */ "newSearch.js" /* rules.inNewSearch */ ]

Step 2: Determine which files have rules.

[ "jquery", /* no rule */ "oldSearchDep2.js", /* no rule */ "oldSearchDep1.js", /* no rule */ "oldSearch.js", /* rules.notInNewSearch */ "newSearchDep2.js", /* no rule */ "newSearchDep1.js”, /* no rule */ "newSearch.js" /* rules.inNewSearch */ ]

Step 3: Run the rules. Filter out all deps that resolved false.

✓

[ "jquery", /* no rule */ "oldSearchDep2.js", /* no rule */ "oldSearchDep1.js", /* no rule */ "oldSearch.js", /* rules.notInNewSearch */ "newSearchDep2.js", /* no rule */ "newSearchDep1.js”, /* no rule */ "newSearch.js" /* rules.inNewSearch */ ]

Step 4: Filter out all extraneous sub deps.

✓

Step 5: Concatenate the files.

[ "jquery", /* no rule */ "newSearchDep2.js", /* no rule */ "newSearchDep1.js”, /* no rule */ "newSearch.js" /* rules.inNewSearch */ ]

buildjavascript

registry

request registry

rulespackager

Wins▶ leverage build time tools ▶ leverage the server ▶ divide and conquer with modules

▶ Templating ▶ Packaging ▶ Bonus Round

Bonus Round

be creative with the registry

"account/bb/models/ratingHistoryModel.js": { "rule": null, "deps": [...], "depsFull": [...], "depsCount": { "underscore": 2, "backbone": 1, "jquery": 2, "common/requirejs-plugins.js": 4, "requirejs-text": 4, "utils/contextData.js": 1, "common/nfNamespace.js": 1 }, "hash": "dd23b163", "fileSize": "1.21 kB", "fileSizeFull": "173.04 kB" }

dependency counting

dependency pruning

file sizes

@import (reference) "/common/_nf_defs.less"; @import (reference) "/member/memberCore.less"; @import (reference) "/components/menu.less"; @import (reference) "/components/breadcrumbs.less";

@import modules

"account/containerResponsive.css": { "rule": null, "deps": [...], "depsFull": [...], "depsCount": [...], "hash": "65a431f3", "fileSize": "709 B", "fileSizeFull": "709 B", "css": { "selectors": 8, "declarationBlocks": 6, "declarations": 17, "mediaQueries": 3 } }

css analysis

the

best part

"cache": { "account/pin.js": "define('account/pin.js', ['member/memberC…", "account/bb/models/changePlanModel.js": "define('account/b…", "account/bb/models/ratingHistoryModel.js": "define('account…", "account/bb/models/viewingActivityModel.js": "define('account…", "account/bb/views/changePlanView.js": "define('account/bb/vi…", "account/bb/views/changePlanView.js": "define('account/bb/vi…", "account/bb/views/emailSubView.js": "define('account/bb/views…", "account/bb/views/viewingActivityView.js": "define('account…", "common/UITracking.js": "define('common/UITracking.js, ['me…", "common/UITrackingOverlay.js": "define('common/UITrackingOve…", … … …

css

mappings

javascript

templates templates

mappings

javascript

css

templates

mappings

javascript

css

UI Bundle

deploy UI bundles

anytime

never touch the file system

< 5ms package response times

Wins▶ static analysis FTW ▶ independent UI deployments ▶ requests never touch the fs ▶ fast package response times

Our Learnings

learn by doing

fail fastmove faster

“I have not failed.I’ve just found 10,000 waysthat won’t work.”

Thomas Edison

simplify

Alex Liu @stinkydofu

thank you

Alex Liu @stinkydofu

questions?