StreamsFunctionalJs - London - May 2014

@darachennis

A <3able Stream// by @TooTallNate - https://gist.github.com/TooTallNate/3947591 // npm install lame ; npm install speaker ; node mp3player.js

var fs = require('fs');

var lame = require('lame');

var Speaker = require('speaker');

fs.createReadStream(process.argv[2])

.pipe(new lame.Decoder())

.on('format', function (format) {

this.pipe(new Speaker(format));

});

A <3able Stream// by @TooTallNate - https://gist.github.com/TooTallNate/3947591 // npm install lame ; npm install speaker ; node mp3player.js

var fs = require('fs');

var lame = require('lame');

var Speaker = require('speaker');

fs.createReadStream(process.argv[2])

.pipe(new lame.Decoder())

.on('format', function (format) {

this.pipe(new Speaker(format));

});

Sound Soupvar fs = require('fs');

var lame = require('lame');

var Speaker = require('speaker');

!// Cache names of mp3's in cwd

function isMp3(e) { return e.indexOf("mp3") > 1 };

var soundz = fs.readdirSync('.').filter(isMp3);

!// Every half second, play a random sound

setInterval(function(){

blend() },500);

function rand(min, max) {

var offset = min;

var range = (max - min) + 1;

return Math.floor( Math.random() * range) + offset;

}

function blend() {

fs.createReadStream(soundz[rand(0,soundz.length-1)])

.pipe(new lame.Decoder())

.on('format', function (format) {

this.pipe(new Speaker(format));

});

}

A brief history in Node.js streams

UNIXy• UNIX $ a | b | c | d

• NODE.js> a.pipe(b).pipe(c).pipe(d)

• same as:

• a.pipe(b);b.pipe(c);c.pipe(d);// but, less typing for the win \o/

Streams 0

• Util.pump(Readable,Writable)

Streams 1

• EventEmitter and a pipe() function.

• pipe() is an instance of the Builder Pattern

• EventEmitter is an instance of Subject/Observer Pattern

Streams 1: Examplevar Stream = require(‘stream').Stream

var util = require('util')

function MyStream () {

Stream.call(this)

}

util.inherits(MyStream, Stream)

// stream logic and state management and back-pressure

// you’re on your own (with the rest of the pipeline)

// this was error prone and painful …

Streams 2

• EventEmitter and a pipe() function.

• Readable, Writable, Duplex, Transform, Passthrough (and classic) ‘behaviours'

• New ‘behaviours’ encourage good ‘backpressure' handling disciplines

Streams 2: Using Readable// From: Stream Handbook, Readable with Push

var Readable = require('stream').Readable;

var rs = new Readable;

rs.push(‘beep ‘);

rs.push(‘boop\n’);

rs.push(null);

rs.pipe(process.stdout);

Streams 2: Using Readable// From: Stream Handbook

var Readable = require('stream').Readable;

var rs = Readable();

var c = 97;

rs._read = function () {

rs.push(String.fromCharCode(c++));

if (c > 'z'.charCodeAt(0)) rs.push(null);

};

rs.pipe(process.stdout); // _read called once, prints ‘a-z'

Streams 2: Pop Quiz!!!$ node readable-popquiz.js

abcdefghijklmnopqrstuvwxyz% $

!

Streams 2: Pop Quiz!!!// From: Stream Handbook (tweaked)

var Readable = require('stream').Readable;

var rs = Readable();

var c = 97;

rs._read = function () {

rs.push(String.fromCharCode(c++));

if (c > 'z'.charCodeAt(0)) rs.push(null);

};

rs.pipe(process.stdout); // _read called once

rs.pipe(process.stdout); // _read called twice

rs.pipe(process.stdout); // _read called three times

rs.pipe(process.stdout); // _read called four times

Streams 2: Pop Quiz!!!$ node readable-popquiz.js

aaaabbbbccccddddeeeeffffgggghhhhiiiijjjjkkkkllllmmmmnnnnooooppppqqqqrrrrssssttttuuuuvvvvwwwwxxxxyyyyzzzz% $

… WAT? (Think about it)

!

Streams 2: Using Writable// From: Stream Handbook

var fs = require('fs');

var ws = fs.createWriteStream('message.txt');

ws.write('beep ‘); // Write …

setTimeout(function () {

ws.end(‘boop\n'); // We’re done writing

}, 1000);

Streams 2: Using Writablevar Writable = require('stream').Writable;

var src = process.stdin;

var dst = new Writable;

dst._write = function (chunk, enc, next) {

console.log(chunk.toString());

next();

};

src.push("hit any key!");

src.pipe(dst);

Streams 2: Transformvar Transform = stream.Transform || require('readable-stream').Transform;

function Hash(algorithm, encoding, options) {

Transform.call(this, options);

this.digester = crypto.createHash(algorithm);

this.encoding = encoding;

}

util.inherits(Hash, Transform);

Hash.prototype._transform = function (chunk, enc, cb) {

var bf = (Buffer.isBuffer(chunk)) ? chunk : new Buffer(chunk, enc);

this.digester.update(bf);

cb();

};

Hash.prototype._flush = function (cb) {

this.push(this.digester.digest(this.encoding));

cb();

};

Streams 2: Transformvar hash = require('./hash.js');

var fs = require('fs');

var rs = fs.createReadStream(‘file.js’);

var md5Hash = hash.Hash('md5', 'base64');

// var sha1Hash = hash.Hash('sha1', 'base64');

rs.pipe(md5Hash).pipe(process.stdout);

Streams 2: Duplex* Independent Readable and Writable channels

* Implement _read and _write

* Example use - network protocols - serial communications with hardware - …

Know a classic stream// From: Stream Handbook

process.stdin.on('data', function (buf) {

console.log(buf);

});

process.stdin.on('end', function () {

console.log('__END__');

});

Streams 3

• Streams2 tweaked

• API the same as Streams2

Useful modules

• @DominicTarr’s event-stream https://github.com/dominictarr/event-stream

• @RVagg’s through2https://github.com/rvagg/through2

• @Raynos’s duplexerhttps://github.com/Raynos/duplexer

• https://github.com/substack/stream-handbook - @SubStack’s Stream handbook

• http://www.nearform.com/nodecrunch/dont-use-nodes-core-stream-module#.U4I71JRdWuk - Why @Rvagg doesn’t use node’s core stream module (& nor should we!)

Good to know …

Meta Programming with Streams and Pipes

Great British Node Conference October 8th 2013

London !

Darach Ennis

Streams 101

! Readable Streams ! IO / throughput oriented ! Events – data, end, error, close ! Methods – pause, resume, end, destroy

! Writeable Streams ! IO / throughput oriented ! Events - drain, pause, resume, error, close ! Methods – write, end, destroy

Streams … 4IO

! IO Oriented

! Finite, not infinite

! Asynchronous

! Lazy

! Assumes in transit data, not in memory forms

! Great for IO. Not efficient for compute

CSV Reader .@maxogden

Beams

Streams for Compute

Beams 101

! Sources ! Where data pours in

! Sinks ! Where results pop out

! Operators ! Filters – Drop or pass on data based on a UDF ! Transforms – Mutate data based on a UDF ! Branch with Filters ! Combine with Sinks

Beams… 4CPU

! Compute Oriented

! Infinite, not finite

! Synchronous

! Eager

! Assumes in memory form, not encoded or wire data

! Convenient for compute. Not good for IO

Beams - Branch

Beams - Combine

Ex – Fly NodeCopter with Streams!

Extend Games to Robots!

Extend?

Meta Programming

A minor problem

! Eager: a.pipe(operation).pipe(z).pipe(a);

! Implies:RangeError: Maximum call stack size exceeded

Goto + Modules = Magic

Extend Beam with Modules

Questions & Thanks

! npm install beam ! Compute oriented streams ! Branch, Combine, Filter, Transform data ! Extend with goto and modules.

! npm install eep ! Beam will be integrated into eep soon ! Eep provides aggregate windows ! Slide, Tumble, Monotonic, Periodic ! Stats Library ! Ordering now supported

! npm install nodesapiens ! Robosapiens V1 for node.js

! Code on github ! https://github.com/darach

Dojo - nodeschool.io

• Basics

• Stream Adventure

• Functional Javascript