From 538bb41ef1b707f573b4aa2592b6c4c9ed73a8f8 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Fri, 20 Feb 2015 03:46:34 +0900 Subject: [PATCH 01/93] Add operation stream library From https://github.com/tyoshino/streams_integration/blob/master/reference-implementation/operation-stream.js --- .../lib/experimental/operation-stream.js | 393 ++++++++++++++++++ reference-implementation/run-tests.js | 2 + .../test/experimental/operation-stream.js | 157 +++++++ 3 files changed, 552 insertions(+) create mode 100644 reference-implementation/lib/experimental/operation-stream.js create mode 100644 reference-implementation/test/experimental/operation-stream.js diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js new file mode 100644 index 000000000..28fe4160d --- /dev/null +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -0,0 +1,393 @@ +export default function createOperationStream(strategy) { + var stream = new OperationStream(strategy); + return { + writable: new WritableOperationStream(stream), + readable: new ReadableOperationStream(stream) + }; +} + +class OperationStatus { + constructor() { + this._state = 'waiting'; + this._result = undefined; + this._readyPromise = new Promise((resolve, reject) => { + this._resolveReadyPromise = resolve; + }); + } + + _onCompletion(v) { + this._state = 'completed'; + this._result = v; + this._resolveReadyPromise(); + this._resolveReadyPromise = undefined; + } + _onError(e) { + this._state = 'errored'; + this._result = e; + this._resolveReadyPromise(); + this._resolveReadyPromise = undefined; + } + + get state() { + return this._state; + } + get result() { + return this._result; + } + get ready() { + return this._readyPromise; + } +} + +class Operation { + constructor(type, argument, status) { + this._type = type; + this._argument = argument; + this._status = status; + } + + get type() { + return this._type; + } + get argument() { + return this._argument; + } + + complete(result) { + this._status._onCompletion(result); + } + error(reason) { + this._status._onError(reason); + } +} + +class OperationStream { + constructor(strategy) { + this._queue = []; + this._queueSize = 0; + + this._strategy = strategy; + this._window = 0; + + this._writableState = 'waiting'; + this._initWritableReadyPromise(); + + this._updateWritableState(); + + this._cancelOperation = undefined; + this._cancelledPromise = new Promise((resolve, reject) => { + this._resolveCancelledPromise = resolve; + }); + + this._readableState = 'waiting'; + this._initReadableReadyPromise(); + + this._abortOperation = undefined; + this._abortedPromise = new Promise((resolve, reject) => { + this._resolveAbortedPromise = resolve; + }); + } + + _initWritableReadyPromise() { + this._writableReadyPromise = new Promise((resolve, reject) => { + this._resolveWritableReadyPromise = resolve; + }); + } + + _initReadableReadyPromise() { + this._readableReadyPromise = new Promise((resolve, reject) => { + this._resolveReadableReadyPromise = resolve; + }); + } + + // Writable side interfaces + + get writableState() { + return this._writableState; + } + get writableReady() { + return this._writableReadyPromise; + } + + get cancelOperation() { + return this._cancelOperation; + } + get cancelled() { + return this._cancelledPromise; + } + + get space() { + if (this._writableState === 'closed' || + this._writableState === 'aborted' || + this._writableState === 'cancelled') { + return undefined; + } + + return this._strategy.space(this._queueSize); + } + + _checkWritableState() { + if (this._writableState === 'closed') { + return Promise.reject(new TypeError('already closed')); + } + if (this._writableState === 'aborted') { + return Promise.reject(new TypeError('already aborted')); + } + if (this._writableState === 'cancelled') { + return Promise.reject(new TypeError('already cancelled')); + } + return undefined; + } + + _updateWritableState() { + const shouldApplyBackpressure = this._strategy.shouldApplyBackpressure(this._queueSize); + if (shouldApplyBackpressure && this._writableState === 'writable') { + this._writableState = 'waiting'; + this._initWritableReadyPromise(); + } else if (!shouldApplyBackpressure && this._writableState === 'waiting') { + this._writableState = 'writable'; + this._resolveWritableReadyPromise(); + } + } + + write(argument) { + const checkResult = this._checkWritableState(); + if (checkResult !== undefined) { + return checkResult; + } + + const size = this._strategy.size(argument); + + const status = new OperationStatus(); + this._queue.push({value: new Operation('data', argument, status), size}); + this._queueSize += size; + + this._updateWritableState(); + + if (this._readableState === 'waiting') { + this._readableState = 'readable'; + this._resolveReadableReadyPromise(); + } + + return status; + } + + close() { + const checkResult = this._checkWritePrecondition(); + if (checkResult !== undefined) { + return checkResult; + } + + this._strategy = undefined; + + const status = new OperationStatus(); + this._queue.push({value: new Operation('close', undefined, status), size: 0}); + + this._writableState = 'closed'; + + if (this._readableStream === 'waiting') { + this._readableState = 'readable'; + this._resolveReadableReadyPromise(); + } + + + return status; + } + + abort(reason) { + if (this._writableState === 'aborted') { + return Promise.reject(new TypeError('already aborted')); + } + if (this._writableState === 'cancelled') { + return Promise.reject(new TypeError('already cancelled')) + } + + for (var i = this._queue.length - 1; i >= 0; --i) { + const op = this._queue[i].value; + op.error(new TypeError('aborted')); + } + this._queue = []; + this._strategy = undefined; + + if (this._writableState === 'waiting') { + this._resolveWritableReadyPromise(); + } + this._writableState = 'aborted'; + + const status = new OperationStatus(); + this._abortOperation = new Operation('abort', reason, status); + this._resolveAbortedPromise(); + + if (this._readableState === 'waiting') { + this._resolveReadableReadyPromise(); + } + this._readableState = 'aborted'; + + return status; + } + + // Readable side interfaces. + + get readableState() { + return this._readableState; + } + get readableReady() { + return this._readableReadyPromise; + } + + get abortOperation() { + return this._abortOperation; + } + get aborted() { + return this._abortedPromise; + } + + get window() { + return this._window; + } + set window(v) { + this._window = v; + + if (this._writableState === 'closed' || + this._writableState === 'aborted' || + this._writableState === 'cancelled') { + return; + } + + this._strategy.onWindowUpdate(v); + this._updateWritableState(); + } + + _checkReadableState() { + if (this._readableState === 'drained') { + throw new TypeError('already drained'); + } + if (this._readableState === 'cancelled') { + throw new TypeError('already cancelled'); + } + if (this._readableState === 'aborted') { + throw new TypeError('already aborted'); + } + } + + read() { + this._checkReadableState(); + + if (this._queue.length === 0) { + throw new TypeError('not readable'); + } + + const entry = this._queue.shift(); + this._queueSize -= entry.size; + + this._updateWritableState(); + + if (this._queue.length === 0) { + if (entry.type === 'close') { + this._readableState = 'drained'; + } else { + this._readableState = 'waiting'; + this._initReadableReadyPromise(); + } + } + + return entry.value; + } + + cancel(reason) { + this._checkReadableState(); + + for (var i = 0; i < this._queue.length; ++i) { + const op = this._queue[i].value; + op.error(new TypeError('cancelled')); + } + this._queue = []; + this._strategy = undefined; + + const status = new OperationStatus(); + this._cancelOperation = new Operation('cancel', reason, status); + this._resolveCancelledPromise(); + + if (this._writableState === 'waiting') { + this._resolveWritableReadyPromise(); + } + this._writableState = 'cancelled'; + + if (this._readableState === 'waiting') { + this._resolveReadableReadyPromise(); + } + this._readableState = 'cancelled'; + + return status; + } +} + +// Wrappers to hide the interfaces of the other side. + +class WritableOperationStream { + constructor(stream) { + this._stream = stream; + } + + get state() { + return this._stream.writableState; + } + get ready() { + return this._stream.writableReady; + } + + get cancelOperation() { + return this._stream.cancelOperation; + } + get cancelled() { + return this._stream.cancelled; + } + + get space() { + return this._stream.space; + } + + write(value) { + return this._stream.write(value); + } + close() { + return this._stream.close(); + } + abort(reason) { + return this._stream.abort(reason); + } +} + +class ReadableOperationStream { + constructor(stream) { + this._stream = stream; + } + + get state() { + return this._stream.readableState; + } + get ready() { + return this._stream.readableReady; + } + + get abortOperation() { + return this._stream.abortOperation; + } + get aborted() { + return this._stream.aborted; + } + + get window() { + return this._stream.window; + } + set window(v) { + this._stream.window = v; + } + + read() { + return this._stream.read(); + } + cancel(reason) { + return this._stream.cancel(reason); + } +} diff --git a/reference-implementation/run-tests.js b/reference-implementation/run-tests.js index 7d6e1463d..cbf9fae39 100644 --- a/reference-implementation/run-tests.js +++ b/reference-implementation/run-tests.js @@ -4,12 +4,14 @@ const path = require('path'); import ReadableStream from './lib/readable-stream'; import WritableStream from './lib/writable-stream'; import ReadableByteStream from './lib/experimental/readable-byte-stream'; +import createOperationStream from './lib/experimental/operation-stream'; import ByteLengthQueuingStrategy from './lib/byte-length-queuing-strategy'; import CountQueuingStrategy from './lib/count-queuing-strategy'; import TransformStream from './lib/transform-stream'; global.ReadableStream = ReadableStream; global.WritableStream = WritableStream; +global.createOperationStream = createOperationStream; global.ReadableByteStream = ReadableByteStream; global.ByteLengthQueuingStrategy = ByteLengthQueuingStrategy; global.CountQueuingStrategy = CountQueuingStrategy; diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js new file mode 100644 index 000000000..42f517db5 --- /dev/null +++ b/reference-implementation/test/experimental/operation-stream.js @@ -0,0 +1,157 @@ +const test = require('tape-catch'); + +test('Operation stream pair is constructed', t => { + const pair = createOperationStream({ + size() { + return 1; + }, + shouldApplyBackpressure() { + return false; + } + }); + + t.end(); +}); + +test('Synchronous write, read and completion of the operation', t => { + const pair = createOperationStream({ + size() { + return 1; + }, + shouldApplyBackpressure(queueSize) { + return queueSize > 0; + } + }); + const wos = pair.writable; + const ros = pair.readable; + + t.equals(wos.state, 'writable'); + t.equals(ros.state, 'waiting'); + + const status = wos.write('hello'); + + t.equals(wos.state, 'waiting'); + t.equals(ros.state, 'readable'); + + t.equals(status.state, 'waiting'); + + const op = ros.read(); + + t.equals(ros.state, 'waiting'); + t.equals(wos.state, 'writable'); + + t.equals(status.state, 'waiting'); + + op.complete('world'); + + t.equals(status.state, 'completed'); + t.equals(status.result, 'world'); + + t.end(); +}); + +test('Asynchronous write, read and completion of the operation', t => { + const pair = createOperationStream({ + size() { + return 1; + }, + shouldApplyBackpressure(queueSize) { + return queueSize > 0; + } + }); + const wos = pair.writable; + const ros = pair.readable; + + t.equals(ros.state, 'waiting'); + + var calledComplete = false; + + ros.ready.then(() => { + t.equals(ros.state, 'readable'); + + const op = ros.read(); + + t.equals(ros.state, 'waiting'); + + op.complete('world'); + calledComplete = true; + }, e => { + t.fail(e); + t.end(); + }); + + t.equals(wos.state, 'writable'); + + const status = wos.write('hello'); + + t.equals(wos.state, 'waiting'); + + t.equals(status.state, 'waiting'); + status.ready.then(() => { + t.equals(calledComplete, true); + + t.equals(status.state, 'completed'); + t.equals(status.result, 'world'); + + t.end(); + }, e => { + t.fail(e); + t.end(); + }) +}); + +test.only('Asynchronous write, read and completion of the operation', t => { + class AdjustableStrategy { + constructor() { + this._window = 0; + } + + size(ab) { + return ab.byteLength; + } + shouldApplyBackpressure(queueSize) { + return queueSize >= this._window; + } + space(queueSize) { + return Math.max(0, this._window - queueSize); + } + onWindowUpdate(window) { + this._window = window; + } + } + + const pair = createOperationStream(new AdjustableStrategy()); + const wos = pair.writable; + const ros = pair.readable; + + t.equals(wos.state, 'waiting'); + + ros.window = 5; + t.equals(wos.state, 'writable'); + t.equals(wos.space, 5); + + ros.window = 0; + const status = wos.write(new ArrayBuffer(10)); + + t.equals(wos.state, 'waiting'); + t.equals(wos.space, 0); + + ros.window = 10 + t.equals(wos.state, 'waiting'); + t.equals(wos.space, 0); + + ros.window = 15 + t.equals(wos.state, 'writable'); + t.equals(wos.space, 5); + + ros.window = 20 + t.equals(wos.state, 'writable'); + t.equals(wos.space, 10); + + ros.read(); + + t.equals(wos.state, 'writable'); + t.equals(wos.space, 20); + + t.end(); +}); From bf34cddf3a62790a52343976f70b8fc1087658d2 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Fri, 20 Feb 2015 05:25:52 +0900 Subject: [PATCH 02/93] Fix operation stream and add a sample of network API implementation example as a test case --- .../lib/experimental/operation-stream.js | 10 +- .../test/experimental/operation-stream.js | 180 ++++++++++++++++-- 2 files changed, 169 insertions(+), 21 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 28fe4160d..e25c14827 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -173,7 +173,7 @@ class OperationStream { } close() { - const checkResult = this._checkWritePrecondition(); + const checkResult = this._checkWritableState(); if (checkResult !== undefined) { return checkResult; } @@ -185,12 +185,11 @@ class OperationStream { this._writableState = 'closed'; - if (this._readableStream === 'waiting') { + if (this._readableState === 'waiting') { this._readableState = 'readable'; this._resolveReadableReadyPromise(); } - return status; } @@ -280,7 +279,10 @@ class OperationStream { const entry = this._queue.shift(); this._queueSize -= entry.size; - this._updateWritableState(); + if (this._writableState === 'writable' || + this._writableState === 'waiting') { + this._updateWritableState(); + } if (this._queue.length === 0) { if (entry.type === 'close') { diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 42f517db5..4e5d5663d 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -36,6 +36,7 @@ test('Synchronous write, read and completion of the operation', t => { t.equals(status.state, 'waiting'); const op = ros.read(); + t.equals(op.argument, 'hello'); t.equals(ros.state, 'waiting'); t.equals(wos.state, 'writable'); @@ -70,6 +71,7 @@ test('Asynchronous write, read and completion of the operation', t => { t.equals(ros.state, 'readable'); const op = ros.read(); + t.equals(op.argument, 'hello'); t.equals(ros.state, 'waiting'); @@ -100,26 +102,26 @@ test('Asynchronous write, read and completion of the operation', t => { }) }); -test.only('Asynchronous write, read and completion of the operation', t => { - class AdjustableStrategy { - constructor() { - this._window = 0; - } +class AdjustableStrategy { + constructor() { + this._window = 0; + } - size(ab) { - return ab.byteLength; - } - shouldApplyBackpressure(queueSize) { - return queueSize >= this._window; - } - space(queueSize) { - return Math.max(0, this._window - queueSize); - } - onWindowUpdate(window) { - this._window = window; - } + size(ab) { + return ab.byteLength; + } + shouldApplyBackpressure(queueSize) { + return queueSize >= this._window; + } + space(queueSize) { + return Math.max(0, this._window - queueSize); } + onWindowUpdate(window) { + this._window = window; + } +} +test('Asynchronous write, read and completion of the operation', t => { const pair = createOperationStream(new AdjustableStrategy()); const wos = pair.writable; const ros = pair.readable; @@ -155,3 +157,147 @@ test.only('Asynchronous write, read and completion of the operation', t => { t.end(); }); + +test.only('Sample implementation of network API with a buffer pool', t => { + const pair = createOperationStream(new AdjustableStrategy()); + const wos = pair.writable; + const ros = pair.readable; + + var bytesRead = 0; + var readableClosed = false; + function pump() { + for (;;) { + if (ros.state === 'readable') { + const op = ros.read(); + if (op.type === 'data') { + const view = op.argument; + for (var i = 0; i < view.byteLength; ++i) { + if (view[i] === 1) { + ++bytesRead; + } + } + } else { + readableClosed = true; + } + op.complete(); + } else if (ros.state === 'waiting') { + ros.ready.then(pump); + return; + } + } + } + pump(); + + ros.window = 64; + + new Promise((resolve, reject) => { + const bufferPool = []; + for (var i = 0; i < 10; ++i) { + bufferPool.push(new ArrayBuffer(10)); + } + + var networkReadPromise = undefined; + + const buffersInUse = []; + + var bytesToWrite = 1024; + + function fakeReadFromNetworkLoop() { + for (;;) { + if (wos.state === 'cancelled') { + reject(); + return; + } + + var hasProgress = false; + + if (buffersInUse.length > 0) { + const entry = buffersInUse[0]; + const status = entry.status; + if (status.state === 'completed') { + buffersInUse.shift(); + + if (entry.buffer === undefined) { + resolve(); + return; + } + + bufferPool.push(entry.buffer); + + hasProgress = true; + } else if (status.state === 'errored') { + reject(); + return; + } + } + + if (networkReadPromise === undefined && bufferPool.length > 0 && wos.state === 'writable') { + const buffer = bufferPool.shift(); + const view = new Uint8Array(buffer); + for (var i = 0; i < view.byteLength; ++i) { + view[0] = 0; + } + + // Fake async network read operation. + networkReadPromise = new Promise((resolve, reject) => { + setTimeout(() => { + const bytesToWriteThisTime = Math.min(bytesToWrite, buffer.byteLength); + const view = new Uint8Array(buffer, 0, bytesToWriteThisTime); + for (var i = 0; i < view.byteLength; ++i) { + view[i] = 1; + } + bytesToWrite -= bytesToWriteThisTime; + if (bytesToWrite === 0) { + resolve({close: true, view}); + } else { + resolve({close: false, view}); + } + }, 0); + }).then(result => { + networkReadPromise = undefined; + if (result.close) { + buffersInUse.push({buffer, status: wos.write(result.view)}); + buffersInUse.push({buffer: undefined, status: wos.close()}); + } else { + buffersInUse.push({buffer, status: wos.write(result.view)}); + } + }); + + hasProgress = true; + } + + if (hasProgress) { + continue; + } + + const promisesToRace = []; + + if (wos.state === 'writable') { + promisesToRace.push(wos.cancelled); + } else if (wos.state === 'waiting') { + promisesToRace.push(wos.ready); + } + + if (networkReadPromise !== undefined) { + promisesToRace.push(networkReadPromise); + } + + if (buffersInUse.length > 0) { + promisesToRace.push(buffersInUse[0].status.ready); + } + + Promise.race(promisesToRace).then(fakeReadFromNetworkLoop); + return; + } + } + fakeReadFromNetworkLoop(); + }).then( + () => { + t.equals(bytesRead, 1024); + t.equals(readableClosed, true); + t.end() + }, e => { + t.fail(e); + t.end(); + }); +}); From aef7cb91289041a5bb2cbc86ea50e25158053d1b Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Fri, 20 Feb 2015 05:30:24 +0900 Subject: [PATCH 03/93] Remove test.only --- reference-implementation/test/experimental/operation-stream.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 4e5d5663d..704a91804 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -158,7 +158,7 @@ test('Asynchronous write, read and completion of the operation', t => { t.end(); }); -test.only('Sample implementation of network API with a buffer pool', t => { +test('Sample implementation of network API with a buffer pool', t => { const pair = createOperationStream(new AdjustableStrategy()); const wos = pair.writable; const ros = pair.readable; From f42962164b8d3906f73f252a2034bd522fd955d7 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Fri, 20 Feb 2015 13:38:08 +0900 Subject: [PATCH 04/93] Add pipe --- .../lib/experimental/operation-stream.js | 54 +++++++++++++++++- reference-implementation/run-tests.js | 3 +- .../test/experimental/operation-stream.js | 55 +++++++++++++++++++ 3 files changed, 110 insertions(+), 2 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index e25c14827..f6d3d39cc 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -1,4 +1,4 @@ -export default function createOperationStream(strategy) { +export function createOperationStream(strategy) { var stream = new OperationStream(strategy); return { writable: new WritableOperationStream(stream), @@ -6,6 +6,58 @@ export default function createOperationStream(strategy) { }; } +function jointOps(op, status) { + function forward() { + if (status.state === 'waiting') { + status.ready.then(forward); + } else if (status.state === 'errored') { + op.error(status.result); + } else { + op.complete(status.result); + } + } + forward(); +} + +export function pipeOperationStreams(readable, writable) { + return new Promise((resolve, reject) => { + function loop() { + for (;;) { + if (readable.state === 'aborted' || writable.state === 'cancelled') { + reject(); + return; + } + + if (readable.state === 'readable' && writable.state === 'writable') { + const op = readable.read(); + const status = writable.write(op.argument); + jointOps(op, status); + + continue; + } + + const promisesToRace = []; + + if (readable.state === 'readable') { + promisesToRace.push(readable.aborted); + } else { + promisesToRace.push(readable.ready); + } + + if (writable.state === 'writable') { + promisesToRace.push(writable.cancelled); + } else { + promisesToRace.push(writable.ready); + } + + Promise.race(promisesToRace).then(loop); + return; + } + } + loop(); + }); +} + class OperationStatus { constructor() { this._state = 'waiting'; diff --git a/reference-implementation/run-tests.js b/reference-implementation/run-tests.js index cbf9fae39..34c622896 100644 --- a/reference-implementation/run-tests.js +++ b/reference-implementation/run-tests.js @@ -4,7 +4,7 @@ const path = require('path'); import ReadableStream from './lib/readable-stream'; import WritableStream from './lib/writable-stream'; import ReadableByteStream from './lib/experimental/readable-byte-stream'; -import createOperationStream from './lib/experimental/operation-stream'; +import { createOperationStream, pipeOperationStreams } from './lib/experimental/operation-stream'; import ByteLengthQueuingStrategy from './lib/byte-length-queuing-strategy'; import CountQueuingStrategy from './lib/count-queuing-strategy'; import TransformStream from './lib/transform-stream'; @@ -12,6 +12,7 @@ import TransformStream from './lib/transform-stream'; global.ReadableStream = ReadableStream; global.WritableStream = WritableStream; global.createOperationStream = createOperationStream; +global.pipeOperationStreams = pipeOperationStreams; global.ReadableByteStream = ReadableByteStream; global.ByteLengthQueuingStrategy = ByteLengthQueuingStrategy; global.CountQueuingStrategy = CountQueuingStrategy; diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 704a91804..deed9bc34 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -121,6 +121,25 @@ class AdjustableStrategy { } } +class AdjustableStringStrategy { + constructor() { + this._window = 0; + } + + size(s) { + return s.length; + } + shouldApplyBackpressure(queueSize) { + return queueSize >= this._window; + } + space(queueSize) { + return Math.max(0, this._window - queueSize); + } + onWindowUpdate(window) { + this._window = window; + } +} + test('Asynchronous write, read and completion of the operation', t => { const pair = createOperationStream(new AdjustableStrategy()); const wos = pair.writable; @@ -158,6 +177,42 @@ test('Asynchronous write, read and completion of the operation', t => { t.end(); }); +test.only('Pipe', t => { + const pair0 = createOperationStream(new AdjustableStringStrategy()); + const wos0 = pair0.writable; + const ros0 = pair0.readable; + + const pair1 = createOperationStream(new AdjustableStringStrategy()); + const wos1 = pair1.writable; + const ros1 = pair1.readable; + + wos0.write('hello'); + wos0.write('world'); + + pipeOperationStreams(ros0, wos1); + + t.equals(ros1.state, 'waiting'); + + ros1.window = 10; + + t.equals(ros1.state, 'waiting'); + + ros1.ready.then(() => { + t.equals(ros1.state, 'readable'); + const op1 = ros1.read(); + t.equals(op1.argument, 'hello'); + + t.equals(ros1.state, 'readable'); + const op2 = ros1.read(); + t.equals(op2.argument, 'world'); + + t.end(); + }).catch(e => { + t.fail(e); + t.end(); + }); +}); + test('Sample implementation of network API with a buffer pool', t => { const pair = createOperationStream(new AdjustableStrategy()); const wos = pair.writable; From e1cc665415d3cc9ddde6f46b967793471f21cf40 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Fri, 20 Feb 2015 13:40:57 +0900 Subject: [PATCH 05/93] In pipe test, also check one operation. --- .../test/experimental/operation-stream.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index deed9bc34..6fd57fc81 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -186,7 +186,7 @@ test.only('Pipe', t => { const wos1 = pair1.writable; const ros1 = pair1.readable; - wos0.write('hello'); + const helloStatus = wos0.write('hello'); wos0.write('world'); pipeOperationStreams(ros0, wos1); @@ -202,11 +202,18 @@ test.only('Pipe', t => { const op1 = ros1.read(); t.equals(op1.argument, 'hello'); + op1.complete('hi'); + t.equals(ros1.state, 'readable'); const op2 = ros1.read(); t.equals(op2.argument, 'world'); - t.end(); + t.equals(helloStatus.state, 'waiting'); + helloStatus.ready.then(() => { + t.equals(helloStatus.state, 'completed'); + t.equals(helloStatus.result, 'hi'); + t.end(); + }); }).catch(e => { t.fail(e); t.end(); From 02e1a6c82e5c3f6c998cb30f875a6ee8b20e9acb Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Fri, 20 Feb 2015 13:46:14 +0900 Subject: [PATCH 06/93] Make pipe work with close --- .../lib/experimental/operation-stream.js | 8 ++++++-- .../test/experimental/operation-stream.js | 17 ++++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index f6d3d39cc..4878147e0 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -30,8 +30,12 @@ export function pipeOperationStreams(readable, writable) { if (readable.state === 'readable' && writable.state === 'writable') { const op = readable.read(); - const status = writable.write(op.argument); - jointOps(op, status); + if (op.type === 'data') { + jointOps(op, writable.write(op.argument)); + } else { + jointOps(op, writable.close(op.argument)); + return; + } continue; } diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 6fd57fc81..c9d7fb4ff 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -188,25 +188,32 @@ test.only('Pipe', t => { const helloStatus = wos0.write('hello'); wos0.write('world'); + wos0.close(); pipeOperationStreams(ros0, wos1); t.equals(ros1.state, 'waiting'); - ros1.window = 10; + ros1.window = 20; t.equals(ros1.state, 'waiting'); ros1.ready.then(() => { t.equals(ros1.state, 'readable'); - const op1 = ros1.read(); - t.equals(op1.argument, 'hello'); + const op0 = ros1.read(); + t.equals(op0.type, 'data'); + t.equals(op0.argument, 'hello'); + + op0.complete('hi'); - op1.complete('hi'); + t.equals(ros1.state, 'readable'); + const op1 = ros1.read(); + t.equals(op1.type, 'data'); + t.equals(op1.argument, 'world'); t.equals(ros1.state, 'readable'); const op2 = ros1.read(); - t.equals(op2.argument, 'world'); + t.equals(op2.type, 'close'); t.equals(helloStatus.state, 'waiting'); helloStatus.ready.then(() => { From ff8e97c99396e75fcbe4326cb2295cdab60fca0a Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Fri, 20 Feb 2015 15:06:41 +0900 Subject: [PATCH 07/93] Factor out fake network reading code into FakeByteSource --- .../test/experimental/operation-stream.js | 227 +++++++++--------- 1 file changed, 114 insertions(+), 113 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index c9d7fb4ff..3a0f53945 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -177,7 +177,7 @@ test('Asynchronous write, read and completion of the operation', t => { t.end(); }); -test.only('Pipe', t => { +test('Pipe', t => { const pair0 = createOperationStream(new AdjustableStringStrategy()); const wos0 = pair0.writable; const ros0 = pair0.readable; @@ -227,146 +227,147 @@ test.only('Pipe', t => { }); }); -test('Sample implementation of network API with a buffer pool', t => { - const pair = createOperationStream(new AdjustableStrategy()); - const wos = pair.writable; - const ros = pair.readable; - - var bytesRead = 0; - var readableClosed = false; - function pump() { - for (;;) { - if (ros.state === 'readable') { - const op = ros.read(); - if (op.type === 'data') { - const view = op.argument; - for (var i = 0; i < view.byteLength; ++i) { - if (view[i] === 1) { - ++bytesRead; - } - } - } else { - readableClosed = true; - } - op.complete(); - } else if (ros.state === 'waiting') { - ros.ready.then(pump); - return; - } - } - } - pump(); +class FakeByteSource { + constructor() { + this._streams = createOperationStream(new AdjustableStrategy()); - ros.window = 64; + this._bytesToWrite = 1024; - new Promise((resolve, reject) => { - const bufferPool = []; + this._bufferPool = []; for (var i = 0; i < 10; ++i) { - bufferPool.push(new ArrayBuffer(10)); + this._bufferPool.push(new ArrayBuffer(10)); } - var networkReadPromise = undefined; + this._buffersInUse = []; - const buffersInUse = []; + this._networkReadPromise = undefined; - var bytesToWrite = 1024; + this._loop(); + } - function fakeReadFromNetworkLoop() { - for (;;) { - if (wos.state === 'cancelled') { - reject(); - return; - } + get stream() { + return this._streams.readable; + } - var hasProgress = false; + _loop() { + const wos = this._streams.writable; - if (buffersInUse.length > 0) { - const entry = buffersInUse[0]; - const status = entry.status; - if (status.state === 'completed') { - buffersInUse.shift(); + for (;;) { + if (wos.state === 'cancelled') { + return; + } - if (entry.buffer === undefined) { - resolve(); - return; - } + var hasProgress = false; - bufferPool.push(entry.buffer); + if (this._buffersInUse.length > 0) { + const entry = this._buffersInUse[0]; + const status = entry.status; + if (status.state === 'completed') { + this._buffersInUse.shift(); - hasProgress = true; - } else if (status.state === 'errored') { - reject(); + if (entry.buffer === undefined) { return; } + + this._bufferPool.push(entry.buffer); + + hasProgress = true; + } else if (status.state === 'errored') { + return; } + } - if (networkReadPromise === undefined && bufferPool.length > 0 && wos.state === 'writable') { - const buffer = bufferPool.shift(); - const view = new Uint8Array(buffer); - for (var i = 0; i < view.byteLength; ++i) { - view[0] = 0; - } + if (this._networkReadPromise === undefined && + this._bufferPool.length > 0 && + wos.state === 'writable') { + const buffer = this._bufferPool.shift(); + const view = new Uint8Array(buffer); + for (var i = 0; i < view.byteLength; ++i) { + view[0] = 0; + } - // Fake async network read operation. - networkReadPromise = new Promise((resolve, reject) => { - setTimeout(() => { - const bytesToWriteThisTime = Math.min(bytesToWrite, buffer.byteLength); - const view = new Uint8Array(buffer, 0, bytesToWriteThisTime); - for (var i = 0; i < view.byteLength; ++i) { - view[i] = 1; - } - bytesToWrite -= bytesToWriteThisTime; - if (bytesToWrite === 0) { - resolve({close: true, view}); - } else { - resolve({close: false, view}); - } - }, 0); - }).then(result => { - networkReadPromise = undefined; - if (result.close) { - buffersInUse.push({buffer, status: wos.write(result.view)}); - buffersInUse.push({buffer: undefined, status: wos.close()}); + // Fake async network read operation. + this._networkReadPromise = new Promise((resolve, reject) => { + setTimeout(() => { + const bytesToWriteThisTime = Math.min(this._bytesToWrite, buffer.byteLength); + const view = new Uint8Array(buffer, 0, bytesToWriteThisTime); + for (var i = 0; i < view.byteLength; ++i) { + view[i] = 1; + } + this._bytesToWrite -= bytesToWriteThisTime; + if (this._bytesToWrite === 0) { + resolve({close: true, view}); } else { - buffersInUse.push({buffer, status: wos.write(result.view)}); + resolve({close: false, view}); } - }); + }, 0); + }).then(result => { + this._networkReadPromise = undefined; + if (result.close) { + this._buffersInUse.push({buffer, status: wos.write(result.view)}); + this._buffersInUse.push({buffer: undefined, status: wos.close()}); + } else { + this._buffersInUse.push({buffer, status: wos.write(result.view)}); + } + }); - hasProgress = true; - } + hasProgress = true; + } - if (hasProgress) { - continue; - } + if (hasProgress) { + continue; + } - const promisesToRace = []; + const promisesToRace = []; - if (wos.state === 'writable') { - promisesToRace.push(wos.cancelled); - } else if (wos.state === 'waiting') { - promisesToRace.push(wos.ready); - } + if (wos.state === 'writable') { + promisesToRace.push(wos.cancelled); + } else if (wos.state === 'waiting') { + promisesToRace.push(wos.ready); + } - if (networkReadPromise !== undefined) { - promisesToRace.push(networkReadPromise); - } + if (this._networkReadPromise !== undefined) { + promisesToRace.push(this._networkReadPromise); + } - if (buffersInUse.length > 0) { - promisesToRace.push(buffersInUse[0].status.ready); - } + if (this._buffersInUse.length > 0) { + promisesToRace.push(this._buffersInUse[0].status.ready); + } + + Promise.race(promisesToRace).then(this._loop.bind(this)); + return; + } + } +} + +test.only('Sample implementation of network API with a buffer pool', t => { + const bs = new FakeByteSource(); + const ros = bs.stream; + ros.window = 64; - Promise.race(promisesToRace).then(fakeReadFromNetworkLoop); + var bytesRead = 0; + function pump() { + for (;;) { + if (ros.state === 'readable') { + const op = ros.read(); + if (op.type === 'data') { + const view = op.argument; + for (var i = 0; i < view.byteLength; ++i) { + if (view[i] === 1) { + ++bytesRead; + } + } + } else { + t.equals(bytesRead, 1024); + + t.end() + } + op.complete(); + } else if (ros.state === 'waiting') { + ros.ready.then(pump); return; } } - fakeReadFromNetworkLoop(); - }).then( - () => { - t.equals(bytesRead, 1024); - t.equals(readableClosed, true); - t.end() - }, e => { - t.fail(e); - t.end(); - }); + } + pump(); }); From c0b48cb7819e9931f0d67a9919f9eef39bcba646 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Fri, 20 Feb 2015 15:39:44 +0900 Subject: [PATCH 08/93] Reorganize strategies in operation stream test --- .../lib/experimental/operation-stream.js | 20 +++- .../test/experimental/operation-stream.js | 109 +++++++++--------- 2 files changed, 68 insertions(+), 61 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 4878147e0..1506d7eab 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -179,7 +179,11 @@ class OperationStream { return undefined; } - return this._strategy.space(this._queueSize); + if ('space' in this._strategy) { + return this._strategy.space(this._queueSize); + } + + return undefined; } _checkWritableState() { @@ -196,7 +200,10 @@ class OperationStream { } _updateWritableState() { - const shouldApplyBackpressure = this._strategy.shouldApplyBackpressure(this._queueSize); + var shouldApplyBackpressure = false; + if ('shouldApplyBackpressure' in this._strategy) { + shouldApplyBackpressure = this._strategy.shouldApplyBackpressure(this._queueSize); + } if (shouldApplyBackpressure && this._writableState === 'writable') { this._writableState = 'waiting'; this._initWritableReadyPromise(); @@ -212,7 +219,10 @@ class OperationStream { return checkResult; } - const size = this._strategy.size(argument); + var size = 1; + if ('size' in this._strategy) { + size = this._strategy.size(argument); + } const status = new OperationStatus(); this._queue.push({value: new Operation('data', argument, status), size}); @@ -309,7 +319,9 @@ class OperationStream { return; } - this._strategy.onWindowUpdate(v); + if ('onWindowUpdate' in this._strategy) { + this._strategy.onWindowUpdate(v); + } this._updateWritableState(); } diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 3a0f53945..09f5ffe65 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -13,15 +13,55 @@ test('Operation stream pair is constructed', t => { t.end(); }); +class NoBackpressureStrategy { +} + +class ApplyBackpressureWhenNonEmptyStrategy { + shouldApplyBackpressure(queueSize) { + return queueSize > 0; + } +} + +class AdjustableArrayBufferStrategy { + constructor() { + this._window = 0; + } + + size(ab) { + return ab.byteLength; + } + shouldApplyBackpressure(queueSize) { + return queueSize >= this._window; + } + space(queueSize) { + return Math.max(0, this._window - queueSize); + } + onWindowUpdate(window) { + this._window = window; + } +} + +class AdjustableStringStrategy { + constructor() { + this._window = 0; + } + + size(s) { + return s.length; + } + shouldApplyBackpressure(queueSize) { + return queueSize >= this._window; + } + space(queueSize) { + return Math.max(0, this._window - queueSize); + } + onWindowUpdate(window) { + this._window = window; + } +} + test('Synchronous write, read and completion of the operation', t => { - const pair = createOperationStream({ - size() { - return 1; - }, - shouldApplyBackpressure(queueSize) { - return queueSize > 0; - } - }); + const pair = createOperationStream(new ApplyBackpressureWhenNonEmptyStrategy()); const wos = pair.writable; const ros = pair.readable; @@ -52,14 +92,7 @@ test('Synchronous write, read and completion of the operation', t => { }); test('Asynchronous write, read and completion of the operation', t => { - const pair = createOperationStream({ - size() { - return 1; - }, - shouldApplyBackpressure(queueSize) { - return queueSize > 0; - } - }); + const pair = createOperationStream(new ApplyBackpressureWhenNonEmptyStrategy()); const wos = pair.writable; const ros = pair.readable; @@ -102,46 +135,8 @@ test('Asynchronous write, read and completion of the operation', t => { }) }); -class AdjustableStrategy { - constructor() { - this._window = 0; - } - - size(ab) { - return ab.byteLength; - } - shouldApplyBackpressure(queueSize) { - return queueSize >= this._window; - } - space(queueSize) { - return Math.max(0, this._window - queueSize); - } - onWindowUpdate(window) { - this._window = window; - } -} - -class AdjustableStringStrategy { - constructor() { - this._window = 0; - } - - size(s) { - return s.length; - } - shouldApplyBackpressure(queueSize) { - return queueSize >= this._window; - } - space(queueSize) { - return Math.max(0, this._window - queueSize); - } - onWindowUpdate(window) { - this._window = window; - } -} - test('Asynchronous write, read and completion of the operation', t => { - const pair = createOperationStream(new AdjustableStrategy()); + const pair = createOperationStream(new AdjustableArrayBufferStrategy()); const wos = pair.writable; const ros = pair.readable; @@ -229,7 +224,7 @@ test('Pipe', t => { class FakeByteSource { constructor() { - this._streams = createOperationStream(new AdjustableStrategy()); + this._streams = createOperationStream(new AdjustableArrayBufferStrategy()); this._bytesToWrite = 1024; @@ -340,7 +335,7 @@ class FakeByteSource { } } -test.only('Sample implementation of network API with a buffer pool', t => { +test('Sample implementation of network API with a buffer pool', t => { const bs = new FakeByteSource(); const ros = bs.stream; ros.window = 64; From 4f658237afcd6b24f4a5d9295d8e8a088fcec81c Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Fri, 20 Feb 2015 16:36:36 +0900 Subject: [PATCH 09/93] Reorganized operation stream file API sample --- reference-implementation/run-tests.js | 7 +- .../test/experimental/operation-stream.js | 116 ++++++++++-------- 2 files changed, 72 insertions(+), 51 deletions(-) diff --git a/reference-implementation/run-tests.js b/reference-implementation/run-tests.js index 34c622896..9f7991850 100644 --- a/reference-implementation/run-tests.js +++ b/reference-implementation/run-tests.js @@ -19,6 +19,7 @@ global.CountQueuingStrategy = CountQueuingStrategy; global.TransformStream = TransformStream; -const tests = glob.sync(path.resolve(__dirname, 'test/*.js')); -const experimentalTests = glob.sync(path.resolve(__dirname, 'test/experimental/*.js')); -tests.concat(experimentalTests).forEach(require); +//const tests = glob.sync(path.resolve(__dirname, 'test/*.js')); +//const experimentalTests = glob.sync(path.resolve(__dirname, 'test/experimental/*.js')); +//tests.concat(experimentalTests).forEach(require); +glob.sync(path.resolve(__dirname, 'test/experimental/operation-stream.js')).forEach(require); diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 09f5ffe65..36017d161 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -222,20 +222,50 @@ test('Pipe', t => { }); }); -class FakeByteSource { +class FakeFile { + constructor() { + this._bytesToWrite = 1024; + } + + readInto(buffer) { + return new Promise((resolve, reject) => { + setTimeout(() => { + try { + const bytesToWriteThisTime = Math.min(this._bytesToWrite, buffer.byteLength); + + const view = new Uint8Array(buffer, 0, bytesToWriteThisTime); + for (var i = 0; i < view.byteLength; ++i) { + view[i] = 1; + } + + this._bytesToWrite -= bytesToWriteThisTime; + + if (this._bytesToWrite === 0) { + resolve({closed: true, view}); + } else { + resolve({closed: false, view}); + } + } catch (e) { + reject(e); + } + }, 0); + }); + } +} + +class FakeByteSourceWithBufferPool { constructor() { this._streams = createOperationStream(new AdjustableArrayBufferStrategy()); - this._bytesToWrite = 1024; + this._file = new FakeFile(); + this._fileReadPromise = undefined; - this._bufferPool = []; + this._buffersAvailable = []; for (var i = 0; i < 10; ++i) { - this._bufferPool.push(new ArrayBuffer(10)); + this._buffersAvailable.push(new ArrayBuffer(10)); } - this._buffersInUse = []; - - this._networkReadPromise = undefined; + this._buffersPassedToUser = []; this._loop(); } @@ -244,6 +274,15 @@ class FakeByteSource { return this._streams.readable; } + _handleFileReadResult(buffer, result) { + this._fileReadPromise = undefined; + const status = this._streams.writable.write(result.view); + this._buffersPassedToUser.push({buffer, status}); + if (result.closed) { + this._streams.writable.close(); + } + } + _loop() { const wos = this._streams.writable; @@ -254,57 +293,35 @@ class FakeByteSource { var hasProgress = false; - if (this._buffersInUse.length > 0) { - const entry = this._buffersInUse[0]; + if (this._buffersPassedToUser.length > 0) { + const entry = this._buffersPassedToUser[0]; const status = entry.status; if (status.state === 'completed') { - this._buffersInUse.shift(); + this._buffersPassedToUser.shift(); - if (entry.buffer === undefined) { - return; - } - - this._bufferPool.push(entry.buffer); + this._buffersAvailable.push(entry.buffer); hasProgress = true; + // Keep going. } else if (status.state === 'errored') { + wos.abort(status.result); return; } } - if (this._networkReadPromise === undefined && - this._bufferPool.length > 0 && + if (this._fileReadPromise === undefined && + this._buffersAvailable.length > 0 && wos.state === 'writable') { - const buffer = this._bufferPool.shift(); + const buffer = this._buffersAvailable.shift(); + + // Clear the acquired buffer for testing. const view = new Uint8Array(buffer); for (var i = 0; i < view.byteLength; ++i) { view[0] = 0; } - // Fake async network read operation. - this._networkReadPromise = new Promise((resolve, reject) => { - setTimeout(() => { - const bytesToWriteThisTime = Math.min(this._bytesToWrite, buffer.byteLength); - const view = new Uint8Array(buffer, 0, bytesToWriteThisTime); - for (var i = 0; i < view.byteLength; ++i) { - view[i] = 1; - } - this._bytesToWrite -= bytesToWriteThisTime; - if (this._bytesToWrite === 0) { - resolve({close: true, view}); - } else { - resolve({close: false, view}); - } - }, 0); - }).then(result => { - this._networkReadPromise = undefined; - if (result.close) { - this._buffersInUse.push({buffer, status: wos.write(result.view)}); - this._buffersInUse.push({buffer: undefined, status: wos.close()}); - } else { - this._buffersInUse.push({buffer, status: wos.write(result.view)}); - } - }); + this._fileReadPromise = this._file.readInto(buffer).then( + this._handleFileReadResult.bind(this, buffer)).catch(e => wos.abort(e)); hasProgress = true; } @@ -321,12 +338,12 @@ class FakeByteSource { promisesToRace.push(wos.ready); } - if (this._networkReadPromise !== undefined) { - promisesToRace.push(this._networkReadPromise); + if (this._fileReadPromise !== undefined) { + promisesToRace.push(this._fileReadPromise); } - if (this._buffersInUse.length > 0) { - promisesToRace.push(this._buffersInUse[0].status.ready); + if (this._buffersPassedToUser.length > 0) { + promisesToRace.push(this._buffersPassedToUser[0].status.ready); } Promise.race(promisesToRace).then(this._loop.bind(this)); @@ -335,8 +352,8 @@ class FakeByteSource { } } -test('Sample implementation of network API with a buffer pool', t => { - const bs = new FakeByteSource(); +test('Sample implementation of file API with a buffer pool', t => { + const bs = new FakeByteSourceWithBufferPool(); const ros = bs.stream; ros.window = 64; @@ -361,6 +378,9 @@ test('Sample implementation of network API with a buffer pool', t => { } else if (ros.state === 'waiting') { ros.ready.then(pump); return; + } else if (ros.state === 'cancelled') { + t.fail(ros.cancelOperation.argument); + t.end(); } } } From 80a88d9486161ab817171cc4c85f7973642e7c23 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Fri, 20 Feb 2015 16:43:28 +0900 Subject: [PATCH 10/93] Tweak on operation stream example --- .../test/experimental/operation-stream.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 36017d161..a4d884359 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -364,23 +364,33 @@ test('Sample implementation of file API with a buffer pool', t => { const op = ros.read(); if (op.type === 'data') { const view = op.argument; + + // Verify contents of the buffer. for (var i = 0; i < view.byteLength; ++i) { if (view[i] === 1) { ++bytesRead; } } - } else { + + // Release the buffer. + op.complete(); + } else if (op.type === 'close') { t.equals(bytesRead, 1024); t.end() + } else { + t.fail('Invalid type: ' + op.type); + t.end(); } - op.complete(); } else if (ros.state === 'waiting') { ros.ready.then(pump); return; } else if (ros.state === 'cancelled') { t.fail(ros.cancelOperation.argument); t.end(); + } else { + t.fail('Unexpected state: ' + ros.state); + t.end(); } } } From 67d4195a6e8a27c4fcd80793b84b67533adb461a Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Fri, 20 Feb 2015 16:50:17 +0900 Subject: [PATCH 11/93] Factor out readAndVerify in operation stream test --- .../test/experimental/operation-stream.js | 87 ++++++++++--------- 1 file changed, 48 insertions(+), 39 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index a4d884359..586198149 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -254,16 +254,13 @@ class FakeFile { } class FakeByteSourceWithBufferPool { - constructor() { + constructor(buffers) { this._streams = createOperationStream(new AdjustableArrayBufferStrategy()); this._file = new FakeFile(); this._fileReadPromise = undefined; - this._buffersAvailable = []; - for (var i = 0; i < 10; ++i) { - this._buffersAvailable.push(new ArrayBuffer(10)); - } + this._buffersAvailable = buffers; this._buffersPassedToUser = []; @@ -352,47 +349,59 @@ class FakeByteSourceWithBufferPool { } } -test('Sample implementation of file API with a buffer pool', t => { - const bs = new FakeByteSourceWithBufferPool(); - const ros = bs.stream; - ros.window = 64; - - var bytesRead = 0; - function pump() { - for (;;) { - if (ros.state === 'readable') { - const op = ros.read(); - if (op.type === 'data') { - const view = op.argument; - - // Verify contents of the buffer. - for (var i = 0; i < view.byteLength; ++i) { - if (view[i] === 1) { - ++bytesRead; +function readAndVerify(ros) { + return new Promise((resolve, reject) => { + var bytesRead = 0; + + function pump() { + for (;;) { + if (ros.state === 'readable') { + const op = ros.read(); + if (op.type === 'data') { + const view = op.argument; + + // Verify contents of the buffer. + for (var i = 0; i < view.byteLength; ++i) { + if (view[i] === 1) { + ++bytesRead; + } } - } - // Release the buffer. - op.complete(); - } else if (op.type === 'close') { - t.equals(bytesRead, 1024); + // Release the buffer. + op.complete(); - t.end() + continue; + } else if (op.type === 'close') { + resolve(bytesRead); + } else { + reject(op.type); + } + } else if (ros.state === 'waiting') { + ros.ready.then(pump); + } else if (ros.state === 'cancelled') { + reject(ros.cancelOperation.argument); } else { - t.fail('Invalid type: ' + op.type); - t.end(); + reject(ros.state); } - } else if (ros.state === 'waiting') { - ros.ready.then(pump); return; - } else if (ros.state === 'cancelled') { - t.fail(ros.cancelOperation.argument); - t.end(); - } else { - t.fail('Unexpected state: ' + ros.state); - t.end(); } } + pump(); + }); +} + +test('Sample implementation of file API with a buffer pool', t => { + const pool = []; + for (var i = 0; i < 10; ++i) { + pool.push(new ArrayBuffer(10)); } - pump(); + + const bs = new FakeByteSourceWithBufferPool(pool); + const ros = bs.stream; + ros.window = 64; + + readAndVerify(ros).then(bytesRead => { + t.equals(bytesRead, 1024); + t.end(); + }); }); From 657f266d83bf8368d260970d13de9507c00faebd Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Mon, 23 Feb 2015 12:37:20 +0900 Subject: [PATCH 12/93] Fix checkWritableState. It should just throw. --- .../lib/experimental/operation-stream.js | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 1506d7eab..2cf9931ee 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -188,15 +188,14 @@ class OperationStream { _checkWritableState() { if (this._writableState === 'closed') { - return Promise.reject(new TypeError('already closed')); + throw new TypeError('already closed'); } if (this._writableState === 'aborted') { - return Promise.reject(new TypeError('already aborted')); + throw new TypeError('already aborted'); } if (this._writableState === 'cancelled') { - return Promise.reject(new TypeError('already cancelled')); + throw new TypeError('already cancelled'); } - return undefined; } _updateWritableState() { @@ -214,10 +213,7 @@ class OperationStream { } write(argument) { - const checkResult = this._checkWritableState(); - if (checkResult !== undefined) { - return checkResult; - } + this._checkWritableState(); var size = 1; if ('size' in this._strategy) { @@ -239,10 +235,7 @@ class OperationStream { } close() { - const checkResult = this._checkWritableState(); - if (checkResult !== undefined) { - return checkResult; - } + this._checkWritableState(); this._strategy = undefined; From 113628f4b6952c99f9a977af5646ad60ecb84817 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Mon, 23 Feb 2015 15:42:08 +0900 Subject: [PATCH 13/93] Add FakeBufferTakingByteSink --- .../lib/experimental/operation-stream.js | 8 +- .../test/experimental/operation-stream.js | 131 +++++++++++++----- 2 files changed, 101 insertions(+), 38 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 2cf9931ee..440a6941b 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -179,7 +179,7 @@ class OperationStream { return undefined; } - if ('space' in this._strategy) { + if (this._strategy.space !== undefined) { return this._strategy.space(this._queueSize); } @@ -200,7 +200,7 @@ class OperationStream { _updateWritableState() { var shouldApplyBackpressure = false; - if ('shouldApplyBackpressure' in this._strategy) { + if (this._strategy.shouldApplyBackpressure !== undefined) { shouldApplyBackpressure = this._strategy.shouldApplyBackpressure(this._queueSize); } if (shouldApplyBackpressure && this._writableState === 'writable') { @@ -216,7 +216,7 @@ class OperationStream { this._checkWritableState(); var size = 1; - if ('size' in this._strategy) { + if (this._strategy.size !== undefined) { size = this._strategy.size(argument); } @@ -312,7 +312,7 @@ class OperationStream { return; } - if ('onWindowUpdate' in this._strategy) { + if (this._strategy.onWindowUpdate !== undefined) { this._strategy.onWindowUpdate(v); } this._updateWritableState(); diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 586198149..96b67517f 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -349,48 +349,75 @@ class FakeByteSourceWithBufferPool { } } -function readAndVerify(ros) { - return new Promise((resolve, reject) => { - var bytesRead = 0; - - function pump() { - for (;;) { - if (ros.state === 'readable') { - const op = ros.read(); - if (op.type === 'data') { - const view = op.argument; - - // Verify contents of the buffer. - for (var i = 0; i < view.byteLength; ++i) { - if (view[i] === 1) { - ++bytesRead; - } - } +class FakeBufferTakingByteSink { + constructor(readableStream) { + this._resultPromise = new Promise((resolve, reject) => { + this._resolveResultPromise = resolve; + this._rejectResultPromise = reject; + }); - // Release the buffer. - op.complete(); + this._bytesRead = 0; - continue; - } else if (op.type === 'close') { - resolve(bytesRead); - } else { - reject(op.type); + if (readableStream === undefined) { + this._streams = createOperationStream(new AdjustableArrayBufferStrategy()); + this._streams.readable.window = 64; + } else { + this._streams = {}; + this._streams.readable = readableStream; + } + + this._loop(); + } + + get result() { + return this._resultPromise; + } + + get stream() { + return this._streams.writable; + } + + _loop() { + const ros = this._streams.readable; + + for (;;) { + if (ros.state === 'readable') { + const op = ros.read(); + if (op.type === 'data') { + const view = op.argument; + + // Verify contents of the buffer. + for (var i = 0; i < view.byteLength; ++i) { + if (view[i] === 1) { + ++this._bytesRead; + } } - } else if (ros.state === 'waiting') { - ros.ready.then(pump); - } else if (ros.state === 'cancelled') { - reject(ros.cancelOperation.argument); + + // Release the buffer. + op.complete(); + + continue; + } else if (op.type === 'close') { + op.complete(); + + this._resolveResultPromise(this._bytesRead); } else { - reject(ros.state); + this._rejectResultPromise(op.type); } - return; + } else if (ros.state === 'waiting') { + ros.ready.then(this._loop.bind(this)).catch(this._rejectResultPromise.bind(this)); + } else if (ros.state === 'cancelled') { + this._rejectResultPromise(ros.cancelOperation.argument); + } else { + this._rejectResultPromise(ros.state); } + return; } - pump(); - }); + this._loop(); + } } -test('Sample implementation of file API with a buffer pool', t => { +test('Sample implementation of file API with a buffer pool (pipe)', t => { const pool = []; for (var i = 0; i < 10; ++i) { pool.push(new ArrayBuffer(10)); @@ -400,8 +427,44 @@ test('Sample implementation of file API with a buffer pool', t => { const ros = bs.stream; ros.window = 64; - readAndVerify(ros).then(bytesRead => { + const sink = new FakeBufferTakingByteSink(); + const wos = sink.stream; + + pipeOperationStreams(ros, wos); + + sink.result.then(bytesRead => { + t.equals(bytesRead, 1024); + + // Check that the buffers are returned to the pool + // TODO + + t.end(); + }).catch(e => { + t.fail(e); + t.end(); + }); +}); + +test('Sample implementation of file API with a buffer pool (direct writing)', t => { + const pool = []; + for (var i = 0; i < 10; ++i) { + pool.push(new ArrayBuffer(10)); + } + + const bs = new FakeByteSourceWithBufferPool(pool); + const ros = bs.stream; + ros.window = 64; + + const sink = new FakeBufferTakingByteSink(ros); + + sink.result.then(bytesRead => { t.equals(bytesRead, 1024); t.end(); + }).catch(e => { + t.fail(e); + t.end(); }); }); + +//test('Sample implementation of zero-copy I/O vector passing', t => { +//}); From 4634d94f08ec6c2b2d773afa6bacefee631b0beb Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Mon, 23 Feb 2015 17:56:04 +0900 Subject: [PATCH 14/93] Add a pipe test from a buffer taking source to a sink with buffer --- .../test/experimental/operation-stream.js | 226 +++++++++++++++++- 1 file changed, 217 insertions(+), 9 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 96b67517f..b3ac7f7cb 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -227,23 +227,22 @@ class FakeFile { this._bytesToWrite = 1024; } - readInto(buffer) { + readInto(view) { return new Promise((resolve, reject) => { setTimeout(() => { try { - const bytesToWriteThisTime = Math.min(this._bytesToWrite, buffer.byteLength); + const bytesToWriteThisTime = Math.min(this._bytesToWrite, view.byteLength); - const view = new Uint8Array(buffer, 0, bytesToWriteThisTime); - for (var i = 0; i < view.byteLength; ++i) { + for (var i = 0; i < bytesToWriteThisTime; ++i) { view[i] = 1; } this._bytesToWrite -= bytesToWriteThisTime; if (this._bytesToWrite === 0) { - resolve({closed: true, view}); + resolve({closed: true, view: new Uint8Array(view.buffer, view.byteOffset, bytesToWriteThisTime)}); } else { - resolve({closed: false, view}); + resolve({closed: false, view: new Uint8Array(view.buffer, view.byteOffset, bytesToWriteThisTime)}); } } catch (e) { reject(e); @@ -317,7 +316,7 @@ class FakeByteSourceWithBufferPool { view[0] = 0; } - this._fileReadPromise = this._file.readInto(buffer).then( + this._fileReadPromise = this._file.readInto(view).then( this._handleFileReadResult.bind(this, buffer)).catch(e => wos.abort(e)); hasProgress = true; @@ -466,5 +465,214 @@ test('Sample implementation of file API with a buffer pool (direct writing)', t }); }); -//test('Sample implementation of zero-copy I/O vector passing', t => { -//}); +class FakeBufferTakingByteSource { + constructor() { + const pair = createOperationStream(new AdjustableArrayBufferStrategy()); + this._readableStream = pair.readable; + this._writableStream = pair.writable; + + this._currentRequest = undefined; + + this._file = new FakeFile(); + + this._fileReadStatus = undefined; + + this._readableStream.window = 1; + + this._loop(); + } + + get writableStream() { + return this._writableStream; + } + + _loop() { + const rs = this._readableStream; + + for (;;) { + if (rs.state === 'aborted') { + if (this._currentRequest !== undefined) { + this._currentRequest.error(rs.abortOperation.argument); + } + + return; + } + + if (this._currentRequest !== undefined && this._fileReadStatus.state !== 'waiting') { + if (this._fileReadStatus.state === 'errored') { + const error = this._fileReadStatus.result; + this._fileReadStatus = undefined + + this._currentRequest.error(error); + this._currentRequest = undefined; + + rs.cancel(error); + + return; + } + + this._currentRequest.complete(this._fileReadStatus.result); + this._currentRequest = undefined; + + this._fileReadStatus = undefined + } + + if (this._currentRequest === undefined && rs.state === 'readable') { + this._currentRequest = rs.read(); + + this._fileReadStatus = {}; + this._fileReadStatus.state = 'waiting'; + this._fileReadStatus.ready = this._file.readInto(this._currentRequest.argument) + .then(result => { + this._fileReadStatus.state = 'completed'; + this._fileReadStatus.result = {closed: result.closed, bytesWritten: result.view.byteLength}; + }).catch(e => { + this._fileReadStatus.state = 'errored'; + this._fileReadStatus.result = e; + }); + + continue; + } + + const promisesToRace = []; + + if (rs.state === 'readable') { + promisesToRace.push(rs.aborted); + } else if (rs.state === 'waiting') { + promisesToRace.push(rs.ready); + } + + if (this._currentRequest !== undefined && this._fileReadStatus.state === 'waiting') { + promisesToRace.push(this._fileReadStatus.ready); + } + + Promise.race(promisesToRace) + .then(this._loop.bind(this)) + .catch(e => rs.cancel.bind(rs, e)); + return; + } + } +} + +class FakeByteSinkWithBuffer { + constructor() { + const pair = createOperationStream(new AdjustableArrayBufferStrategy()); + this._readableStream = pair.readable; + this._writableStream = pair.writable; + + this._currentReadStatus = undefined; + + this._buffer = new ArrayBuffer(16); + + this._resultPromise = new Promise((resolve, reject) => { + this._resolveResultPromise = resolve; + this._rejectResultPromise = reject; + }); + this._bytesRead = 0; + + this._loop(); + } + + get readableStream() { + return this._readableStream; + } + + get result() { + return this._resultPromise; + } + + _loop() { + const ws = this._writableStream; + + for (;;) { + if (this._currentReadStatus !== undefined && this._currentReadStatus.state === 'errored') { + const error = this._currentReadStatus.result; + this._currentReadStatus = undefined; + + ws.abort(error); + this._rejectResultPromise(error); + + return; + } + + if (ws.state === 'cancelled') { + this._rejectResultPromise(ws.cancelOperation.argument); + return; + } + + let hasProgress = false; + + if (this._currentReadStatus !== undefined && this._currentReadStatus.state === 'completed') { + const bytesWritten = this._currentReadStatus.result.bytesWritten; + const closed = this._currentReadStatus.result.closed; + this._currentReadStatus = undefined; + + // Verify contents of the buffer. + const view = new Uint8Array(this._buffer, 0, bytesWritten); + for (var i = 0; i < bytesWritten; ++i) { + if (view[i] === 1) { + ++this._bytesRead; + } + } + + if (closed) { + this._resolveResultPromise(this._bytesRead); + return; + } + + hasProgress = true; + } + + if (this._currentReadStatus === undefined && ws.state === 'writable') { + this._currentReadStatus = ws.write(new Uint8Array(this._buffer)); + + hasProgress = true; + } + + if (hasProgress) { + continue; + } + + const promisesToRace = []; + + if (ws.state === 'writable') { + promisesToRace.push(ws.cancelled); + } else if (ws.state === 'waiting') { + promisesToRace.push(ws.ready); + } + + if (this._currentReadStatus !== undefined && this._currentReadStatus.state === 'waiting') { + promisesToRace.push(this._currentReadStatus.ready); + } + + Promise.race(promisesToRace) + .then(this._loop.bind(this)) + .catch(e => this._rejectResultPromise.bind(this, e)); + return; + } + } +} + +test('Piping from a buffer taking source to a sink with buffer', t => { + const pool = []; + for (var i = 0; i < 10; ++i) { + pool.push(new ArrayBuffer(10)); + } + + const source = new FakeBufferTakingByteSource(); + const ws = source.writableStream; + + const sink = new FakeByteSinkWithBuffer(); + const rs = sink.readableStream; + rs.window = 16; + + pipeOperationStreams(rs, ws); + + sink.result.then(bytesRead => { + t.equals(bytesRead, 1024); + t.end(); + }).catch(e => { + t.fail(e); + t.end(); + }); +}); From 5d94d0391c347ef1e7d9dccf6e8329e404c4ccd1 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Mon, 23 Feb 2015 18:14:22 +0900 Subject: [PATCH 15/93] Improve readability of "source with a buffer pool" test cases --- .../test/experimental/operation-stream.js | 106 ++++++++++-------- 1 file changed, 57 insertions(+), 49 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index b3ac7f7cb..793764b2c 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -254,7 +254,9 @@ class FakeFile { class FakeByteSourceWithBufferPool { constructor(buffers) { - this._streams = createOperationStream(new AdjustableArrayBufferStrategy()); + const pair = createOperationStream(new AdjustableArrayBufferStrategy()); + this._readableStream = pair.readable; + this._writableStream = pair.writable; this._file = new FakeFile(); this._fileReadPromise = undefined; @@ -266,24 +268,24 @@ class FakeByteSourceWithBufferPool { this._loop(); } - get stream() { - return this._streams.readable; + get readableStream() { + return this._readableStream; } _handleFileReadResult(buffer, result) { this._fileReadPromise = undefined; - const status = this._streams.writable.write(result.view); + const status = this._writableStream.write(result.view); this._buffersPassedToUser.push({buffer, status}); if (result.closed) { - this._streams.writable.close(); + this._writableStream.close(); } } _loop() { - const wos = this._streams.writable; + const ws = this._writableStream; for (;;) { - if (wos.state === 'cancelled') { + if (ws.state === 'cancelled') { return; } @@ -300,14 +302,14 @@ class FakeByteSourceWithBufferPool { hasProgress = true; // Keep going. } else if (status.state === 'errored') { - wos.abort(status.result); + ws.abort(status.result); return; } } if (this._fileReadPromise === undefined && this._buffersAvailable.length > 0 && - wos.state === 'writable') { + ws.state === 'writable') { const buffer = this._buffersAvailable.shift(); // Clear the acquired buffer for testing. @@ -316,8 +318,9 @@ class FakeByteSourceWithBufferPool { view[0] = 0; } - this._fileReadPromise = this._file.readInto(view).then( - this._handleFileReadResult.bind(this, buffer)).catch(e => wos.abort(e)); + this._fileReadPromise = this._file.readInto(view) + .then(this._handleFileReadResult.bind(this, buffer)) + .catch(e => ws.abort(e)); hasProgress = true; } @@ -328,10 +331,10 @@ class FakeByteSourceWithBufferPool { const promisesToRace = []; - if (wos.state === 'writable') { - promisesToRace.push(wos.cancelled); - } else if (wos.state === 'waiting') { - promisesToRace.push(wos.ready); + if (ws.state === 'writable') { + promisesToRace.push(ws.cancelled); + } else if (ws.state === 'waiting') { + promisesToRace.push(ws.ready); } if (this._fileReadPromise !== undefined) { @@ -342,7 +345,9 @@ class FakeByteSourceWithBufferPool { promisesToRace.push(this._buffersPassedToUser[0].status.ready); } - Promise.race(promisesToRace).then(this._loop.bind(this)); + Promise.race(promisesToRace) + .then(this._loop.bind(this)) + .catch(e => ws.abort.bind(ws, e)); return; } } @@ -350,38 +355,40 @@ class FakeByteSourceWithBufferPool { class FakeBufferTakingByteSink { constructor(readableStream) { + if (readableStream === undefined) { + const pair = createOperationStream(new AdjustableArrayBufferStrategy()); + this._readableStream = pair.readable; + this._writableStream = pair.writable; + } else { + this._readableStream = readableStream; + this._writableStream = undefined; + } + this._resultPromise = new Promise((resolve, reject) => { this._resolveResultPromise = resolve; this._rejectResultPromise = reject; }); - this._bytesRead = 0; - if (readableStream === undefined) { - this._streams = createOperationStream(new AdjustableArrayBufferStrategy()); - this._streams.readable.window = 64; - } else { - this._streams = {}; - this._streams.readable = readableStream; - } + this._readableStream.window = 64; this._loop(); } - get result() { - return this._resultPromise; + get writableStream() { + return this._writableStream; } - get stream() { - return this._streams.writable; + get result() { + return this._resultPromise; } _loop() { - const ros = this._streams.readable; + const rs = this._readableStream; for (;;) { - if (ros.state === 'readable') { - const op = ros.read(); + if (rs.state === 'readable') { + const op = rs.read(); if (op.type === 'data') { const view = op.argument; @@ -397,39 +404,41 @@ class FakeBufferTakingByteSink { continue; } else if (op.type === 'close') { + // Acknowledge the closure. op.complete(); this._resolveResultPromise(this._bytesRead); } else { this._rejectResultPromise(op.type); } - } else if (ros.state === 'waiting') { - ros.ready.then(this._loop.bind(this)).catch(this._rejectResultPromise.bind(this)); - } else if (ros.state === 'cancelled') { - this._rejectResultPromise(ros.cancelOperation.argument); + } else if (rs.state === 'waiting') { + rs.ready + .then(this._loop.bind(this)) + .catch(this._rejectResultPromise.bind(this)); + } else if (rs.state === 'aborted') { + this._rejectResultPromise(rs.abortOperation.argument); } else { - this._rejectResultPromise(ros.state); + this._rejectResultPromise(rs.state); } return; } - this._loop(); } } -test('Sample implementation of file API with a buffer pool (pipe)', t => { +test('Piping from a source with a buffer pool to a buffer taking sink', t => { const pool = []; for (var i = 0; i < 10; ++i) { pool.push(new ArrayBuffer(10)); } - const bs = new FakeByteSourceWithBufferPool(pool); - const ros = bs.stream; - ros.window = 64; + const source = new FakeByteSourceWithBufferPool(pool); + const rs = source.readableStream; + rs.window = 64; const sink = new FakeBufferTakingByteSink(); - const wos = sink.stream; + const ws = sink.writableStream; - pipeOperationStreams(ros, wos); + pipeOperationStreams(rs, ws); sink.result.then(bytesRead => { t.equals(bytesRead, 1024); @@ -444,18 +453,17 @@ test('Sample implementation of file API with a buffer pool (pipe)', t => { }); }); -test('Sample implementation of file API with a buffer pool (direct writing)', t => { +test('Consuming bytes from a source with a buffer pool via the ReadableStream interface', t => { const pool = []; for (var i = 0; i < 10; ++i) { pool.push(new ArrayBuffer(10)); } - const bs = new FakeByteSourceWithBufferPool(pool); - const ros = bs.stream; - ros.window = 64; - - const sink = new FakeBufferTakingByteSink(ros); + const source = new FakeByteSourceWithBufferPool(pool); + const rs = source.readableStream; + rs.window = 64; + const sink = new FakeBufferTakingByteSink(rs); sink.result.then(bytesRead => { t.equals(bytesRead, 1024); t.end(); From 8241b5c65dfffe80a784aaa4fc58c57cfc198a59 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Tue, 24 Feb 2015 12:00:01 +0900 Subject: [PATCH 16/93] Simplify piping code in tests --- .../test/experimental/operation-stream.js | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 793764b2c..958a7c268 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -432,13 +432,11 @@ test('Piping from a source with a buffer pool to a buffer taking sink', t => { } const source = new FakeByteSourceWithBufferPool(pool); - const rs = source.readableStream; - rs.window = 64; + source.readableStream.window = 64; const sink = new FakeBufferTakingByteSink(); - const ws = sink.writableStream; - pipeOperationStreams(rs, ws); + pipeOperationStreams(source.readableStream, sink.writableStream); sink.result.then(bytesRead => { t.equals(bytesRead, 1024); @@ -460,10 +458,9 @@ test('Consuming bytes from a source with a buffer pool via the ReadableStream in } const source = new FakeByteSourceWithBufferPool(pool); - const rs = source.readableStream; - rs.window = 64; + source.readableStream.window = 64; - const sink = new FakeBufferTakingByteSink(rs); + const sink = new FakeBufferTakingByteSink(source.readableStream); sink.result.then(bytesRead => { t.equals(bytesRead, 1024); t.end(); @@ -668,13 +665,11 @@ test('Piping from a buffer taking source to a sink with buffer', t => { } const source = new FakeBufferTakingByteSource(); - const ws = source.writableStream; const sink = new FakeByteSinkWithBuffer(); - const rs = sink.readableStream; - rs.window = 16; + sink.readableStream.window = 16; - pipeOperationStreams(rs, ws); + pipeOperationStreams(sink.readableStream, source.writableStream); sink.result.then(bytesRead => { t.equals(bytesRead, 1024); From f33a4a675a441cb2ccc6c0eda8bd53da74961941 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Tue, 24 Feb 2015 12:07:18 +0900 Subject: [PATCH 17/93] Factor out view filling code --- .../test/experimental/operation-stream.js | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 958a7c268..1b20ec239 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -222,6 +222,15 @@ test('Pipe', t => { }); }); +function fillArrayBufferView(view, c, size) { + if (size === undefined) { + size = view.byteLength; + } + for (var i = 0; i < size; ++i) { + view[i] = c; + } +} + class FakeFile { constructor() { this._bytesToWrite = 1024; @@ -233,9 +242,7 @@ class FakeFile { try { const bytesToWriteThisTime = Math.min(this._bytesToWrite, view.byteLength); - for (var i = 0; i < bytesToWriteThisTime; ++i) { - view[i] = 1; - } + fillArrayBufferView(view, 1, bytesToWriteThisTime); this._bytesToWrite -= bytesToWriteThisTime; @@ -314,9 +321,7 @@ class FakeByteSourceWithBufferPool { // Clear the acquired buffer for testing. const view = new Uint8Array(buffer); - for (var i = 0; i < view.byteLength; ++i) { - view[0] = 0; - } + fillArrayBufferView(view, 0); this._fileReadPromise = this._file.readInto(view) .then(this._handleFileReadResult.bind(this, buffer)) From 7dd8a196f717df91634813bab82c9f42f713c73f Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Tue, 24 Feb 2015 12:42:22 +0900 Subject: [PATCH 18/93] Factor out handlers --- .../test/experimental/operation-stream.js | 112 +++++++++++------- 1 file changed, 68 insertions(+), 44 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 1b20ec239..f57c99b5b 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -496,6 +496,46 @@ class FakeBufferTakingByteSource { return this._writableStream; } + _handleFileReadCompletion() { + this._currentRequest.complete(this._fileReadStatus.result); + this._currentRequest = undefined; + + this._fileReadStatus = undefined + } + + _handleFileReadError() { + const error = this._fileReadStatus.result; + + this._currentRequest.error(error); + this._currentRequest = undefined; + + this._readableStream.cancel(error); + + this._fileReadStatus = undefined; + } + + _transformReadFromFileFulfillment(result) { + this._fileReadStatus.state = 'completed'; + this._fileReadStatus.result = {closed: result.closed, bytesWritten: result.view.byteLength}; + } + + _transformReadFromFileRejection(e) { + this._fileReadStatus.state = 'errored'; + this._fileReadStatus.result = e; + } + + _readFromFile() { + this._currentRequest = this._readableStream.read(); + + const status = {}; + status.state = 'waiting'; + status.ready = this._file.readInto(this._currentRequest.argument) + .then(this._transformReadFromFileFulfillment.bind(this)) + .catch(this._transformReadFromFileRejection.bind(this)); + + this._fileReadStatus = status; + } + _loop() { const rs = this._readableStream; @@ -508,39 +548,17 @@ class FakeBufferTakingByteSource { return; } - if (this._currentRequest !== undefined && this._fileReadStatus.state !== 'waiting') { + if (this._currentRequest !== undefined) { if (this._fileReadStatus.state === 'errored') { - const error = this._fileReadStatus.result; - this._fileReadStatus = undefined - - this._currentRequest.error(error); - this._currentRequest = undefined; - - rs.cancel(error); - + this._handleFileReadError(); return; + } else if (this._fileReadStatus.state === 'completed') { + this._handleFileReadCompletion(); } - - this._currentRequest.complete(this._fileReadStatus.result); - this._currentRequest = undefined; - - this._fileReadStatus = undefined } - if (this._currentRequest === undefined && rs.state === 'readable') { - this._currentRequest = rs.read(); - - this._fileReadStatus = {}; - this._fileReadStatus.state = 'waiting'; - this._fileReadStatus.ready = this._file.readInto(this._currentRequest.argument) - .then(result => { - this._fileReadStatus.state = 'completed'; - this._fileReadStatus.result = {closed: result.closed, bytesWritten: result.view.byteLength}; - }).catch(e => { - this._fileReadStatus.state = 'errored'; - this._fileReadStatus.result = e; - }); - + if (rs.state === 'readable' && this._currentRequest === undefined) { + this._readFromFile(); continue; } @@ -552,13 +570,13 @@ class FakeBufferTakingByteSource { promisesToRace.push(rs.ready); } - if (this._currentRequest !== undefined && this._fileReadStatus.state === 'waiting') { + if (this._fileReadStatus !== undefined && this._fileReadStatus.state === 'waiting') { promisesToRace.push(this._fileReadStatus.ready); } Promise.race(promisesToRace) .then(this._loop.bind(this)) - .catch(e => rs.cancel.bind(rs, e)); + .catch(rs.cancel.bind(rs)); return; } } @@ -591,6 +609,23 @@ class FakeByteSinkWithBuffer { return this._resultPromise; } + _handleReadCompletion() { + const bytesWritten = this._currentReadStatus.result.bytesWritten; + const closed = this._currentReadStatus.result.closed; + + // Verify contents of the buffer. + const view = new Uint8Array(this._buffer, 0, bytesWritten); + for (var i = 0; i < bytesWritten; ++i) { + if (view[i] === 1) { + ++this._bytesRead; + } + } + + this._currentReadStatus = undefined; + + return closed; + } + _loop() { const ws = this._writableStream; @@ -613,18 +648,7 @@ class FakeByteSinkWithBuffer { let hasProgress = false; if (this._currentReadStatus !== undefined && this._currentReadStatus.state === 'completed') { - const bytesWritten = this._currentReadStatus.result.bytesWritten; - const closed = this._currentReadStatus.result.closed; - this._currentReadStatus = undefined; - - // Verify contents of the buffer. - const view = new Uint8Array(this._buffer, 0, bytesWritten); - for (var i = 0; i < bytesWritten; ++i) { - if (view[i] === 1) { - ++this._bytesRead; - } - } - + const closed = this._handleReadCompletion(); if (closed) { this._resolveResultPromise(this._bytesRead); return; @@ -633,7 +657,7 @@ class FakeByteSinkWithBuffer { hasProgress = true; } - if (this._currentReadStatus === undefined && ws.state === 'writable') { + if (ws.state === 'writable' && this._currentReadStatus === undefined) { this._currentReadStatus = ws.write(new Uint8Array(this._buffer)); hasProgress = true; @@ -657,7 +681,7 @@ class FakeByteSinkWithBuffer { Promise.race(promisesToRace) .then(this._loop.bind(this)) - .catch(e => this._rejectResultPromise.bind(this, e)); + .catch(this._rejectResultPromise.bind(this)); return; } } From 383196bebe0ef3248ea1a2431102380dab3699fc Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Tue, 24 Feb 2015 12:53:30 +0900 Subject: [PATCH 19/93] Add catch() and asserts --- .../lib/experimental/operation-stream.js | 7 ++++++- .../test/experimental/operation-stream.js | 18 +++++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 440a6941b..9fbad77be 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -33,6 +33,7 @@ export function pipeOperationStreams(readable, writable) { if (op.type === 'data') { jointOps(op, writable.write(op.argument)); } else { + // Assert: op.type === 'close'. jointOps(op, writable.close(op.argument)); return; } @@ -45,16 +46,20 @@ export function pipeOperationStreams(readable, writable) { if (readable.state === 'readable') { promisesToRace.push(readable.aborted); } else { + // Assert: readable.state === 'readable'. promisesToRace.push(readable.ready); } if (writable.state === 'writable') { promisesToRace.push(writable.cancelled); } else { + // Assert: writable.state === 'writable'. promisesToRace.push(writable.ready); } - Promise.race(promisesToRace).then(loop); + Promise.race(promisesToRace) + .then(loop) + .catch(reject); return; } } diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index f57c99b5b..ae7f6decf 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -185,7 +185,11 @@ test('Pipe', t => { wos0.write('world'); wos0.close(); - pipeOperationStreams(ros0, wos1); + pipeOperationStreams(ros0, wos1) + .catch(e => { + t.fail(e); + t.end(); + }); t.equals(ros1.state, 'waiting'); @@ -441,7 +445,11 @@ test('Piping from a source with a buffer pool to a buffer taking sink', t => { const sink = new FakeBufferTakingByteSink(); - pipeOperationStreams(source.readableStream, sink.writableStream); + pipeOperationStreams(source.readableStream, sink.writableStream) + .catch(e => { + t.fail(e); + t.end(); + }); sink.result.then(bytesRead => { t.equals(bytesRead, 1024); @@ -698,7 +706,11 @@ test('Piping from a buffer taking source to a sink with buffer', t => { const sink = new FakeByteSinkWithBuffer(); sink.readableStream.window = 16; - pipeOperationStreams(sink.readableStream, source.writableStream); + pipeOperationStreams(sink.readableStream, source.writableStream) + .catch(e => { + t.fail(e); + t.end(); + }); sink.result.then(bytesRead => { t.equals(bytesRead, 1024); From 1b48552363cadd11a41f36a3c903ca11b81638b8 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Tue, 24 Feb 2015 13:07:20 +0900 Subject: [PATCH 20/93] Make pipeOperationStreams fulfill the returned promise. --- .../lib/experimental/operation-stream.js | 1 + .../test/experimental/operation-stream.js | 36 +++++++++++-------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 9fbad77be..b11c0031f 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -35,6 +35,7 @@ export function pipeOperationStreams(readable, writable) { } else { // Assert: op.type === 'close'. jointOps(op, writable.close(op.argument)); + resolve(); return; } diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index ae7f6decf..f1c018729 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -445,11 +445,11 @@ test('Piping from a source with a buffer pool to a buffer taking sink', t => { const sink = new FakeBufferTakingByteSink(); - pipeOperationStreams(source.readableStream, sink.writableStream) - .catch(e => { - t.fail(e); - t.end(); - }); + const pipePromise = pipeOperationStreams(source.readableStream, sink.writableStream) + pipePromise.catch(e => { + t.fail(e); + t.end(); + }); sink.result.then(bytesRead => { t.equals(bytesRead, 1024); @@ -457,7 +457,7 @@ test('Piping from a source with a buffer pool to a buffer taking sink', t => { // Check that the buffers are returned to the pool // TODO - t.end(); + pipePromise.then(() => t.end()); }).catch(e => { t.fail(e); t.end(); @@ -533,11 +533,18 @@ class FakeBufferTakingByteSource { } _readFromFile() { - this._currentRequest = this._readableStream.read(); + const op = this._readableStream.read(); + + if (op.type === 'close') { + return; + } + // Assert: op.type === 'data'. + + this._currentRequest = op; const status = {}; status.state = 'waiting'; - status.ready = this._file.readInto(this._currentRequest.argument) + status.ready = this._file.readInto(op.argument) .then(this._transformReadFromFileFulfillment.bind(this)) .catch(this._transformReadFromFileRejection.bind(this)); @@ -658,6 +665,7 @@ class FakeByteSinkWithBuffer { if (this._currentReadStatus !== undefined && this._currentReadStatus.state === 'completed') { const closed = this._handleReadCompletion(); if (closed) { + ws.close(); this._resolveResultPromise(this._bytesRead); return; } @@ -706,15 +714,15 @@ test('Piping from a buffer taking source to a sink with buffer', t => { const sink = new FakeByteSinkWithBuffer(); sink.readableStream.window = 16; - pipeOperationStreams(sink.readableStream, source.writableStream) - .catch(e => { - t.fail(e); - t.end(); - }); + const pipePromise = pipeOperationStreams(sink.readableStream, source.writableStream); + pipePromise.catch(e => { + t.fail(e); + t.end(); + }); sink.result.then(bytesRead => { t.equals(bytesRead, 1024); - t.end(); + pipePromise.then(() => t.end()); }).catch(e => { t.fail(e); t.end(); From a797e4bea3683e08a2eae324c4e7ad51e70389f3 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Tue, 24 Feb 2015 13:09:36 +0900 Subject: [PATCH 21/93] Factor out select() in pipeOperationStreams() --- .../lib/experimental/operation-stream.js | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index b11c0031f..f02fe847d 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -21,6 +21,28 @@ function jointOps(op, status) { export function pipeOperationStreams(readable, writable) { return new Promise((resolve, reject) => { + function select() { + const promisesToRace = []; + + if (readable.state === 'readable') { + promisesToRace.push(readable.aborted); + } else { + // Assert: readable.state === 'readable'. + promisesToRace.push(readable.ready); + } + + if (writable.state === 'writable') { + promisesToRace.push(writable.cancelled); + } else { + // Assert: writable.state === 'writable'. + promisesToRace.push(writable.ready); + } + + Promise.race(promisesToRace) + .then(loop) + .catch(reject); + } + function loop() { for (;;) { if (readable.state === 'aborted' || writable.state === 'cancelled') { @@ -42,25 +64,7 @@ export function pipeOperationStreams(readable, writable) { continue; } - const promisesToRace = []; - - if (readable.state === 'readable') { - promisesToRace.push(readable.aborted); - } else { - // Assert: readable.state === 'readable'. - promisesToRace.push(readable.ready); - } - - if (writable.state === 'writable') { - promisesToRace.push(writable.cancelled); - } else { - // Assert: writable.state === 'writable'. - promisesToRace.push(writable.ready); - } - - Promise.race(promisesToRace) - .then(loop) - .catch(reject); + select(); return; } } From 2f4ef65438e20abd9a4702331b47cbb4845aeff8 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Tue, 24 Feb 2015 13:12:29 +0900 Subject: [PATCH 22/93] Factor out selecting function in tests --- .../test/experimental/operation-stream.js | 72 +++++++++++-------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index f1c018729..0d692fdd0 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -551,6 +551,26 @@ class FakeBufferTakingByteSource { this._fileReadStatus = status; } + _select() { + const promisesToRace = []; + + const rs = this._readableStream; + + if (rs.state === 'readable') { + promisesToRace.push(rs.aborted); + } else if (rs.state === 'waiting') { + promisesToRace.push(rs.ready); + } + + if (this._fileReadStatus !== undefined && this._fileReadStatus.state === 'waiting') { + promisesToRace.push(this._fileReadStatus.ready); + } + + Promise.race(promisesToRace) + .then(this._loop.bind(this)) + .catch(rs.cancel.bind(rs)); + } + _loop() { const rs = this._readableStream; @@ -577,21 +597,7 @@ class FakeBufferTakingByteSource { continue; } - const promisesToRace = []; - - if (rs.state === 'readable') { - promisesToRace.push(rs.aborted); - } else if (rs.state === 'waiting') { - promisesToRace.push(rs.ready); - } - - if (this._fileReadStatus !== undefined && this._fileReadStatus.state === 'waiting') { - promisesToRace.push(this._fileReadStatus.ready); - } - - Promise.race(promisesToRace) - .then(this._loop.bind(this)) - .catch(rs.cancel.bind(rs)); + this._select(); return; } } @@ -641,6 +647,26 @@ class FakeByteSinkWithBuffer { return closed; } + _select() { + const promisesToRace = []; + + const ws = this._writableStream; + + if (ws.state === 'writable') { + promisesToRace.push(ws.cancelled); + } else if (ws.state === 'waiting') { + promisesToRace.push(ws.ready); + } + + if (this._currentReadStatus !== undefined && this._currentReadStatus.state === 'waiting') { + promisesToRace.push(this._currentReadStatus.ready); + } + + Promise.race(promisesToRace) + .then(this._loop.bind(this)) + .catch(this._rejectResultPromise.bind(this)); + } + _loop() { const ws = this._writableStream; @@ -683,21 +709,7 @@ class FakeByteSinkWithBuffer { continue; } - const promisesToRace = []; - - if (ws.state === 'writable') { - promisesToRace.push(ws.cancelled); - } else if (ws.state === 'waiting') { - promisesToRace.push(ws.ready); - } - - if (this._currentReadStatus !== undefined && this._currentReadStatus.state === 'waiting') { - promisesToRace.push(this._currentReadStatus.ready); - } - - Promise.race(promisesToRace) - .then(this._loop.bind(this)) - .catch(this._rejectResultPromise.bind(this)); + this._select(); return; } } From d405f987ddcdec75c600eee1d6ce85bbd69bc279 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Tue, 24 Feb 2015 13:14:10 +0900 Subject: [PATCH 23/93] One more select factoring out --- .../test/experimental/operation-stream.js | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 0d692fdd0..9648d92ac 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -292,6 +292,30 @@ class FakeByteSourceWithBufferPool { } } + _select() { + const promisesToRace = []; + + const ws = this._writableStream; + + if (ws.state === 'writable') { + promisesToRace.push(ws.cancelled); + } else if (ws.state === 'waiting') { + promisesToRace.push(ws.ready); + } + + if (this._fileReadPromise !== undefined) { + promisesToRace.push(this._fileReadPromise); + } + + if (this._buffersPassedToUser.length > 0) { + promisesToRace.push(this._buffersPassedToUser[0].status.ready); + } + + Promise.race(promisesToRace) + .then(this._loop.bind(this)) + .catch(ws.abort.bind(ws)); + } + _loop() { const ws = this._writableStream; @@ -329,7 +353,7 @@ class FakeByteSourceWithBufferPool { this._fileReadPromise = this._file.readInto(view) .then(this._handleFileReadResult.bind(this, buffer)) - .catch(e => ws.abort(e)); + .catch(ws.abort); hasProgress = true; } @@ -338,25 +362,7 @@ class FakeByteSourceWithBufferPool { continue; } - const promisesToRace = []; - - if (ws.state === 'writable') { - promisesToRace.push(ws.cancelled); - } else if (ws.state === 'waiting') { - promisesToRace.push(ws.ready); - } - - if (this._fileReadPromise !== undefined) { - promisesToRace.push(this._fileReadPromise); - } - - if (this._buffersPassedToUser.length > 0) { - promisesToRace.push(this._buffersPassedToUser[0].status.ready); - } - - Promise.race(promisesToRace) - .then(this._loop.bind(this)) - .catch(e => ws.abort.bind(ws, e)); + this._select(); return; } } From 22dbba5e62d9cddb61e1e6f955136684950855e5 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Tue, 24 Feb 2015 13:21:58 +0900 Subject: [PATCH 24/93] Add pool return check --- .../test/experimental/operation-stream.js | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 9648d92ac..2c8bbfed8 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -460,10 +460,13 @@ test('Piping from a source with a buffer pool to a buffer taking sink', t => { sink.result.then(bytesRead => { t.equals(bytesRead, 1024); - // Check that the buffers are returned to the pool - // TODO - - pipePromise.then(() => t.end()); + pipePromise.then(() => { + Promise.resolve().then(() => { + // Check that the buffers have been returned to the pool. + t.equals(pool.length, 10); + t.end(); + }); + }); }).catch(e => { t.fail(e); t.end(); @@ -482,7 +485,14 @@ test('Consuming bytes from a source with a buffer pool via the ReadableStream in const sink = new FakeBufferTakingByteSink(source.readableStream); sink.result.then(bytesRead => { t.equals(bytesRead, 1024); - t.end(); + + Promise.resolve().then(() => { + Promise.resolve().then(() => { + // Check that the buffers have been returned to the pool. + t.equals(pool.length, 10); + t.end(); + }); + }); }).catch(e => { t.fail(e); t.end(); From 1b99b8284ae2b7fc70c9e87d707d4e41b7a6e0d3 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Tue, 24 Feb 2015 14:53:37 +0900 Subject: [PATCH 25/93] Add waitSpaceChange and make pipeOperationStream() forward window info. --- .../lib/experimental/operation-stream.js | 73 ++++++++++++++++--- .../test/experimental/operation-stream.js | 21 ++++-- 2 files changed, 75 insertions(+), 19 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index f02fe847d..e9c524557 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -20,6 +20,8 @@ function jointOps(op, status) { } export function pipeOperationStreams(readable, writable) { + const oldWindow = readable.window; + return new Promise((resolve, reject) => { function select() { const promisesToRace = []; @@ -36,32 +38,45 @@ export function pipeOperationStreams(readable, writable) { } else { // Assert: writable.state === 'writable'. promisesToRace.push(writable.ready); + promisesToRace.push(writable.waitSpaceChange()); } Promise.race(promisesToRace) .then(loop) - .catch(reject); + .catch(e => { + readable.window = oldWindow; + reject(); + }); } function loop() { for (;;) { if (readable.state === 'aborted' || writable.state === 'cancelled') { + readable.window = oldWindow; reject(); + return; } - if (readable.state === 'readable' && writable.state === 'writable') { - const op = readable.read(); - if (op.type === 'data') { - jointOps(op, writable.write(op.argument)); + if (writable.state === 'writable') { + if (readable.state === 'readable') { + const op = readable.read(); + if (op.type === 'data') { + jointOps(op, writable.write(op.argument)); + } else { + // Assert: op.type === 'close'. + jointOps(op, writable.close(op.argument)); + + readable.window = oldWindow; + resolve(); + + return; + } + + continue; } else { - // Assert: op.type === 'close'. - jointOps(op, writable.close(op.argument)); - resolve(); - return; + readable.window = writable.space; } - - continue; } select(); @@ -133,6 +148,10 @@ class OperationStream { this._queueSize = 0; this._strategy = strategy; + + this._lastSpace = undefined; + this._spaceChangePromise = undefined; + this._window = 0; this._writableState = 'waiting'; @@ -195,6 +214,18 @@ class OperationStream { return undefined; } + waitSpaceChange() { + if (this._spaceChangePromise !== undefined) { + return this._spaceChangePromise; + } + + this._spaceChangePromise = new Promise((resolve, reject) => { + this._resolveSpaceChangePromise = resolve; + }); + this._lastSpace = this.space; + + return this._spaceChangePromise; + } _checkWritableState() { if (this._writableState === 'closed') { @@ -209,7 +240,7 @@ class OperationStream { } _updateWritableState() { - var shouldApplyBackpressure = false; + let shouldApplyBackpressure = false; if (this._strategy.shouldApplyBackpressure !== undefined) { shouldApplyBackpressure = this._strategy.shouldApplyBackpressure(this._queueSize); } @@ -220,6 +251,14 @@ class OperationStream { this._writableState = 'writable'; this._resolveWritableReadyPromise(); } + + if (this._spaceChangePromise !== undefined && this._lastSpace !== this.space) { + this._resolveSpaceChangePromise(); + + this._lastSpace = undefined; + this._spaceChangePromise = undefined; + this._resolveSpaceChangePromise = undefined; + } } write(argument) { @@ -416,9 +455,19 @@ class WritableOperationStream { return this._stream.cancelled; } + get window() { + return this._stream.window; + } + set window(v) { + this._stream.window = v; + } + get space() { return this._stream.space; } + waitSpaceChange() { + return this._stream.waitSpaceChange(); + } write(value) { return this._stream.write(value); diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 2c8bbfed8..f22980cd0 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -181,9 +181,15 @@ test('Pipe', t => { const wos1 = pair1.writable; const ros1 = pair1.readable; - const helloStatus = wos0.write('hello'); - wos0.write('world'); - wos0.close(); + var helloStatus; + t.equals(wos0.state, 'waiting'); + wos0.ready.then(() => { + t.equals(wos0.state, 'writable'); + + helloStatus = wos0.write('hello'); + wos0.write('world'); + wos0.close(); + }); pipeOperationStreams(ros0, wos1) .catch(e => { @@ -385,8 +391,6 @@ class FakeBufferTakingByteSink { }); this._bytesRead = 0; - this._readableStream.window = 64; - this._loop(); } @@ -447,10 +451,11 @@ test('Piping from a source with a buffer pool to a buffer taking sink', t => { } const source = new FakeByteSourceWithBufferPool(pool); - source.readableStream.window = 64; const sink = new FakeBufferTakingByteSink(); + sink.writableStream.window = 64; + // pipeOperationStreams automatically adjusts source.window. const pipePromise = pipeOperationStreams(source.readableStream, sink.writableStream) pipePromise.catch(e => { t.fail(e); @@ -738,9 +743,11 @@ test('Piping from a buffer taking source to a sink with buffer', t => { } const source = new FakeBufferTakingByteSource(); + source.writableStream.window = 16; const sink = new FakeByteSinkWithBuffer(); - sink.readableStream.window = 16; + // This also works. + // sink.readableStream.window = 16; const pipePromise = pipeOperationStreams(sink.readableStream, source.writableStream); pipePromise.catch(e => { From 8de0798ded384ae9aa640b9cd7836e9535dd760b Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Tue, 24 Feb 2015 14:55:38 +0900 Subject: [PATCH 26/93] Move jointOps into pipeOperationStreams --- .../lib/experimental/operation-stream.js | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index e9c524557..1332cb19f 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -6,23 +6,23 @@ export function createOperationStream(strategy) { }; } -function jointOps(op, status) { - function forward() { - if (status.state === 'waiting') { - status.ready.then(forward); - } else if (status.state === 'errored') { - op.error(status.result); - } else { - op.complete(status.result); - } - } - forward(); -} - export function pipeOperationStreams(readable, writable) { const oldWindow = readable.window; return new Promise((resolve, reject) => { + function jointOps(op, status) { + function forward() { + if (status.state === 'waiting') { + status.ready.then(forward); + } else if (status.state === 'errored') { + op.error(status.result); + } else { + op.complete(status.result); + } + } + forward(); + } + function select() { const promisesToRace = []; From 020d7fdfd137481978ae22d76ab362298fcaf13e Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Tue, 24 Feb 2015 15:42:06 +0900 Subject: [PATCH 27/93] Add byteCountingPipe --- .../lib/experimental/operation-stream.js | 56 ++++++------- reference-implementation/run-tests.js | 4 +- .../test/experimental/operation-stream.js | 84 +++++++++++++++++++ 3 files changed, 115 insertions(+), 29 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 1332cb19f..ef47fe918 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -6,6 +6,27 @@ export function createOperationStream(strategy) { }; } +export function selectOperationStreams(readable, writable) { + const promisesToRace = []; + + if (readable.state === 'readable') { + promisesToRace.push(readable.aborted); + } else { + // Assert: readable.state === 'readable'. + promisesToRace.push(readable.ready); + } + + if (writable.state === 'writable') { + promisesToRace.push(writable.cancelled); + } else { + // Assert: writable.state === 'writable'. + promisesToRace.push(writable.ready); + promisesToRace.push(writable.waitSpaceChange()); + } + + return Promise.race(promisesToRace); +} + export function pipeOperationStreams(readable, writable) { const oldWindow = readable.window; @@ -23,32 +44,6 @@ export function pipeOperationStreams(readable, writable) { forward(); } - function select() { - const promisesToRace = []; - - if (readable.state === 'readable') { - promisesToRace.push(readable.aborted); - } else { - // Assert: readable.state === 'readable'. - promisesToRace.push(readable.ready); - } - - if (writable.state === 'writable') { - promisesToRace.push(writable.cancelled); - } else { - // Assert: writable.state === 'writable'. - promisesToRace.push(writable.ready); - promisesToRace.push(writable.waitSpaceChange()); - } - - Promise.race(promisesToRace) - .then(loop) - .catch(e => { - readable.window = oldWindow; - reject(); - }); - } - function loop() { for (;;) { if (readable.state === 'aborted' || writable.state === 'cancelled') { @@ -65,7 +60,7 @@ export function pipeOperationStreams(readable, writable) { jointOps(op, writable.write(op.argument)); } else { // Assert: op.type === 'close'. - jointOps(op, writable.close(op.argument)); + jointOps(op, writable.close()); readable.window = oldWindow; resolve(); @@ -79,7 +74,12 @@ export function pipeOperationStreams(readable, writable) { } } - select(); + selectOperationStreams(readable, writable) + .then(loop) + .catch(e => { + readable.window = oldWindow; + reject(); + }); return; } } diff --git a/reference-implementation/run-tests.js b/reference-implementation/run-tests.js index 9f7991850..e11a092a8 100644 --- a/reference-implementation/run-tests.js +++ b/reference-implementation/run-tests.js @@ -4,7 +4,8 @@ const path = require('path'); import ReadableStream from './lib/readable-stream'; import WritableStream from './lib/writable-stream'; import ReadableByteStream from './lib/experimental/readable-byte-stream'; -import { createOperationStream, pipeOperationStreams } from './lib/experimental/operation-stream'; +import { createOperationStream, pipeOperationStreams, selectOperationStreams + } from './lib/experimental/operation-stream'; import ByteLengthQueuingStrategy from './lib/byte-length-queuing-strategy'; import CountQueuingStrategy from './lib/count-queuing-strategy'; import TransformStream from './lib/transform-stream'; @@ -13,6 +14,7 @@ global.ReadableStream = ReadableStream; global.WritableStream = WritableStream; global.createOperationStream = createOperationStream; global.pipeOperationStreams = pipeOperationStreams; +global.selectOperationStreams = selectOperationStreams; global.ReadableByteStream = ReadableByteStream; global.ByteLengthQueuingStrategy = ByteLengthQueuingStrategy; global.CountQueuingStrategy = CountQueuingStrategy; diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index f22980cd0..51fc17c12 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -232,6 +232,90 @@ test('Pipe', t => { }); }); +test('Byte counting transform stream', t => { + const pair0 = createOperationStream(new AdjustableStringStrategy()); + const wos0 = pair0.writable; + const ros0 = pair0.readable; + + const pair1 = createOperationStream(new AdjustableStringStrategy()); + const wos1 = pair1.writable; + const ros1 = pair1.readable; + + wos0.write('hello'); + wos0.write('world'); + wos0.write('goodbye'); + wos0.close(); + + function byteCountingPipe(readable, writable) { + let count = 0; + + return new Promise((resolve, reject) => { + function loop() { + for (;;) { + if (readable.state === 'aborted' || writable.state === 'cancelled') { + readable.window = oldWindow; + reject(); + + return; + } + + if (writable.state === 'writable') { + if (readable.state === 'readable') { + const op = readable.read(); + if (op.type === 'data') { + count += op.argument.length; + op.complete(); + } else { + writable.write(count); + writable.close(); + op.complete(); + + readable.window = oldWindow; + resolve(); + + return; + } + + continue; + } else { + if (writable.space > 0) { + readable.window = 1; + } + } + } + + selectOperationStreams(readable, writable) + .then(loop) + .catch(e => { + readable.window = oldWindow; + reject(); + }); + return; + } + } + loop(); + }); + } + + byteCountingPipe(ros0, wos1) + .catch(e => { + t.fail(e); + t.end(); + }); + + ros1.window = 1; + + ros1.ready.then(() => { + const op = ros1.read(); + t.equals(op.type, 'data'); + t.equals(op.argument, 17); + t.end(); + }).catch(e => { + t.fail(e); + t.end(); + }); +}); + function fillArrayBufferView(view, c, size) { if (size === undefined) { size = view.byteLength; From 9dd062cf0aed00553126e8f2f880132c8f87f776 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Tue, 24 Feb 2015 15:44:16 +0900 Subject: [PATCH 28/93] Fix window saving code --- .../lib/experimental/operation-stream.js | 4 ++-- .../test/experimental/operation-stream.js | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index ef47fe918..0456d5b0b 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -28,9 +28,9 @@ export function selectOperationStreams(readable, writable) { } export function pipeOperationStreams(readable, writable) { - const oldWindow = readable.window; - return new Promise((resolve, reject) => { + const oldWindow = readable.window; + function jointOps(op, status) { function forward() { if (status.state === 'waiting') { diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 51fc17c12..8194d05ed 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -247,9 +247,11 @@ test('Byte counting transform stream', t => { wos0.close(); function byteCountingPipe(readable, writable) { - let count = 0; - return new Promise((resolve, reject) => { + const oldWindow = readable.window; + + let count = 0; + function loop() { for (;;) { if (readable.state === 'aborted' || writable.state === 'cancelled') { From 2d7de48d4699fe23e95c70f6761af851e0ac6405 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Tue, 24 Feb 2015 16:25:53 +0900 Subject: [PATCH 29/93] Move puller and filler into FakeFile --- .../test/experimental/operation-stream.js | 447 +++++++++--------- 1 file changed, 230 insertions(+), 217 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 8194d05ed..f69c60b0b 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -332,131 +332,261 @@ class FakeFile { this._bytesToWrite = 1024; } - readInto(view) { - return new Promise((resolve, reject) => { - setTimeout(() => { - try { - const bytesToWriteThisTime = Math.min(this._bytesToWrite, view.byteLength); + createStreamForPassiveReading(buffers) { + class Puller { + constructor(file, buffers) { + const pair = createOperationStream(new AdjustableArrayBufferStrategy()); + this._readableStream = pair.readable; + this._writableStream = pair.writable; - fillArrayBufferView(view, 1, bytesToWriteThisTime); + this._file = file; + this._fileReadPromise = undefined; - this._bytesToWrite -= bytesToWriteThisTime; + this._buffersAvailable = buffers; - if (this._bytesToWrite === 0) { - resolve({closed: true, view: new Uint8Array(view.buffer, view.byteOffset, bytesToWriteThisTime)}); - } else { - resolve({closed: false, view: new Uint8Array(view.buffer, view.byteOffset, bytesToWriteThisTime)}); - } - } catch (e) { - reject(e); + this._buffersPassedToUser = []; + + this._loop(); + } + + get readableStream() { + return this._readableStream; + } + + _handleFileReadResult(buffer, result) { + this._fileReadPromise = undefined; + const status = this._writableStream.write(result.view); + this._buffersPassedToUser.push({buffer, status}); + if (result.closed) { + this._writableStream.close(); } - }, 0); - }); - } -} + } -class FakeByteSourceWithBufferPool { - constructor(buffers) { - const pair = createOperationStream(new AdjustableArrayBufferStrategy()); - this._readableStream = pair.readable; - this._writableStream = pair.writable; + _select() { + const promisesToRace = []; - this._file = new FakeFile(); - this._fileReadPromise = undefined; + const ws = this._writableStream; - this._buffersAvailable = buffers; + if (ws.state === 'writable') { + promisesToRace.push(ws.cancelled); + } else if (ws.state === 'waiting') { + promisesToRace.push(ws.ready); + } - this._buffersPassedToUser = []; + if (this._fileReadPromise !== undefined) { + promisesToRace.push(this._fileReadPromise); + } - this._loop(); - } + if (this._buffersPassedToUser.length > 0) { + promisesToRace.push(this._buffersPassedToUser[0].status.ready); + } - get readableStream() { - return this._readableStream; - } + Promise.race(promisesToRace) + .then(this._loop.bind(this)) + .catch(ws.abort.bind(ws)); + } - _handleFileReadResult(buffer, result) { - this._fileReadPromise = undefined; - const status = this._writableStream.write(result.view); - this._buffersPassedToUser.push({buffer, status}); - if (result.closed) { - this._writableStream.close(); - } - } + _loop() { + const ws = this._writableStream; - _select() { - const promisesToRace = []; + for (;;) { + if (ws.state === 'cancelled') { + return; + } - const ws = this._writableStream; + var hasProgress = false; - if (ws.state === 'writable') { - promisesToRace.push(ws.cancelled); - } else if (ws.state === 'waiting') { - promisesToRace.push(ws.ready); - } + if (this._buffersPassedToUser.length > 0) { + const entry = this._buffersPassedToUser[0]; + const status = entry.status; + if (status.state === 'completed') { + this._buffersPassedToUser.shift(); - if (this._fileReadPromise !== undefined) { - promisesToRace.push(this._fileReadPromise); - } + this._buffersAvailable.push(entry.buffer); - if (this._buffersPassedToUser.length > 0) { - promisesToRace.push(this._buffersPassedToUser[0].status.ready); + hasProgress = true; + // Keep going. + } else if (status.state === 'errored') { + ws.abort(status.result); + return; + } + } + + if (this._fileReadPromise === undefined && + this._buffersAvailable.length > 0 && + ws.state === 'writable') { + const buffer = this._buffersAvailable.shift(); + + // Clear the acquired buffer for testing. + const view = new Uint8Array(buffer); + fillArrayBufferView(view, 0); + + this._fileReadPromise = this._file._readInto(view) + .then(this._handleFileReadResult.bind(this, buffer)) + .catch(ws.abort); + + hasProgress = true; + } + + if (hasProgress) { + continue; + } + + this._select(); + return; + } + } } - Promise.race(promisesToRace) - .then(this._loop.bind(this)) - .catch(ws.abort.bind(ws)); + const puller = new Puller(this, buffers); + return puller.readableStream; } - _loop() { - const ws = this._writableStream; + createStreamForReadingWithBuffer() { + class Filler { + constructor(file) { + const pair = createOperationStream(new AdjustableArrayBufferStrategy()); + this._readableStream = pair.readable; + this._writableStream = pair.writable; - for (;;) { - if (ws.state === 'cancelled') { - return; + this._currentRequest = undefined; + + this._file = file; + + this._fileReadStatus = undefined; + + this._readableStream.window = 1; + + this._loop(); + } + + get writableStream() { + return this._writableStream; } - var hasProgress = false; + _handleFileReadCompletion() { + this._currentRequest.complete(this._fileReadStatus.result); + this._currentRequest = undefined; + + this._fileReadStatus = undefined + } - if (this._buffersPassedToUser.length > 0) { - const entry = this._buffersPassedToUser[0]; - const status = entry.status; - if (status.state === 'completed') { - this._buffersPassedToUser.shift(); + _handleFileReadError() { + const error = this._fileReadStatus.result; - this._buffersAvailable.push(entry.buffer); + this._currentRequest.error(error); + this._currentRequest = undefined; - hasProgress = true; - // Keep going. - } else if (status.state === 'errored') { - ws.abort(status.result); + this._readableStream.cancel(error); + + this._fileReadStatus = undefined; + } + + _transformReadFromFileFulfillment(result) { + this._fileReadStatus.state = 'completed'; + this._fileReadStatus.result = {closed: result.closed, bytesWritten: result.view.byteLength}; + } + + _transformReadFromFileRejection(e) { + this._fileReadStatus.state = 'errored'; + this._fileReadStatus.result = e; + } + + _readFromFile() { + const op = this._readableStream.read(); + + if (op.type === 'close') { return; } + // Assert: op.type === 'data'. + + this._currentRequest = op; + + const status = {}; + status.state = 'waiting'; + status.ready = this._file._readInto(op.argument) + .then(this._transformReadFromFileFulfillment.bind(this)) + .catch(this._transformReadFromFileRejection.bind(this)); + + this._fileReadStatus = status; } - if (this._fileReadPromise === undefined && - this._buffersAvailable.length > 0 && - ws.state === 'writable') { - const buffer = this._buffersAvailable.shift(); + _select() { + const promisesToRace = []; - // Clear the acquired buffer for testing. - const view = new Uint8Array(buffer); - fillArrayBufferView(view, 0); + const rs = this._readableStream; - this._fileReadPromise = this._file.readInto(view) - .then(this._handleFileReadResult.bind(this, buffer)) - .catch(ws.abort); + if (rs.state === 'readable') { + promisesToRace.push(rs.aborted); + } else if (rs.state === 'waiting') { + promisesToRace.push(rs.ready); + } - hasProgress = true; - } + if (this._fileReadStatus !== undefined && this._fileReadStatus.state === 'waiting') { + promisesToRace.push(this._fileReadStatus.ready); + } - if (hasProgress) { - continue; + Promise.race(promisesToRace) + .then(this._loop.bind(this)) + .catch(rs.cancel.bind(rs)); } - this._select(); - return; + _loop() { + const rs = this._readableStream; + + for (;;) { + if (rs.state === 'aborted') { + if (this._currentRequest !== undefined) { + this._currentRequest.error(rs.abortOperation.argument); + } + + return; + } + + if (this._currentRequest !== undefined) { + if (this._fileReadStatus.state === 'errored') { + this._handleFileReadError(); + return; + } else if (this._fileReadStatus.state === 'completed') { + this._handleFileReadCompletion(); + } + } + + if (rs.state === 'readable' && this._currentRequest === undefined) { + this._readFromFile(); + continue; + } + + this._select(); + return; + } + } } + + const filler = new Filler(this); + return filler.writableStream; + } + + _readInto(view) { + return new Promise((resolve, reject) => { + setTimeout(() => { + try { + const bytesToWriteThisTime = Math.min(this._bytesToWrite, view.byteLength); + + fillArrayBufferView(view, 1, bytesToWriteThisTime); + + this._bytesToWrite -= bytesToWriteThisTime; + + if (this._bytesToWrite === 0) { + resolve({closed: true, view: new Uint8Array(view.buffer, view.byteOffset, bytesToWriteThisTime)}); + } else { + resolve({closed: false, view: new Uint8Array(view.buffer, view.byteOffset, bytesToWriteThisTime)}); + } + } catch (e) { + reject(e); + } + }, 0); + }); } } @@ -536,13 +666,14 @@ test('Piping from a source with a buffer pool to a buffer taking sink', t => { pool.push(new ArrayBuffer(10)); } - const source = new FakeByteSourceWithBufferPool(pool); + const file = new FakeFile(); const sink = new FakeBufferTakingByteSink(); sink.writableStream.window = 64; - // pipeOperationStreams automatically adjusts source.window. - const pipePromise = pipeOperationStreams(source.readableStream, sink.writableStream) + // pipeOperationStreams automatically adjusts window of the readable side. + const pipePromise = pipeOperationStreams( + file.createStreamForPassiveReading(pool), sink.writableStream) pipePromise.catch(e => { t.fail(e); t.end(); @@ -570,10 +701,11 @@ test('Consuming bytes from a source with a buffer pool via the ReadableStream in pool.push(new ArrayBuffer(10)); } - const source = new FakeByteSourceWithBufferPool(pool); - source.readableStream.window = 64; + const file = new FakeFile(); + const rs = file.createStreamForPassiveReading(pool); + rs.window = 64; - const sink = new FakeBufferTakingByteSink(source.readableStream); + const sink = new FakeBufferTakingByteSink(rs); sink.result.then(bytesRead => { t.equals(bytesRead, 1024); @@ -590,126 +722,6 @@ test('Consuming bytes from a source with a buffer pool via the ReadableStream in }); }); -class FakeBufferTakingByteSource { - constructor() { - const pair = createOperationStream(new AdjustableArrayBufferStrategy()); - this._readableStream = pair.readable; - this._writableStream = pair.writable; - - this._currentRequest = undefined; - - this._file = new FakeFile(); - - this._fileReadStatus = undefined; - - this._readableStream.window = 1; - - this._loop(); - } - - get writableStream() { - return this._writableStream; - } - - _handleFileReadCompletion() { - this._currentRequest.complete(this._fileReadStatus.result); - this._currentRequest = undefined; - - this._fileReadStatus = undefined - } - - _handleFileReadError() { - const error = this._fileReadStatus.result; - - this._currentRequest.error(error); - this._currentRequest = undefined; - - this._readableStream.cancel(error); - - this._fileReadStatus = undefined; - } - - _transformReadFromFileFulfillment(result) { - this._fileReadStatus.state = 'completed'; - this._fileReadStatus.result = {closed: result.closed, bytesWritten: result.view.byteLength}; - } - - _transformReadFromFileRejection(e) { - this._fileReadStatus.state = 'errored'; - this._fileReadStatus.result = e; - } - - _readFromFile() { - const op = this._readableStream.read(); - - if (op.type === 'close') { - return; - } - // Assert: op.type === 'data'. - - this._currentRequest = op; - - const status = {}; - status.state = 'waiting'; - status.ready = this._file.readInto(op.argument) - .then(this._transformReadFromFileFulfillment.bind(this)) - .catch(this._transformReadFromFileRejection.bind(this)); - - this._fileReadStatus = status; - } - - _select() { - const promisesToRace = []; - - const rs = this._readableStream; - - if (rs.state === 'readable') { - promisesToRace.push(rs.aborted); - } else if (rs.state === 'waiting') { - promisesToRace.push(rs.ready); - } - - if (this._fileReadStatus !== undefined && this._fileReadStatus.state === 'waiting') { - promisesToRace.push(this._fileReadStatus.ready); - } - - Promise.race(promisesToRace) - .then(this._loop.bind(this)) - .catch(rs.cancel.bind(rs)); - } - - _loop() { - const rs = this._readableStream; - - for (;;) { - if (rs.state === 'aborted') { - if (this._currentRequest !== undefined) { - this._currentRequest.error(rs.abortOperation.argument); - } - - return; - } - - if (this._currentRequest !== undefined) { - if (this._fileReadStatus.state === 'errored') { - this._handleFileReadError(); - return; - } else if (this._fileReadStatus.state === 'completed') { - this._handleFileReadCompletion(); - } - } - - if (rs.state === 'readable' && this._currentRequest === undefined) { - this._readFromFile(); - continue; - } - - this._select(); - return; - } - } -} - class FakeByteSinkWithBuffer { constructor() { const pair = createOperationStream(new AdjustableArrayBufferStrategy()); @@ -828,14 +840,15 @@ test('Piping from a buffer taking source to a sink with buffer', t => { pool.push(new ArrayBuffer(10)); } - const source = new FakeBufferTakingByteSource(); - source.writableStream.window = 16; + const file = new FakeFile(); + const writableStream = file.createStreamForReadingWithBuffer(); + writableStream.window = 16; const sink = new FakeByteSinkWithBuffer(); // This also works. // sink.readableStream.window = 16; - const pipePromise = pipeOperationStreams(sink.readableStream, source.writableStream); + const pipePromise = pipeOperationStreams(sink.readableStream, writableStream); pipePromise.catch(e => { t.fail(e); t.end(); From 1d87cce517332f2dc9d56e558575004b5156cf75 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Tue, 24 Feb 2015 17:14:58 +0900 Subject: [PATCH 30/93] Bunch of reorganization to merge stream creation methods into source/sink --- .../test/experimental/operation-stream.js | 340 ++++++++++-------- 1 file changed, 185 insertions(+), 155 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index f69c60b0b..6635082c6 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -327,12 +327,12 @@ function fillArrayBufferView(view, c, size) { } } -class FakeFile { +class FakeFileBackedByteSource { constructor() { this._bytesToWrite = 1024; } - createStreamForPassiveReading(buffers) { + createBufferProducingStreamWithPool(buffers) { class Puller { constructor(file, buffers) { const pair = createOperationStream(new AdjustableArrayBufferStrategy()); @@ -442,7 +442,7 @@ class FakeFile { return puller.readableStream; } - createStreamForReadingWithBuffer() { + createBufferFillingStream() { class Filler { constructor(file) { const pair = createOperationStream(new AdjustableArrayBufferStrategy()); @@ -590,90 +590,21 @@ class FakeFile { } } -class FakeBufferTakingByteSink { - constructor(readableStream) { - if (readableStream === undefined) { - const pair = createOperationStream(new AdjustableArrayBufferStrategy()); - this._readableStream = pair.readable; - this._writableStream = pair.writable; - } else { - this._readableStream = readableStream; - this._writableStream = undefined; - } - - this._resultPromise = new Promise((resolve, reject) => { - this._resolveResultPromise = resolve; - this._rejectResultPromise = reject; - }); - this._bytesRead = 0; - - this._loop(); - } - - get writableStream() { - return this._writableStream; - } - - get result() { - return this._resultPromise; - } - - _loop() { - const rs = this._readableStream; - - for (;;) { - if (rs.state === 'readable') { - const op = rs.read(); - if (op.type === 'data') { - const view = op.argument; - - // Verify contents of the buffer. - for (var i = 0; i < view.byteLength; ++i) { - if (view[i] === 1) { - ++this._bytesRead; - } - } - - // Release the buffer. - op.complete(); - - continue; - } else if (op.type === 'close') { - // Acknowledge the closure. - op.complete(); - - this._resolveResultPromise(this._bytesRead); - } else { - this._rejectResultPromise(op.type); - } - } else if (rs.state === 'waiting') { - rs.ready - .then(this._loop.bind(this)) - .catch(this._rejectResultPromise.bind(this)); - } else if (rs.state === 'aborted') { - this._rejectResultPromise(rs.abortOperation.argument); - } else { - this._rejectResultPromise(rs.state); - } - return; - } - } -} - test('Piping from a source with a buffer pool to a buffer taking sink', t => { const pool = []; for (var i = 0; i < 10; ++i) { pool.push(new ArrayBuffer(10)); } - const file = new FakeFile(); + const file = new FakeFileBackedByteSource(); - const sink = new FakeBufferTakingByteSink(); - sink.writableStream.window = 64; + const sink = new BytesSetToOneExpectingByteSink(); + const bufferConsumingStream = sink.createBufferConsumingStream(); + bufferConsumingStream.window = 64; // pipeOperationStreams automatically adjusts window of the readable side. const pipePromise = pipeOperationStreams( - file.createStreamForPassiveReading(pool), sink.writableStream) + file.createBufferProducingStreamWithPool(pool), bufferConsumingStream) pipePromise.catch(e => { t.fail(e); t.end(); @@ -701,11 +632,14 @@ test('Consuming bytes from a source with a buffer pool via the ReadableStream in pool.push(new ArrayBuffer(10)); } - const file = new FakeFile(); - const rs = file.createStreamForPassiveReading(pool); - rs.window = 64; + const file = new FakeFileBackedByteSource(); + const bufferProducingStream = file.createBufferProducingStreamWithPool(pool); + bufferProducingStream.window = 64; + + const sink = new BytesSetToOneExpectingByteSink(); + + sink.writeFrom(bufferProducingStream); - const sink = new FakeBufferTakingByteSink(rs); sink.result.then(bytesRead => { t.equals(bytesRead, 1024); @@ -722,15 +656,9 @@ test('Consuming bytes from a source with a buffer pool via the ReadableStream in }); }); -class FakeByteSinkWithBuffer { - constructor() { - const pair = createOperationStream(new AdjustableArrayBufferStrategy()); - this._readableStream = pair.readable; - this._writableStream = pair.writable; - - this._currentReadStatus = undefined; - - this._buffer = new ArrayBuffer(16); +class BytesSetToOneExpectingByteSinkInternalWriter { + constructor(sink, readableStream) { + this._readableStream = readableStream; this._resultPromise = new Promise((resolve, reject) => { this._resolveResultPromise = resolve; @@ -738,100 +666,201 @@ class FakeByteSinkWithBuffer { }); this._bytesRead = 0; + this._sink = sink; + this._loop(); } - get readableStream() { - return this._readableStream; - } + _loop() { + const rs = this._readableStream; - get result() { - return this._resultPromise; - } + for (;;) { + if (rs.state === 'readable') { + const op = rs.read(); + if (op.type === 'data') { + const view = op.argument; + + // Verify contents of the buffer. + for (var i = 0; i < view.byteLength; ++i) { + if (view[i] === 1) { + this._sink._count(1); + } + } + + // Release the buffer. + op.complete(); - _handleReadCompletion() { - const bytesWritten = this._currentReadStatus.result.bytesWritten; - const closed = this._currentReadStatus.result.closed; + continue; + } else if (op.type === 'close') { + // Acknowledge the closure. + op.complete(); - // Verify contents of the buffer. - const view = new Uint8Array(this._buffer, 0, bytesWritten); - for (var i = 0; i < bytesWritten; ++i) { - if (view[i] === 1) { - ++this._bytesRead; + this._sink._complete(); + } else { + this._sink._error(op.type); + } + } else if (rs.state === 'waiting') { + rs.ready + .then(this._loop.bind(this)) + .catch(this._sink._error.bind(this._sink)); + } else if (rs.state === 'aborted') { + this._sink._error(rs.abortOperation.argument); + } else { + this._sink._error(rs.state); } + return; } + } +} - this._currentReadStatus = undefined; +class BytesSetToOneExpectingByteSink { + constructor() { + this._bytesRead = 0; - return closed; + this._resultPromise = new Promise((resolve, reject) => { + this._resolveResultPromise = resolve; + this._rejectResultPromise = reject; + }); } - _select() { - const promisesToRace = []; - - const ws = this._writableStream; + get result() { + return this._resultPromise; + } - if (ws.state === 'writable') { - promisesToRace.push(ws.cancelled); - } else if (ws.state === 'waiting') { - promisesToRace.push(ws.ready); - } + _count(size) { + this._bytesRead += size; + } - if (this._currentReadStatus !== undefined && this._currentReadStatus.state === 'waiting') { - promisesToRace.push(this._currentReadStatus.ready); - } + _complete() { + this._resolveResultPromise(this._bytesRead); + } - Promise.race(promisesToRace) - .then(this._loop.bind(this)) - .catch(this._rejectResultPromise.bind(this)); + _error(e) { + this._rejectResultPromise(e); } - _loop() { - const ws = this._writableStream; + createBufferProducingStream() { + class BufferProvidingWriter { + constructor(sink) { + const pair = createOperationStream(new AdjustableArrayBufferStrategy()); + this._readableStream = pair.readable; + this._writableStream = pair.writable; - for (;;) { - if (this._currentReadStatus !== undefined && this._currentReadStatus.state === 'errored') { - const error = this._currentReadStatus.result; this._currentReadStatus = undefined; - ws.abort(error); - this._rejectResultPromise(error); + this._buffer = new ArrayBuffer(16); + + this._sink = sink; - return; + this._loop(); } - if (ws.state === 'cancelled') { - this._rejectResultPromise(ws.cancelOperation.argument); - return; + get readableStream() { + return this._readableStream; } - let hasProgress = false; + _handleReadCompletion() { + const bytesWritten = this._currentReadStatus.result.bytesWritten; + const closed = this._currentReadStatus.result.closed; - if (this._currentReadStatus !== undefined && this._currentReadStatus.state === 'completed') { - const closed = this._handleReadCompletion(); - if (closed) { - ws.close(); - this._resolveResultPromise(this._bytesRead); - return; + // Verify contents of the buffer. + const view = new Uint8Array(this._buffer, 0, bytesWritten); + for (var i = 0; i < bytesWritten; ++i) { + if (view[i] === 1) { + this._sink._count(1); + } } - hasProgress = true; + this._currentReadStatus = undefined; + + return closed; } - if (ws.state === 'writable' && this._currentReadStatus === undefined) { - this._currentReadStatus = ws.write(new Uint8Array(this._buffer)); + _select() { + const promisesToRace = []; - hasProgress = true; - } + const ws = this._writableStream; - if (hasProgress) { - continue; + if (ws.state === 'writable') { + promisesToRace.push(ws.cancelled); + } else if (ws.state === 'waiting') { + promisesToRace.push(ws.ready); + } + + if (this._currentReadStatus !== undefined && this._currentReadStatus.state === 'waiting') { + promisesToRace.push(this._currentReadStatus.ready); + } + + Promise.race(promisesToRace) + .then(this._loop.bind(this)) + .catch(this._sink._error.bind(this._sink)); } - this._select(); - return; + _loop() { + const ws = this._writableStream; + + for (;;) { + if (this._currentReadStatus !== undefined && this._currentReadStatus.state === 'errored') { + const error = this._currentReadStatus.result; + this._currentReadStatus = undefined; + + ws.abort(error); + this._sink._error(error); + + return; + } + + if (ws.state === 'cancelled') { + this._sink._error(ws.cancelOperation.argument); + return; + } + + let hasProgress = false; + + if (this._currentReadStatus !== undefined && this._currentReadStatus.state === 'completed') { + const closed = this._handleReadCompletion(); + if (closed) { + ws.close(); + this._sink._complete(); + return; + } + + hasProgress = true; + } + + if (ws.state === 'writable' && this._currentReadStatus === undefined) { + this._currentReadStatus = ws.write(new Uint8Array(this._buffer)); + + hasProgress = true; + } + + if (hasProgress) { + continue; + } + + this._select(); + return; + } + } } + + const writer = new BufferProvidingWriter(this); + return writer.readableStream; } + + createBufferConsumingStream() { + const pair = createOperationStream(new AdjustableArrayBufferStrategy()); + this._readableStream = pair.readable; + this._writableStream = pair.writable; + + new BytesSetToOneExpectingByteSinkInternalWriter(this, pair.readable); + return pair.writable; + } + + writeFrom(readableStream) { + new BytesSetToOneExpectingByteSinkInternalWriter(this, readableStream); + } + } test('Piping from a buffer taking source to a sink with buffer', t => { @@ -840,15 +869,16 @@ test('Piping from a buffer taking source to a sink with buffer', t => { pool.push(new ArrayBuffer(10)); } - const file = new FakeFile(); - const writableStream = file.createStreamForReadingWithBuffer(); - writableStream.window = 16; - - const sink = new FakeByteSinkWithBuffer(); + const file = new FakeFileBackedByteSource(); + const bufferFillingStream = file.createBufferFillingStream(); + bufferFillingStream.window = 16; // This also works. - // sink.readableStream.window = 16; + // writableBufferProvidingStream.window = 16; + + const sink = new BytesSetToOneExpectingByteSink(); + const writableBufferProducingStream = sink.createBufferProducingStream(); - const pipePromise = pipeOperationStreams(sink.readableStream, writableStream); + const pipePromise = pipeOperationStreams(writableBufferProducingStream, bufferFillingStream); pipePromise.catch(e => { t.fail(e); t.end(); From acd80343cab980a397fc7b97745fc2b046e6a1cc Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Tue, 24 Feb 2015 17:17:09 +0900 Subject: [PATCH 31/93] Group tests --- .../test/experimental/operation-stream.js | 132 +++++++++--------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 6635082c6..d31f469ef 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -590,72 +590,6 @@ class FakeFileBackedByteSource { } } -test('Piping from a source with a buffer pool to a buffer taking sink', t => { - const pool = []; - for (var i = 0; i < 10; ++i) { - pool.push(new ArrayBuffer(10)); - } - - const file = new FakeFileBackedByteSource(); - - const sink = new BytesSetToOneExpectingByteSink(); - const bufferConsumingStream = sink.createBufferConsumingStream(); - bufferConsumingStream.window = 64; - - // pipeOperationStreams automatically adjusts window of the readable side. - const pipePromise = pipeOperationStreams( - file.createBufferProducingStreamWithPool(pool), bufferConsumingStream) - pipePromise.catch(e => { - t.fail(e); - t.end(); - }); - - sink.result.then(bytesRead => { - t.equals(bytesRead, 1024); - - pipePromise.then(() => { - Promise.resolve().then(() => { - // Check that the buffers have been returned to the pool. - t.equals(pool.length, 10); - t.end(); - }); - }); - }).catch(e => { - t.fail(e); - t.end(); - }); -}); - -test('Consuming bytes from a source with a buffer pool via the ReadableStream interface', t => { - const pool = []; - for (var i = 0; i < 10; ++i) { - pool.push(new ArrayBuffer(10)); - } - - const file = new FakeFileBackedByteSource(); - const bufferProducingStream = file.createBufferProducingStreamWithPool(pool); - bufferProducingStream.window = 64; - - const sink = new BytesSetToOneExpectingByteSink(); - - sink.writeFrom(bufferProducingStream); - - sink.result.then(bytesRead => { - t.equals(bytesRead, 1024); - - Promise.resolve().then(() => { - Promise.resolve().then(() => { - // Check that the buffers have been returned to the pool. - t.equals(pool.length, 10); - t.end(); - }); - }); - }).catch(e => { - t.fail(e); - t.end(); - }); -}); - class BytesSetToOneExpectingByteSinkInternalWriter { constructor(sink, readableStream) { this._readableStream = readableStream; @@ -863,6 +797,72 @@ class BytesSetToOneExpectingByteSink { } +test('Piping from a source with a buffer pool to a buffer taking sink', t => { + const pool = []; + for (var i = 0; i < 10; ++i) { + pool.push(new ArrayBuffer(10)); + } + + const file = new FakeFileBackedByteSource(); + + const sink = new BytesSetToOneExpectingByteSink(); + const bufferConsumingStream = sink.createBufferConsumingStream(); + bufferConsumingStream.window = 64; + + // pipeOperationStreams automatically adjusts window of the readable side. + const pipePromise = pipeOperationStreams( + file.createBufferProducingStreamWithPool(pool), bufferConsumingStream) + pipePromise.catch(e => { + t.fail(e); + t.end(); + }); + + sink.result.then(bytesRead => { + t.equals(bytesRead, 1024); + + pipePromise.then(() => { + Promise.resolve().then(() => { + // Check that the buffers have been returned to the pool. + t.equals(pool.length, 10); + t.end(); + }); + }); + }).catch(e => { + t.fail(e); + t.end(); + }); +}); + +test('Consuming bytes from a source with a buffer pool via the ReadableStream interface', t => { + const pool = []; + for (var i = 0; i < 10; ++i) { + pool.push(new ArrayBuffer(10)); + } + + const file = new FakeFileBackedByteSource(); + const bufferProducingStream = file.createBufferProducingStreamWithPool(pool); + bufferProducingStream.window = 64; + + const sink = new BytesSetToOneExpectingByteSink(); + + sink.writeFrom(bufferProducingStream); + + sink.result.then(bytesRead => { + t.equals(bytesRead, 1024); + + Promise.resolve().then(() => { + Promise.resolve().then(() => { + // Check that the buffers have been returned to the pool. + t.equals(pool.length, 10); + t.end(); + }); + }); + }).catch(e => { + t.fail(e); + t.end(); + }); +}); + test('Piping from a buffer taking source to a sink with buffer', t => { const pool = []; for (var i = 0; i < 10; ++i) { From 9b10507df06811353c939024e3e71c313d85abb0 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Tue, 24 Feb 2015 17:35:26 +0900 Subject: [PATCH 32/93] Add comment about createBufferFillingStream --- .../test/experimental/operation-stream.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index d31f469ef..555796b46 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -442,6 +442,22 @@ class FakeFileBackedByteSource { return puller.readableStream; } + // Returns a WritableOperationStream. + // + // Example semantics: + // + // POSIX socket: + // - The stream becomes writable when epoll(7) returns, and stays to be writable until read(2) returns EAGAIN. + // - The status returned on write() call on the stream will set to the number of bytes written into the buffer passed + // on write() call. + // + // Blocking or async I/O interfaces which takes a buffer on reading function call: + // - The stream is always writable. + // - Same as POSIX socket. + // + // I/O that tells us how much data can be read: + // - Hide window setter/getter to the user but adjust based on the info (how much data can be read), so that the user + // can know the info via space getter. createBufferFillingStream() { class Filler { constructor(file) { From 9084385c693d0da2f820485138fc7c7b22b0a83e Mon Sep 17 00:00:00 2001 From: rieszgithub Date: Wed, 25 Feb 2015 02:19:09 +0900 Subject: [PATCH 33/93] Remove unnecessary accessor from internal classes --- .../test/experimental/operation-stream.js | 30 +++++++------------ 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 555796b46..c260ba594 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -460,10 +460,8 @@ class FakeFileBackedByteSource { // can know the info via space getter. createBufferFillingStream() { class Filler { - constructor(file) { - const pair = createOperationStream(new AdjustableArrayBufferStrategy()); - this._readableStream = pair.readable; - this._writableStream = pair.writable; + constructor(file, readableStream) { + this._readableStream = readableStream; this._currentRequest = undefined; @@ -476,10 +474,6 @@ class FakeFileBackedByteSource { this._loop(); } - get writableStream() { - return this._writableStream; - } - _handleFileReadCompletion() { this._currentRequest.complete(this._fileReadStatus.result); this._currentRequest = undefined; @@ -579,8 +573,9 @@ class FakeFileBackedByteSource { } } - const filler = new Filler(this); - return filler.writableStream; + const pair = createOperationStream(new AdjustableArrayBufferStrategy()); + const filler = new Filler(this, pair.readable); + return pair.writable; } _readInto(view) { @@ -691,10 +686,8 @@ class BytesSetToOneExpectingByteSink { createBufferProducingStream() { class BufferProvidingWriter { - constructor(sink) { - const pair = createOperationStream(new AdjustableArrayBufferStrategy()); - this._readableStream = pair.readable; - this._writableStream = pair.writable; + constructor(sink, writableStream) { + this._writableStream = writableStream; this._currentReadStatus = undefined; @@ -705,10 +698,6 @@ class BytesSetToOneExpectingByteSink { this._loop(); } - get readableStream() { - return this._readableStream; - } - _handleReadCompletion() { const bytesWritten = this._currentReadStatus.result.bytesWritten; const closed = this._currentReadStatus.result.closed; @@ -794,8 +783,9 @@ class BytesSetToOneExpectingByteSink { } } - const writer = new BufferProvidingWriter(this); - return writer.readableStream; + const pair = createOperationStream(new AdjustableArrayBufferStrategy()); + const writer = new BufferProvidingWriter(this, pair.writable); + return pair.readable; } createBufferConsumingStream() { From ce8a3312aa0db9986dfdc54238e9e3dc7cbd50e6 Mon Sep 17 00:00:00 2001 From: rieszgithub Date: Wed, 25 Feb 2015 02:33:22 +0900 Subject: [PATCH 34/93] Clean up --- .../test/experimental/operation-stream.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index c260ba594..841a05c59 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -574,7 +574,7 @@ class FakeFileBackedByteSource { } const pair = createOperationStream(new AdjustableArrayBufferStrategy()); - const filler = new Filler(this, pair.readable); + new Filler(this, pair.readable); return pair.writable; } @@ -588,10 +588,11 @@ class FakeFileBackedByteSource { this._bytesToWrite -= bytesToWriteThisTime; + const writtenRegion = new Uint8Array(view.buffer, view.byteOffset, bytesToWriteThisTime); if (this._bytesToWrite === 0) { - resolve({closed: true, view: new Uint8Array(view.buffer, view.byteOffset, bytesToWriteThisTime)}); + resolve({closed: true, view: writtenRegion}); } else { - resolve({closed: false, view: new Uint8Array(view.buffer, view.byteOffset, bytesToWriteThisTime)}); + resolve({closed: false, view: writtenRegion}); } } catch (e) { reject(e); From ae610b57e532adc7426a59801c1c8f242e289805 Mon Sep 17 00:00:00 2001 From: rieszgithub Date: Wed, 25 Feb 2015 02:35:00 +0900 Subject: [PATCH 35/93] Remove stale code --- .../test/experimental/operation-stream.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 841a05c59..b06b09fb5 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -606,12 +606,6 @@ class BytesSetToOneExpectingByteSinkInternalWriter { constructor(sink, readableStream) { this._readableStream = readableStream; - this._resultPromise = new Promise((resolve, reject) => { - this._resolveResultPromise = resolve; - this._rejectResultPromise = reject; - }); - this._bytesRead = 0; - this._sink = sink; this._loop(); From f7ccb6ff22d2c5028f34fa61e8bd0432c65a6b0a Mon Sep 17 00:00:00 2001 From: rieszgithub Date: Wed, 25 Feb 2015 02:39:45 +0900 Subject: [PATCH 36/93] Clean up --- .../test/experimental/operation-stream.js | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index b06b09fb5..10d3ae9ce 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -334,10 +334,8 @@ class FakeFileBackedByteSource { createBufferProducingStreamWithPool(buffers) { class Puller { - constructor(file, buffers) { - const pair = createOperationStream(new AdjustableArrayBufferStrategy()); - this._readableStream = pair.readable; - this._writableStream = pair.writable; + constructor(file, buffers, writableStream) { + this._writableStream = writableStream; this._file = file; this._fileReadPromise = undefined; @@ -349,10 +347,6 @@ class FakeFileBackedByteSource { this._loop(); } - get readableStream() { - return this._readableStream; - } - _handleFileReadResult(buffer, result) { this._fileReadPromise = undefined; const status = this._writableStream.write(result.view); @@ -438,8 +432,9 @@ class FakeFileBackedByteSource { } } - const puller = new Puller(this, buffers); - return puller.readableStream; + const pair = createOperationStream(new AdjustableArrayBufferStrategy()); + new Puller(this, buffers, pair.writable); + return pair.readable; } // Returns a WritableOperationStream. @@ -462,6 +457,7 @@ class FakeFileBackedByteSource { class Filler { constructor(file, readableStream) { this._readableStream = readableStream; + this._readableStream.window = 1; this._currentRequest = undefined; @@ -469,8 +465,6 @@ class FakeFileBackedByteSource { this._fileReadStatus = undefined; - this._readableStream.window = 1; - this._loop(); } @@ -779,15 +773,12 @@ class BytesSetToOneExpectingByteSink { } const pair = createOperationStream(new AdjustableArrayBufferStrategy()); - const writer = new BufferProvidingWriter(this, pair.writable); + new BufferProvidingWriter(this, pair.writable); return pair.readable; } createBufferConsumingStream() { const pair = createOperationStream(new AdjustableArrayBufferStrategy()); - this._readableStream = pair.readable; - this._writableStream = pair.writable; - new BytesSetToOneExpectingByteSinkInternalWriter(this, pair.readable); return pair.writable; } From 815649bb91325e1dbd4c26bd95fbc13ac8383ddf Mon Sep 17 00:00:00 2001 From: rieszgithub Date: Wed, 25 Feb 2015 02:41:47 +0900 Subject: [PATCH 37/93] Remove unused code --- .../test/experimental/operation-stream.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 10d3ae9ce..90e0345fc 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -856,11 +856,6 @@ test('Consuming bytes from a source with a buffer pool via the ReadableStream in }); test('Piping from a buffer taking source to a sink with buffer', t => { - const pool = []; - for (var i = 0; i < 10; ++i) { - pool.push(new ArrayBuffer(10)); - } - const file = new FakeFileBackedByteSource(); const bufferFillingStream = file.createBufferFillingStream(); bufferFillingStream.window = 16; From 3772a9e49063ac97df400fff40128dac54f83038 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Wed, 25 Feb 2015 16:06:55 +0900 Subject: [PATCH 38/93] Fix precondition check in abort() --- reference-implementation/lib/experimental/operation-stream.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 0456d5b0b..12ca589d5 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -303,10 +303,10 @@ class OperationStream { abort(reason) { if (this._writableState === 'aborted') { - return Promise.reject(new TypeError('already aborted')); + throw new TypeError('already aborted'); } if (this._writableState === 'cancelled') { - return Promise.reject(new TypeError('already cancelled')) + throw new TypeError('already cancelled'); } for (var i = this._queue.length - 1; i >= 0; --i) { From 8b6462a94e70c4d7b1894ae2fb70538554ff1500 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Wed, 25 Feb 2015 16:10:00 +0900 Subject: [PATCH 39/93] Remove ToRace from the variable holding promises to pass to Promise.race() --- .../lib/experimental/operation-stream.js | 16 +++++----- .../test/experimental/operation-stream.js | 32 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 12ca589d5..421e70046 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -1,5 +1,5 @@ export function createOperationStream(strategy) { - var stream = new OperationStream(strategy); + const stream = new OperationStream(strategy); return { writable: new WritableOperationStream(stream), readable: new ReadableOperationStream(stream) @@ -7,24 +7,24 @@ export function createOperationStream(strategy) { } export function selectOperationStreams(readable, writable) { - const promisesToRace = []; + const promises = []; if (readable.state === 'readable') { - promisesToRace.push(readable.aborted); + promises.push(readable.aborted); } else { // Assert: readable.state === 'readable'. - promisesToRace.push(readable.ready); + promises.push(readable.ready); } if (writable.state === 'writable') { - promisesToRace.push(writable.cancelled); + promises.push(writable.cancelled); } else { // Assert: writable.state === 'writable'. - promisesToRace.push(writable.ready); - promisesToRace.push(writable.waitSpaceChange()); + promises.push(writable.ready); + promises.push(writable.waitSpaceChange()); } - return Promise.race(promisesToRace); + return Promise.race(promises); } export function pipeOperationStreams(readable, writable) { diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 90e0345fc..e51e4cd40 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -357,25 +357,25 @@ class FakeFileBackedByteSource { } _select() { - const promisesToRace = []; + const promises = []; const ws = this._writableStream; if (ws.state === 'writable') { - promisesToRace.push(ws.cancelled); + promises.push(ws.cancelled); } else if (ws.state === 'waiting') { - promisesToRace.push(ws.ready); + promises.push(ws.ready); } if (this._fileReadPromise !== undefined) { - promisesToRace.push(this._fileReadPromise); + promises.push(this._fileReadPromise); } if (this._buffersPassedToUser.length > 0) { - promisesToRace.push(this._buffersPassedToUser[0].status.ready); + promises.push(this._buffersPassedToUser[0].status.ready); } - Promise.race(promisesToRace) + Promise.race(promises) .then(this._loop.bind(this)) .catch(ws.abort.bind(ws)); } @@ -516,21 +516,21 @@ class FakeFileBackedByteSource { } _select() { - const promisesToRace = []; + const promises = []; const rs = this._readableStream; if (rs.state === 'readable') { - promisesToRace.push(rs.aborted); + promises.push(rs.aborted); } else if (rs.state === 'waiting') { - promisesToRace.push(rs.ready); + promises.push(rs.ready); } if (this._fileReadStatus !== undefined && this._fileReadStatus.state === 'waiting') { - promisesToRace.push(this._fileReadStatus.ready); + promises.push(this._fileReadStatus.ready); } - Promise.race(promisesToRace) + Promise.race(promises) .then(this._loop.bind(this)) .catch(rs.cancel.bind(rs)); } @@ -705,21 +705,21 @@ class BytesSetToOneExpectingByteSink { } _select() { - const promisesToRace = []; + const promises = []; const ws = this._writableStream; if (ws.state === 'writable') { - promisesToRace.push(ws.cancelled); + promises.push(ws.cancelled); } else if (ws.state === 'waiting') { - promisesToRace.push(ws.ready); + promises.push(ws.ready); } if (this._currentReadStatus !== undefined && this._currentReadStatus.state === 'waiting') { - promisesToRace.push(this._currentReadStatus.ready); + promises.push(this._currentReadStatus.ready); } - Promise.race(promisesToRace) + Promise.race(promises) .then(this._loop.bind(this)) .catch(this._sink._error.bind(this._sink)); } From afb66110ac14f993b2b34b8d0e34b02cbc806e33 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Wed, 25 Feb 2015 16:22:28 +0900 Subject: [PATCH 40/93] Fix rejection path in pipeOperationStreams() --- .../lib/experimental/operation-stream.js | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 421e70046..2c2ae48be 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -31,6 +31,11 @@ export function pipeOperationStreams(readable, writable) { return new Promise((resolve, reject) => { const oldWindow = readable.window; + function restoreWindowAndReject(e) { + readable.window = oldWindow; + reject(e); + } + function jointOps(op, status) { function forward() { if (status.state === 'waiting') { @@ -46,10 +51,12 @@ export function pipeOperationStreams(readable, writable) { function loop() { for (;;) { - if (readable.state === 'aborted' || writable.state === 'cancelled') { - readable.window = oldWindow; - reject(); - + if (readable.state === 'aborted') { + restoreWindowAndReject(new TypeError('readable is aborted')); + return; + } + if (writable.state === 'cancelled') { + restoreWindowAndReject(new TypeError('writable is cancelled')); return; } @@ -58,13 +65,15 @@ export function pipeOperationStreams(readable, writable) { const op = readable.read(); if (op.type === 'data') { jointOps(op, writable.write(op.argument)); - } else { - // Assert: op.type === 'close'. + } else if (op.type === 'close') { jointOps(op, writable.close()); readable.window = oldWindow; resolve(); + return; + } else { + restoreWindowAndReject(new TypeError('unexpected operation type: ' + op.type)); return; } @@ -76,10 +85,7 @@ export function pipeOperationStreams(readable, writable) { selectOperationStreams(readable, writable) .then(loop) - .catch(e => { - readable.window = oldWindow; - reject(); - }); + .catch(restoreWindowAndReject); return; } } From ed46e5c309bc397f9b51a25592eca92ab9e9c753 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Wed, 25 Feb 2015 17:09:02 +0900 Subject: [PATCH 41/93] Move pipeOperationStreams to ReadableOperationStream and rename to pipeTo --- .../lib/experimental/operation-stream.js | 133 +++++++++--------- .../test/experimental/operation-stream.js | 7 +- 2 files changed, 70 insertions(+), 70 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 2c2ae48be..d0751a04e 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -27,72 +27,6 @@ export function selectOperationStreams(readable, writable) { return Promise.race(promises); } -export function pipeOperationStreams(readable, writable) { - return new Promise((resolve, reject) => { - const oldWindow = readable.window; - - function restoreWindowAndReject(e) { - readable.window = oldWindow; - reject(e); - } - - function jointOps(op, status) { - function forward() { - if (status.state === 'waiting') { - status.ready.then(forward); - } else if (status.state === 'errored') { - op.error(status.result); - } else { - op.complete(status.result); - } - } - forward(); - } - - function loop() { - for (;;) { - if (readable.state === 'aborted') { - restoreWindowAndReject(new TypeError('readable is aborted')); - return; - } - if (writable.state === 'cancelled') { - restoreWindowAndReject(new TypeError('writable is cancelled')); - return; - } - - if (writable.state === 'writable') { - if (readable.state === 'readable') { - const op = readable.read(); - if (op.type === 'data') { - jointOps(op, writable.write(op.argument)); - } else if (op.type === 'close') { - jointOps(op, writable.close()); - - readable.window = oldWindow; - resolve(); - - return; - } else { - restoreWindowAndReject(new TypeError('unexpected operation type: ' + op.type)); - return; - } - - continue; - } else { - readable.window = writable.space; - } - } - - selectOperationStreams(readable, writable) - .then(loop) - .catch(restoreWindowAndReject); - return; - } - } - loop(); - }); -} - class OperationStatus { constructor() { this._state = 'waiting'; @@ -515,6 +449,73 @@ class ReadableOperationStream { read() { return this._stream.read(); } + pipeTo(dest) { + return new Promise((resolve, reject) => { + const source = this; + + const oldWindow = source.window; + + function restoreWindowAndReject(e) { + source.window = oldWindow; + reject(e); + } + + function jointOps(op, status) { + function forward() { + if (status.state === 'waiting') { + status.ready.then(forward); + } else if (status.state === 'errored') { + op.error(status.result); + } else { + op.complete(status.result); + } + } + forward(); + } + + function loop() { + for (;;) { + if (source.state === 'aborted') { + restoreWindowAndReject(new TypeError('aborted')); + return; + } + if (dest.state === 'cancelled') { + restoreWindowAndReject(new TypeError('dest is cancelled')); + return; + } + + if (dest.state === 'writable') { + if (source.state === 'readable') { + const op = source.read(); + if (op.type === 'data') { + jointOps(op, dest.write(op.argument)); + } else if (op.type === 'close') { + jointOps(op, dest.close()); + + source.window = oldWindow; + resolve(); + + return; + } else { + restoreWindowAndReject(new TypeError('unexpected operation type: ' + op.type)); + return; + } + + continue; + } else { + source.window = dest.space; + } + } + + selectOperationStreams(source, dest) + .then(loop) + .catch(restoreWindowAndReject); + return; + } + } + loop(); + }); + } cancel(reason) { return this._stream.cancel(reason); } diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index e51e4cd40..f5551c50f 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -191,7 +191,7 @@ test('Pipe', t => { wos0.close(); }); - pipeOperationStreams(ros0, wos1) + ros0.pipeTo(wos1) .catch(e => { t.fail(e); t.end(); @@ -802,8 +802,7 @@ test('Piping from a source with a buffer pool to a buffer taking sink', t => { bufferConsumingStream.window = 64; // pipeOperationStreams automatically adjusts window of the readable side. - const pipePromise = pipeOperationStreams( - file.createBufferProducingStreamWithPool(pool), bufferConsumingStream) + const pipePromise = file.createBufferProducingStreamWithPool(pool).pipeTo(bufferConsumingStream); pipePromise.catch(e => { t.fail(e); t.end(); @@ -865,7 +864,7 @@ test('Piping from a buffer taking source to a sink with buffer', t => { const sink = new BytesSetToOneExpectingByteSink(); const writableBufferProducingStream = sink.createBufferProducingStream(); - const pipePromise = pipeOperationStreams(writableBufferProducingStream, bufferFillingStream); + const pipePromise = writableBufferProducingStream.pipeTo(bufferFillingStream); pipePromise.catch(e => { t.fail(e); t.end(); From 41a96d6d0254273a701b23a41e6e6252dee54a4e Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Wed, 25 Feb 2015 17:10:10 +0900 Subject: [PATCH 42/93] Small fixes --- .../test/experimental/operation-stream.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index f5551c50f..8e9096bcb 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -152,15 +152,15 @@ test('Asynchronous write, read and completion of the operation', t => { t.equals(wos.state, 'waiting'); t.equals(wos.space, 0); - ros.window = 10 + ros.window = 10; t.equals(wos.state, 'waiting'); t.equals(wos.space, 0); - ros.window = 15 + ros.window = 15; t.equals(wos.state, 'writable'); t.equals(wos.space, 5); - ros.window = 20 + ros.window = 20; t.equals(wos.state, 'writable'); t.equals(wos.space, 10); @@ -801,7 +801,7 @@ test('Piping from a source with a buffer pool to a buffer taking sink', t => { const bufferConsumingStream = sink.createBufferConsumingStream(); bufferConsumingStream.window = 64; - // pipeOperationStreams automatically adjusts window of the readable side. + // pipeTo automatically adjusts window of the readable side. const pipePromise = file.createBufferProducingStreamWithPool(pool).pipeTo(bufferConsumingStream); pipePromise.catch(e => { t.fail(e); From ce08395004ce8b0073365e1890156da1c87c45db Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Thu, 26 Feb 2015 01:16:04 +0900 Subject: [PATCH 43/93] Revert the change to move pipeOperationStreams to pipeTo and fix some issues --- .../lib/experimental/operation-stream.js | 152 ++++++++++-------- reference-implementation/run-tests.js | 3 +- .../test/experimental/operation-stream.js | 100 ++++++------ 3 files changed, 142 insertions(+), 113 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index d0751a04e..110bea345 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -6,6 +6,19 @@ export function createOperationStream(strategy) { }; } +export function jointOps(op, status) { + function forward() { + if (status.state === 'waiting') { + status.ready.then(forward); + } else if (status.state === 'errored') { + op.error(status.result); + } else { + op.complete(status.result); + } + } + forward(); +} + export function selectOperationStreams(readable, writable) { const promises = []; @@ -27,6 +40,78 @@ export function selectOperationStreams(readable, writable) { return Promise.race(promises); } +// Pipes data from source to dest with no transformation. Abort signal, cancel signal and space are also propagated +// between source and dest. +export function pipeOperationStreams(source, dest) { + return new Promise((resolve, reject) => { + const oldWindow = source.window; + + function restoreWindowAndReject(e) { + source.window = oldWindow; + reject(e); + } + + function disposeStreams(error) { + if (dest.state !== 'cancelled') { + dest.cancel(error); + } + if (source.state !== 'aborted') { + source.abort(error); + } + restoreWindowAndReject(error); + } + + function loop() { + for (;;) { + if (source.state === 'aborted') { + if (dest.state !== 'cancelled') { + jointOps(source.abortOperation, dest.cancel(source.abortOperation.argument)); + } + restoreWindowAndReject(new TypeError('aborted')); + return; + } + if (dest.state === 'cancelled') { + if (source.state !== 'aborted') { + jointOps(dest.cancelOperation, source.abort(dest.cancelOperation.argument)); + } + restoreWindowAndReject(new TypeError('dest is cancelled')); + return; + } + + if (dest.state === 'writable') { + if (source.state === 'readable') { + const op = source.read(); + if (op.type === 'data') { + jointOps(op, dest.write(op.argument)); + } else if (op.type === 'close') { + jointOps(op, dest.close()); + + source.window = oldWindow; + resolve(); + + return; + } else { + const error = new TypeError('unexpected operation type: ' + op.type); + disposeStreams(error); + return; + } + + continue; + } else { + source.window = dest.space; + } + } + + selectOperationStreams(source, dest) + .then(loop) + .catch(disposeStreams); + return; + } + } + loop(); + }); +} + class OperationStatus { constructor() { this._state = 'waiting'; @@ -449,73 +534,6 @@ class ReadableOperationStream { read() { return this._stream.read(); } - pipeTo(dest) { - return new Promise((resolve, reject) => { - const source = this; - - const oldWindow = source.window; - - function restoreWindowAndReject(e) { - source.window = oldWindow; - reject(e); - } - - function jointOps(op, status) { - function forward() { - if (status.state === 'waiting') { - status.ready.then(forward); - } else if (status.state === 'errored') { - op.error(status.result); - } else { - op.complete(status.result); - } - } - forward(); - } - - function loop() { - for (;;) { - if (source.state === 'aborted') { - restoreWindowAndReject(new TypeError('aborted')); - return; - } - if (dest.state === 'cancelled') { - restoreWindowAndReject(new TypeError('dest is cancelled')); - return; - } - - if (dest.state === 'writable') { - if (source.state === 'readable') { - const op = source.read(); - if (op.type === 'data') { - jointOps(op, dest.write(op.argument)); - } else if (op.type === 'close') { - jointOps(op, dest.close()); - - source.window = oldWindow; - resolve(); - - return; - } else { - restoreWindowAndReject(new TypeError('unexpected operation type: ' + op.type)); - return; - } - - continue; - } else { - source.window = dest.space; - } - } - - selectOperationStreams(source, dest) - .then(loop) - .catch(restoreWindowAndReject); - return; - } - } - loop(); - }); - } cancel(reason) { return this._stream.cancel(reason); } diff --git a/reference-implementation/run-tests.js b/reference-implementation/run-tests.js index e11a092a8..1f30fbafd 100644 --- a/reference-implementation/run-tests.js +++ b/reference-implementation/run-tests.js @@ -4,7 +4,7 @@ const path = require('path'); import ReadableStream from './lib/readable-stream'; import WritableStream from './lib/writable-stream'; import ReadableByteStream from './lib/experimental/readable-byte-stream'; -import { createOperationStream, pipeOperationStreams, selectOperationStreams +import { createOperationStream, jointOps, pipeOperationStreams, selectOperationStreams } from './lib/experimental/operation-stream'; import ByteLengthQueuingStrategy from './lib/byte-length-queuing-strategy'; import CountQueuingStrategy from './lib/count-queuing-strategy'; @@ -13,6 +13,7 @@ import TransformStream from './lib/transform-stream'; global.ReadableStream = ReadableStream; global.WritableStream = WritableStream; global.createOperationStream = createOperationStream; +global.jointOps = jointOps; global.pipeOperationStreams = pipeOperationStreams; global.selectOperationStreams = selectOperationStreams; global.ReadableByteStream = ReadableByteStream; diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 8e9096bcb..422018607 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -132,7 +132,7 @@ test('Asynchronous write, read and completion of the operation', t => { }, e => { t.fail(e); t.end(); - }) + }); }); test('Asynchronous write, read and completion of the operation', t => { @@ -172,7 +172,7 @@ test('Asynchronous write, read and completion of the operation', t => { t.end(); }); -test('Pipe', t => { +test('pipeOperationStreams', t => { const pair0 = createOperationStream(new AdjustableStringStrategy()); const wos0 = pair0.writable; const ros0 = pair0.readable; @@ -191,7 +191,7 @@ test('Pipe', t => { wos0.close(); }); - ros0.pipeTo(wos1) + pipeOperationStreams(ros0, wos1) .catch(e => { t.fail(e); t.end(); @@ -232,66 +232,63 @@ test('Pipe', t => { }); }); -test('Byte counting transform stream', t => { - const pair0 = createOperationStream(new AdjustableStringStrategy()); - const wos0 = pair0.writable; - const ros0 = pair0.readable; - - const pair1 = createOperationStream(new AdjustableStringStrategy()); - const wos1 = pair1.writable; - const ros1 = pair1.readable; - - wos0.write('hello'); - wos0.write('world'); - wos0.write('goodbye'); - wos0.close(); - - function byteCountingPipe(readable, writable) { +test('Transformation example: Byte counting', t => { + function byteCountingTransform(source, dest) { return new Promise((resolve, reject) => { - const oldWindow = readable.window; - let count = 0; + function disposeStreams(error) { + if (dest.state !== 'cancelled') { + dest.cancel(error); + } + if (source.state !== 'aborted') { + source.abort(error); + } + } + function loop() { for (;;) { - if (readable.state === 'aborted' || writable.state === 'cancelled') { - readable.window = oldWindow; - reject(); - + if (source.state === 'aborted') { + if (dest.state !== 'cancelled') { + jointOps(source.abortOperation, dest.cancel(source.abortOperation.argument)); + } + return; + } + if (dest.state === 'cancelled') { + if (source.state !== 'aborted') { + jointOps(dest.cancelOperation, source.abort(dest.cancelOperation.argument)); + } return; } - if (writable.state === 'writable') { - if (readable.state === 'readable') { - const op = readable.read(); + if (dest.state === 'writable') { + if (source.state === 'readable') { + const op = source.read(); if (op.type === 'data') { count += op.argument.length; op.complete(); - } else { - writable.write(count); - writable.close(); + } else if (op.type === 'close') { + dest.write(count); + dest.close(); op.complete(); - readable.window = oldWindow; - resolve(); - + return; + } else { + disposeStreams(new TypeError('unexpected operation type: ' + op.type)); return; } continue; } else { - if (writable.space > 0) { - readable.window = 1; + if (dest.space > 0) { + source.window = 1; } } } - selectOperationStreams(readable, writable) - .then(loop) - .catch(e => { - readable.window = oldWindow; - reject(); - }); + selectOperationStreams(source, dest) + .then(loop) + .catch(disposeStreams); return; } } @@ -299,12 +296,25 @@ test('Byte counting transform stream', t => { }); } - byteCountingPipe(ros0, wos1) + const pair0 = createOperationStream(new AdjustableStringStrategy()); + const wos0 = pair0.writable; + const ros0 = pair0.readable; + + const pair1 = createOperationStream(new AdjustableStringStrategy()); + const wos1 = pair1.writable; + const ros1 = pair1.readable; + + byteCountingTransform(ros0, wos1) .catch(e => { t.fail(e); t.end(); }); + wos0.write('hello'); + wos0.write('world'); + wos0.write('goodbye'); + wos0.close(); + ros1.window = 1; ros1.ready.then(() => { @@ -801,8 +811,8 @@ test('Piping from a source with a buffer pool to a buffer taking sink', t => { const bufferConsumingStream = sink.createBufferConsumingStream(); bufferConsumingStream.window = 64; - // pipeTo automatically adjusts window of the readable side. - const pipePromise = file.createBufferProducingStreamWithPool(pool).pipeTo(bufferConsumingStream); + // pipeOperationStreams automatically adjusts window of the readable side. + const pipePromise = pipeOperationStreams(file.createBufferProducingStreamWithPool(pool), bufferConsumingStream); pipePromise.catch(e => { t.fail(e); t.end(); @@ -864,7 +874,7 @@ test('Piping from a buffer taking source to a sink with buffer', t => { const sink = new BytesSetToOneExpectingByteSink(); const writableBufferProducingStream = sink.createBufferProducingStream(); - const pipePromise = writableBufferProducingStream.pipeTo(bufferFillingStream); + const pipePromise = pipeOperationStreams(writableBufferProducingStream, bufferFillingStream); pipePromise.catch(e => { t.fail(e); t.end(); From 5cfaa5c4e07f1aa74a2e12346ee8409e730b3ece Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Thu, 26 Feb 2015 02:18:36 +0900 Subject: [PATCH 44/93] Fix abort/cancel signal propagation in pipeOperationStreams and add tests --- .../lib/experimental/operation-stream.js | 6 +- .../test/experimental/operation-stream.js | 165 +++++++++++++++++- 2 files changed, 167 insertions(+), 4 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 110bea345..b4c3c90e3 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -65,14 +65,14 @@ export function pipeOperationStreams(source, dest) { for (;;) { if (source.state === 'aborted') { if (dest.state !== 'cancelled') { - jointOps(source.abortOperation, dest.cancel(source.abortOperation.argument)); + jointOps(source.abortOperation, dest.abort(source.abortOperation.argument)); } restoreWindowAndReject(new TypeError('aborted')); return; } if (dest.state === 'cancelled') { if (source.state !== 'aborted') { - jointOps(dest.cancelOperation, source.abort(dest.cancelOperation.argument)); + jointOps(dest.cancelOperation, source.cancel(dest.cancelOperation.argument)); } restoreWindowAndReject(new TypeError('dest is cancelled')); return; @@ -436,7 +436,7 @@ class OperationStream { for (var i = 0; i < this._queue.length; ++i) { const op = this._queue[i].value; - op.error(new TypeError('cancelled')); + op.error(reason); } this._queue = []; this._strategy = undefined; diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 422018607..55d8d182c 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -172,7 +172,77 @@ test('Asynchronous write, read and completion of the operation', t => { t.end(); }); -test('pipeOperationStreams', t => { +test('abort()', t => { + t.plan(8); + + const pair = createOperationStream(new AdjustableStringStrategy()); + const wos = pair.writable; + const ros = pair.readable; + + const helloStatus = wos.write('hello'); + const worldStatus = wos.write('world'); + + const testError = new TypeError('foo'); + const testCompletion = 'good'; + + const status = wos.abort(testError); + status.ready.then(() => { + t.equals(status.state, 'completed', 'status.state'); + t.equals(status.result, testCompletion, 'status.result'); + }); + + helloStatus.ready.then(() => { + t.equals(helloStatus.state, 'errored', 'helloStatus.state'); + t.equals(helloStatus.result.message, 'aborted', 'helloStatus.result'); + }); + worldStatus.ready.then(() => { + t.equals(worldStatus.state, 'errored', 'worldStatus.state'); + t.equals(worldStatus.result.message, 'aborted', 'worldStatus.result'); + }); + ros.aborted.then(() => { + t.equals(ros.state, 'aborted', 'ros.state'); + t.equals(ros.abortOperation.argument, testError, 'ros.abortOperation.argument'); + + ros.abortOperation.complete(testCompletion); + }); +}); + +test('cancel()', t => { + t.plan(8); + + const pair = createOperationStream(new AdjustableStringStrategy()); + const wos = pair.writable; + const ros = pair.readable; + + const helloStatus = wos.write('hello'); + const worldStatus = wos.write('world'); + + const testError = new TypeError('foo'); + const testCompletion = 'good'; + + const status = ros.cancel(testError); + status.ready.then(() => { + t.equals(status.state, 'completed', 'status.state'); + t.equals(status.result, testCompletion, 'status.result'); + }); + + helloStatus.ready.then(() => { + t.equals(helloStatus.state, 'errored', 'helloStatus.state'); + t.equals(helloStatus.result, testError, 'helloStatus.result'); + }); + worldStatus.ready.then(() => { + t.equals(worldStatus.state, 'errored', 'worldStatus.state'); + t.equals(worldStatus.result, testError, 'worldStatus.result'); + }); + wos.cancelled.then(() => { + t.equals(wos.state, 'cancelled', 'wos.state'); + t.equals(wos.cancelOperation.argument, testError, 'wos.cancelOperation.argument'); + + wos.cancelOperation.complete(testCompletion); + }); +}); + +test('pipeOperationStreams()', t => { const pair0 = createOperationStream(new AdjustableStringStrategy()); const wos0 = pair0.writable; const ros0 = pair0.readable; @@ -183,10 +253,13 @@ test('pipeOperationStreams', t => { var helloStatus; t.equals(wos0.state, 'waiting'); + // Check that wos0 becomes writable. wos0.ready.then(() => { t.equals(wos0.state, 'writable'); helloStatus = wos0.write('hello'); + + // Just write without state check. wos0.write('world'); wos0.close(); }); @@ -232,6 +305,96 @@ test('pipeOperationStreams', t => { }); }); +test('pipeOperationStreams(): abort() propagation', t => { + t.plan(9); + + const pair0 = createOperationStream(new AdjustableStringStrategy()); + const wos0 = pair0.writable; + const ros0 = pair0.readable; + + const pair1 = createOperationStream(new AdjustableStringStrategy()); + const wos1 = pair1.writable; + const ros1 = pair1.readable; + + const helloStatus = wos0.write('hello'); + const worldStatus = wos0.write('world'); + + const testError = new TypeError('foo'); + const testCompletion = 'good'; + + const pipePromise = pipeOperationStreams(ros0, wos1); + pipePromise + .then( + v => t.fail('pipePromise is fulfilled with ' + v), + e => t.equals(e.message, 'aborted', 'rejection reason of pipePromise')); + + const status = wos0.abort(testError); + status.ready.then(() => { + t.equals(status.state, 'completed', 'status.state'); + t.equals(status.result, testCompletion, 'status.result'); + }); + + helloStatus.ready.then(() => { + t.equals(helloStatus.state, 'errored', 'helloStatus.state'); + t.equals(helloStatus.result.message, 'aborted', 'helloStatus.result'); + }); + worldStatus.ready.then(() => { + t.equals(worldStatus.state, 'errored', 'worldStatus.state'); + t.equals(worldStatus.result.message, 'aborted', 'worldStatus.result'); + }); + ros1.aborted.then(() => { + t.equals(ros1.state, 'aborted', 'ros.state'); + t.equals(ros1.abortOperation.argument, testError, 'ros1.abortOperation.argument'); + + ros1.abortOperation.complete(testCompletion); + }); +}); + +test('pipeOperationStreams(): cancel() propagation', t => { + t.plan(9); + + const pair0 = createOperationStream(new AdjustableStringStrategy()); + const wos0 = pair0.writable; + const ros0 = pair0.readable; + + const pair1 = createOperationStream(new AdjustableStringStrategy()); + const wos1 = pair1.writable; + const ros1 = pair1.readable; + + const helloStatus = wos0.write('hello'); + const worldStatus = wos0.write('world'); + + const testError = new TypeError('foo'); + const testCompletion = 'good'; + + const pipePromise = pipeOperationStreams(ros0, wos1); + pipePromise + .then( + v => t.fail('pipePromise is fulfilled with ' + v), + e => t.equals(e.message, 'dest is cancelled', 'rejection reason of pipePromise')); + + const status = ros1.cancel(testError); + status.ready.then(() => { + t.equals(status.state, 'completed', 'status.state'); + t.equals(status.result, testCompletion, 'status.result'); + }); + + helloStatus.ready.then(() => { + t.equals(helloStatus.state, 'errored', 'helloStatus.state'); + t.equals(helloStatus.result, testError, 'helloStatus.result'); + }); + worldStatus.ready.then(() => { + t.equals(worldStatus.state, 'errored', 'worldStatus.state'); + t.equals(worldStatus.result, testError, 'worldStatus.result'); + }); + wos0.cancelled.then(() => { + t.equals(wos0.state, 'cancelled', 'wos0.state'); + t.equals(wos0.cancelOperation.argument, testError, 'wos0.cancelOperation.argument'); + + wos0.cancelOperation.complete(testCompletion); + }); +}); + test('Transformation example: Byte counting', t => { function byteCountingTransform(source, dest) { return new Promise((resolve, reject) => { From 25d507689079efbcfe650376e1aa2877b9a55895 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Thu, 26 Feb 2015 02:24:04 +0900 Subject: [PATCH 45/93] Add a comment to selectOperationStreams and remove stale asserts --- reference-implementation/lib/experimental/operation-stream.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index b4c3c90e3..11e85cf71 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -19,20 +19,19 @@ export function jointOps(op, status) { forward(); } +// Exported as a helper for building transformation. export function selectOperationStreams(readable, writable) { const promises = []; if (readable.state === 'readable') { promises.push(readable.aborted); } else { - // Assert: readable.state === 'readable'. promises.push(readable.ready); } if (writable.state === 'writable') { promises.push(writable.cancelled); } else { - // Assert: writable.state === 'writable'. promises.push(writable.ready); promises.push(writable.waitSpaceChange()); } From 6d2f0518913f64667ace99988d03402c8edcb5f0 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Thu, 26 Feb 2015 02:26:31 +0900 Subject: [PATCH 46/93] Make jointOps no-op for unknown state --- reference-implementation/lib/experimental/operation-stream.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 11e85cf71..823c6eec5 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -12,7 +12,7 @@ export function jointOps(op, status) { status.ready.then(forward); } else if (status.state === 'errored') { op.error(status.result); - } else { + } else if (status.state === 'completed') { op.complete(status.result); } } From 913ade9d2c581d9a0dae10c76e6f91a3717b7961 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Thu, 26 Feb 2015 02:34:17 +0900 Subject: [PATCH 47/93] Check state on abort/cancel --- .../test/experimental/operation-stream.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 55d8d182c..bba01f036 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -173,7 +173,7 @@ test('Asynchronous write, read and completion of the operation', t => { }); test('abort()', t => { - t.plan(8); + t.plan(9); const pair = createOperationStream(new AdjustableStringStrategy()); const wos = pair.writable; @@ -186,6 +186,7 @@ test('abort()', t => { const testCompletion = 'good'; const status = wos.abort(testError); + t.equals(wos.state, 'aborted', 'wos.state'); status.ready.then(() => { t.equals(status.state, 'completed', 'status.state'); t.equals(status.result, testCompletion, 'status.result'); @@ -208,7 +209,7 @@ test('abort()', t => { }); test('cancel()', t => { - t.plan(8); + t.plan(9); const pair = createOperationStream(new AdjustableStringStrategy()); const wos = pair.writable; @@ -221,6 +222,7 @@ test('cancel()', t => { const testCompletion = 'good'; const status = ros.cancel(testError); + t.equals(ros.state, 'cancelled', 'ros.state'); status.ready.then(() => { t.equals(status.state, 'completed', 'status.state'); t.equals(status.result, testCompletion, 'status.result'); From 5c0b6cf1bd3819aeb485e2e3636b588dd972e830 Mon Sep 17 00:00:00 2001 From: rieszgithub Date: Fri, 27 Feb 2015 00:58:02 +0900 Subject: [PATCH 48/93] Rename OperationStream to OperationQueue since Stream is not always mean something queue-backed --- .../lib/experimental/operation-stream.js | 6 ++-- reference-implementation/run-tests.js | 4 +-- .../test/experimental/operation-stream.js | 36 +++++++++---------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 823c6eec5..39adb8045 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -1,5 +1,5 @@ -export function createOperationStream(strategy) { - const stream = new OperationStream(strategy); +export function createOperationQueue(strategy) { + const stream = new OperationQueue(strategy); return { writable: new WritableOperationStream(stream), readable: new ReadableOperationStream(stream) @@ -166,7 +166,7 @@ class Operation { } } -class OperationStream { +class OperationQueue { constructor(strategy) { this._queue = []; this._queueSize = 0; diff --git a/reference-implementation/run-tests.js b/reference-implementation/run-tests.js index 1f30fbafd..a9ca8bccc 100644 --- a/reference-implementation/run-tests.js +++ b/reference-implementation/run-tests.js @@ -4,7 +4,7 @@ const path = require('path'); import ReadableStream from './lib/readable-stream'; import WritableStream from './lib/writable-stream'; import ReadableByteStream from './lib/experimental/readable-byte-stream'; -import { createOperationStream, jointOps, pipeOperationStreams, selectOperationStreams +import { createOperationQueue, jointOps, pipeOperationStreams, selectOperationStreams } from './lib/experimental/operation-stream'; import ByteLengthQueuingStrategy from './lib/byte-length-queuing-strategy'; import CountQueuingStrategy from './lib/count-queuing-strategy'; @@ -12,7 +12,7 @@ import TransformStream from './lib/transform-stream'; global.ReadableStream = ReadableStream; global.WritableStream = WritableStream; -global.createOperationStream = createOperationStream; +global.createOperationQueue = createOperationQueue; global.jointOps = jointOps; global.pipeOperationStreams = pipeOperationStreams; global.selectOperationStreams = selectOperationStreams; diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index bba01f036..02f328da1 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -1,7 +1,7 @@ const test = require('tape-catch'); test('Operation stream pair is constructed', t => { - const pair = createOperationStream({ + const pair = createOperationQueue({ size() { return 1; }, @@ -61,7 +61,7 @@ class AdjustableStringStrategy { } test('Synchronous write, read and completion of the operation', t => { - const pair = createOperationStream(new ApplyBackpressureWhenNonEmptyStrategy()); + const pair = createOperationQueue(new ApplyBackpressureWhenNonEmptyStrategy()); const wos = pair.writable; const ros = pair.readable; @@ -92,7 +92,7 @@ test('Synchronous write, read and completion of the operation', t => { }); test('Asynchronous write, read and completion of the operation', t => { - const pair = createOperationStream(new ApplyBackpressureWhenNonEmptyStrategy()); + const pair = createOperationQueue(new ApplyBackpressureWhenNonEmptyStrategy()); const wos = pair.writable; const ros = pair.readable; @@ -136,7 +136,7 @@ test('Asynchronous write, read and completion of the operation', t => { }); test('Asynchronous write, read and completion of the operation', t => { - const pair = createOperationStream(new AdjustableArrayBufferStrategy()); + const pair = createOperationQueue(new AdjustableArrayBufferStrategy()); const wos = pair.writable; const ros = pair.readable; @@ -175,7 +175,7 @@ test('Asynchronous write, read and completion of the operation', t => { test('abort()', t => { t.plan(9); - const pair = createOperationStream(new AdjustableStringStrategy()); + const pair = createOperationQueue(new AdjustableStringStrategy()); const wos = pair.writable; const ros = pair.readable; @@ -211,7 +211,7 @@ test('abort()', t => { test('cancel()', t => { t.plan(9); - const pair = createOperationStream(new AdjustableStringStrategy()); + const pair = createOperationQueue(new AdjustableStringStrategy()); const wos = pair.writable; const ros = pair.readable; @@ -245,11 +245,11 @@ test('cancel()', t => { }); test('pipeOperationStreams()', t => { - const pair0 = createOperationStream(new AdjustableStringStrategy()); + const pair0 = createOperationQueue(new AdjustableStringStrategy()); const wos0 = pair0.writable; const ros0 = pair0.readable; - const pair1 = createOperationStream(new AdjustableStringStrategy()); + const pair1 = createOperationQueue(new AdjustableStringStrategy()); const wos1 = pair1.writable; const ros1 = pair1.readable; @@ -310,11 +310,11 @@ test('pipeOperationStreams()', t => { test('pipeOperationStreams(): abort() propagation', t => { t.plan(9); - const pair0 = createOperationStream(new AdjustableStringStrategy()); + const pair0 = createOperationQueue(new AdjustableStringStrategy()); const wos0 = pair0.writable; const ros0 = pair0.readable; - const pair1 = createOperationStream(new AdjustableStringStrategy()); + const pair1 = createOperationQueue(new AdjustableStringStrategy()); const wos1 = pair1.writable; const ros1 = pair1.readable; @@ -355,11 +355,11 @@ test('pipeOperationStreams(): abort() propagation', t => { test('pipeOperationStreams(): cancel() propagation', t => { t.plan(9); - const pair0 = createOperationStream(new AdjustableStringStrategy()); + const pair0 = createOperationQueue(new AdjustableStringStrategy()); const wos0 = pair0.writable; const ros0 = pair0.readable; - const pair1 = createOperationStream(new AdjustableStringStrategy()); + const pair1 = createOperationQueue(new AdjustableStringStrategy()); const wos1 = pair1.writable; const ros1 = pair1.readable; @@ -461,11 +461,11 @@ test('Transformation example: Byte counting', t => { }); } - const pair0 = createOperationStream(new AdjustableStringStrategy()); + const pair0 = createOperationQueue(new AdjustableStringStrategy()); const wos0 = pair0.writable; const ros0 = pair0.readable; - const pair1 = createOperationStream(new AdjustableStringStrategy()); + const pair1 = createOperationQueue(new AdjustableStringStrategy()); const wos1 = pair1.writable; const ros1 = pair1.readable; @@ -607,7 +607,7 @@ class FakeFileBackedByteSource { } } - const pair = createOperationStream(new AdjustableArrayBufferStrategy()); + const pair = createOperationQueue(new AdjustableArrayBufferStrategy()); new Puller(this, buffers, pair.writable); return pair.readable; } @@ -742,7 +742,7 @@ class FakeFileBackedByteSource { } } - const pair = createOperationStream(new AdjustableArrayBufferStrategy()); + const pair = createOperationQueue(new AdjustableArrayBufferStrategy()); new Filler(this, pair.readable); return pair.writable; } @@ -947,13 +947,13 @@ class BytesSetToOneExpectingByteSink { } } - const pair = createOperationStream(new AdjustableArrayBufferStrategy()); + const pair = createOperationQueue(new AdjustableArrayBufferStrategy()); new BufferProvidingWriter(this, pair.writable); return pair.readable; } createBufferConsumingStream() { - const pair = createOperationStream(new AdjustableArrayBufferStrategy()); + const pair = createOperationQueue(new AdjustableArrayBufferStrategy()); new BytesSetToOneExpectingByteSinkInternalWriter(this, pair.readable); return pair.writable; } From a7c50c655355294751ee81442bde6b92ae9bc505 Mon Sep 17 00:00:00 2001 From: rieszgithub Date: Fri, 27 Feb 2015 01:03:46 +0900 Subject: [PATCH 49/93] Rename wrappers for OperationQueue so that they don't sound like general interfaces --- .../lib/experimental/operation-stream.js | 14 +++++++------- .../test/experimental/operation-stream.js | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 39adb8045..9778617fa 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -1,8 +1,8 @@ export function createOperationQueue(strategy) { - const stream = new OperationQueue(strategy); + const queue = new OperationQueue(strategy); return { - writable: new WritableOperationStream(stream), - readable: new ReadableOperationStream(stream) + writable: new OperationQueueWritableSide(queue), + readable: new OperationQueueReadableSide(queue) }; } @@ -458,9 +458,8 @@ class OperationQueue { } } -// Wrappers to hide the interfaces of the other side. - -class WritableOperationStream { +// A wrapper to expose only the interfaces of writable side implementing the WritableOperationStream interface. +class OperationQueueWritableSide { constructor(stream) { this._stream = stream; } @@ -504,7 +503,8 @@ class WritableOperationStream { } } -class ReadableOperationStream { +// A wrapper to expose only the interfaces of readable side implementing the ReadableOperationStream interface. +class OperationQueueReadableSide { constructor(stream) { this._stream = stream; } diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 02f328da1..e168670a1 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -999,7 +999,7 @@ test('Piping from a source with a buffer pool to a buffer taking sink', t => { }); }); -test('Consuming bytes from a source with a buffer pool via the ReadableStream interface', t => { +test('Consuming bytes from a source with a buffer pool via the ReadableOperationStream interface', t => { const pool = []; for (var i = 0; i < 10; ++i) { pool.push(new ArrayBuffer(10)); From 337588f2b7db0dacc8626adfebbc0825486590ac Mon Sep 17 00:00:00 2001 From: rieszgithub Date: Fri, 27 Feb 2015 01:06:10 +0900 Subject: [PATCH 50/93] Explain that createOperationQueue is for creating queue-backed streams --- reference-implementation/lib/experimental/operation-stream.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 9778617fa..0db09f304 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -1,3 +1,5 @@ +// Creates a pair of WritableOperationStream implementation and ReadableOperationStream implementation that are +// connected with a queue. This can be used for creating queue-backed operation streams. export function createOperationQueue(strategy) { const queue = new OperationQueue(strategy); return { From 8394449c0d5fa9007ab760db48616f30bee409b7 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Fri, 27 Feb 2015 15:26:41 +0900 Subject: [PATCH 51/93] Rename read() to readOperation() and add a syntax sugar read() --- .../lib/experimental/operation-stream.js | 12 +++++++++- .../test/experimental/operation-stream.js | 23 +++++++++---------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 0db09f304..c4f482cde 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -81,7 +81,7 @@ export function pipeOperationStreams(source, dest) { if (dest.state === 'writable') { if (source.state === 'readable') { - const op = source.read(); + const op = source.readOperation(); if (op.type === 'data') { jointOps(op, dest.write(op.argument)); } else if (op.type === 'close') { @@ -533,9 +533,19 @@ class OperationQueueReadableSide { } read() { + const op = this._stream.read(); + if (op.type === 'data') { + return op.argument; + } else if (op.type === 'close') { + return ReadableOperationStream.EOS; + } + } + readOperation() { return this._stream.read(); } cancel(reason) { return this._stream.cancel(reason); } } + +ReadableOperationStream.EOS = {}; diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index e168670a1..46a106253 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -75,7 +75,7 @@ test('Synchronous write, read and completion of the operation', t => { t.equals(status.state, 'waiting'); - const op = ros.read(); + const op = ros.readOperation(); t.equals(op.argument, 'hello'); t.equals(ros.state, 'waiting'); @@ -103,7 +103,7 @@ test('Asynchronous write, read and completion of the operation', t => { ros.ready.then(() => { t.equals(ros.state, 'readable'); - const op = ros.read(); + const op = ros.readOperation(); t.equals(op.argument, 'hello'); t.equals(ros.state, 'waiting'); @@ -280,19 +280,19 @@ test('pipeOperationStreams()', t => { ros1.ready.then(() => { t.equals(ros1.state, 'readable'); - const op0 = ros1.read(); + const op0 = ros1.readOperation(); t.equals(op0.type, 'data'); t.equals(op0.argument, 'hello'); op0.complete('hi'); t.equals(ros1.state, 'readable'); - const op1 = ros1.read(); + const op1 = ros1.readOperation(); t.equals(op1.type, 'data'); t.equals(op1.argument, 'world'); t.equals(ros1.state, 'readable'); - const op2 = ros1.read(); + const op2 = ros1.readOperation(); t.equals(op2.type, 'close'); t.equals(helloStatus.state, 'waiting'); @@ -428,7 +428,7 @@ test('Transformation example: Byte counting', t => { if (dest.state === 'writable') { if (source.state === 'readable') { - const op = source.read(); + const op = source.readOperation(); if (op.type === 'data') { count += op.argument.length; op.complete(); @@ -483,9 +483,8 @@ test('Transformation example: Byte counting', t => { ros1.window = 1; ros1.ready.then(() => { - const op = ros1.read(); - t.equals(op.type, 'data'); - t.equals(op.argument, 17); + const v = ros1.read(); + t.equals(v, 17); t.end(); }).catch(e => { t.fail(e); @@ -647,7 +646,7 @@ class FakeFileBackedByteSource { this._currentRequest.complete(this._fileReadStatus.result); this._currentRequest = undefined; - this._fileReadStatus = undefined + this._fileReadStatus = undefined; } _handleFileReadError() { @@ -672,7 +671,7 @@ class FakeFileBackedByteSource { } _readFromFile() { - const op = this._readableStream.read(); + const op = this._readableStream.readOperation(); if (op.type === 'close') { return; @@ -785,7 +784,7 @@ class BytesSetToOneExpectingByteSinkInternalWriter { for (;;) { if (rs.state === 'readable') { - const op = rs.read(); + const op = rs.readOperation(); if (op.type === 'data') { const view = op.argument; From adfd15ca23812986c22a608c31bcd7343f884995 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Fri, 27 Feb 2015 15:34:20 +0900 Subject: [PATCH 52/93] Rename cancelled to errored --- .../lib/experimental/operation-stream.js | 19 +++++++++++-------- .../test/experimental/operation-stream.js | 8 ++++---- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index c4f482cde..584825363 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -32,7 +32,7 @@ export function selectOperationStreams(readable, writable) { } if (writable.state === 'writable') { - promises.push(writable.cancelled); + promises.push(writable.errored); } else { promises.push(writable.ready); promises.push(writable.waitSpaceChange()); @@ -186,8 +186,8 @@ class OperationQueue { this._updateWritableState(); this._cancelOperation = undefined; - this._cancelledPromise = new Promise((resolve, reject) => { - this._resolveCancelledPromise = resolve; + this._erroredPromise = new Promise((resolve, reject) => { + this._resolveErroredPromise = resolve; }); this._readableState = 'waiting'; @@ -223,8 +223,8 @@ class OperationQueue { get cancelOperation() { return this._cancelOperation; } - get cancelled() { - return this._cancelledPromise; + get errored() { + return this._erroredPromise; } get space() { @@ -444,7 +444,7 @@ class OperationQueue { const status = new OperationStatus(); this._cancelOperation = new Operation('cancel', reason, status); - this._resolveCancelledPromise(); + this._resolveErroredPromise(); if (this._writableState === 'waiting') { this._resolveWritableReadyPromise(); @@ -476,8 +476,8 @@ class OperationQueueWritableSide { get cancelOperation() { return this._stream.cancelOperation; } - get cancelled() { - return this._stream.cancelled; + get errored() { + return this._stream.errored; } get window() { @@ -548,4 +548,7 @@ class OperationQueueReadableSide { } } +class ReadableOperationStream { +} + ReadableOperationStream.EOS = {}; diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 46a106253..a035d4854 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -236,7 +236,7 @@ test('cancel()', t => { t.equals(worldStatus.state, 'errored', 'worldStatus.state'); t.equals(worldStatus.result, testError, 'worldStatus.result'); }); - wos.cancelled.then(() => { + wos.errored.then(() => { t.equals(wos.state, 'cancelled', 'wos.state'); t.equals(wos.cancelOperation.argument, testError, 'wos.cancelOperation.argument'); @@ -389,7 +389,7 @@ test('pipeOperationStreams(): cancel() propagation', t => { t.equals(worldStatus.state, 'errored', 'worldStatus.state'); t.equals(worldStatus.result, testError, 'worldStatus.result'); }); - wos0.cancelled.then(() => { + wos0.errored.then(() => { t.equals(wos0.state, 'cancelled', 'wos0.state'); t.equals(wos0.cancelOperation.argument, testError, 'wos0.cancelOperation.argument'); @@ -536,7 +536,7 @@ class FakeFileBackedByteSource { const ws = this._writableStream; if (ws.state === 'writable') { - promises.push(ws.cancelled); + promises.push(ws.errored); } else if (ws.state === 'waiting') { promises.push(ws.ready); } @@ -884,7 +884,7 @@ class BytesSetToOneExpectingByteSink { const ws = this._writableStream; if (ws.state === 'writable') { - promises.push(ws.cancelled); + promises.push(ws.errored); } else if (ws.state === 'waiting') { promises.push(ws.ready); } From ddd60a5333f7bd427230a601405485c0ae665fae Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Fri, 27 Feb 2015 15:38:30 +0900 Subject: [PATCH 53/93] Rename aborted to errored --- .../lib/experimental/operation-stream.js | 37 ++++++++++--------- .../test/experimental/operation-stream.js | 6 +-- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 584825363..8bdc5d8ad 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -26,7 +26,7 @@ export function selectOperationStreams(readable, writable) { const promises = []; if (readable.state === 'readable') { - promises.push(readable.aborted); + promises.push(readable.errored); } else { promises.push(readable.ready); } @@ -183,20 +183,19 @@ class OperationQueue { this._writableState = 'waiting'; this._initWritableReadyPromise(); - this._updateWritableState(); - - this._cancelOperation = undefined; this._erroredPromise = new Promise((resolve, reject) => { this._resolveErroredPromise = resolve; }); + this._updateWritableState(); + + this._cancelOperation = undefined; + + this._abortOperation = undefined; + this._readableState = 'waiting'; this._initReadableReadyPromise(); - this._abortOperation = undefined; - this._abortedPromise = new Promise((resolve, reject) => { - this._resolveAbortedPromise = resolve; - }); } _initWritableReadyPromise() { @@ -211,6 +210,12 @@ class OperationQueue { }); } + // Common interfaces. + + get errored() { + return this._erroredPromise; + } + // Writable side interfaces get writableState() { @@ -223,9 +228,6 @@ class OperationQueue { get cancelOperation() { return this._cancelOperation; } - get errored() { - return this._erroredPromise; - } get space() { if (this._writableState === 'closed' || @@ -342,6 +344,8 @@ class OperationQueue { this._queue = []; this._strategy = undefined; + this._resolveErroredPromise(); + if (this._writableState === 'waiting') { this._resolveWritableReadyPromise(); } @@ -349,7 +353,6 @@ class OperationQueue { const status = new OperationStatus(); this._abortOperation = new Operation('abort', reason, status); - this._resolveAbortedPromise(); if (this._readableState === 'waiting') { this._resolveReadableReadyPromise(); @@ -371,9 +374,6 @@ class OperationQueue { get abortOperation() { return this._abortOperation; } - get aborted() { - return this._abortedPromise; - } get window() { return this._window; @@ -442,9 +442,10 @@ class OperationQueue { this._queue = []; this._strategy = undefined; + this._resolveErroredPromise(); + const status = new OperationStatus(); this._cancelOperation = new Operation('cancel', reason, status); - this._resolveErroredPromise(); if (this._writableState === 'waiting') { this._resolveWritableReadyPromise(); @@ -521,8 +522,8 @@ class OperationQueueReadableSide { get abortOperation() { return this._stream.abortOperation; } - get aborted() { - return this._stream.aborted; + get errored() { + return this._stream.errored; } get window() { diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index a035d4854..2225cb2cc 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -200,7 +200,7 @@ test('abort()', t => { t.equals(worldStatus.state, 'errored', 'worldStatus.state'); t.equals(worldStatus.result.message, 'aborted', 'worldStatus.result'); }); - ros.aborted.then(() => { + ros.errored.then(() => { t.equals(ros.state, 'aborted', 'ros.state'); t.equals(ros.abortOperation.argument, testError, 'ros.abortOperation.argument'); @@ -344,7 +344,7 @@ test('pipeOperationStreams(): abort() propagation', t => { t.equals(worldStatus.state, 'errored', 'worldStatus.state'); t.equals(worldStatus.result.message, 'aborted', 'worldStatus.result'); }); - ros1.aborted.then(() => { + ros1.errored.then(() => { t.equals(ros1.state, 'aborted', 'ros.state'); t.equals(ros1.abortOperation.argument, testError, 'ros1.abortOperation.argument'); @@ -695,7 +695,7 @@ class FakeFileBackedByteSource { const rs = this._readableStream; if (rs.state === 'readable') { - promises.push(rs.aborted); + promises.push(rs.errored); } else if (rs.state === 'waiting') { promises.push(rs.ready); } From 6e3f9a1870af46fc98ef7f34e27b0125722cdc3f Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Fri, 27 Feb 2015 15:47:14 +0900 Subject: [PATCH 54/93] Change semantics of ready and rename to readable and writable --- .../lib/experimental/operation-stream.js | 54 +++++++++---------- .../test/experimental/operation-stream.js | 16 +++--- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 8bdc5d8ad..f2a7ecf05 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -28,13 +28,13 @@ export function selectOperationStreams(readable, writable) { if (readable.state === 'readable') { promises.push(readable.errored); } else { - promises.push(readable.ready); + promises.push(readable.readable); } if (writable.state === 'writable') { promises.push(writable.errored); } else { - promises.push(writable.ready); + promises.push(writable.writable); promises.push(writable.waitSpaceChange()); } @@ -181,7 +181,7 @@ class OperationQueue { this._window = 0; this._writableState = 'waiting'; - this._initWritableReadyPromise(); + this._initWritablePromise(); this._erroredPromise = new Promise((resolve, reject) => { this._resolveErroredPromise = resolve; @@ -194,19 +194,19 @@ class OperationQueue { this._abortOperation = undefined; this._readableState = 'waiting'; - this._initReadableReadyPromise(); + this._initReadablePromise(); } - _initWritableReadyPromise() { - this._writableReadyPromise = new Promise((resolve, reject) => { - this._resolveWritableReadyPromise = resolve; + _initWritablePromise() { + this._writablePromise = new Promise((resolve, reject) => { + this._resolveWritablePromise = resolve; }); } - _initReadableReadyPromise() { - this._readableReadyPromise = new Promise((resolve, reject) => { - this._resolveReadableReadyPromise = resolve; + _initReadablePromise() { + this._readablePromise = new Promise((resolve, reject) => { + this._resolveReadablePromise = resolve; }); } @@ -221,8 +221,8 @@ class OperationQueue { get writableState() { return this._writableState; } - get writableReady() { - return this._writableReadyPromise; + get writable() { + return this._writablePromise; } get cancelOperation() { @@ -274,10 +274,10 @@ class OperationQueue { } if (shouldApplyBackpressure && this._writableState === 'writable') { this._writableState = 'waiting'; - this._initWritableReadyPromise(); + this._initWritablePromise(); } else if (!shouldApplyBackpressure && this._writableState === 'waiting') { this._writableState = 'writable'; - this._resolveWritableReadyPromise(); + this._resolveWritablePromise(); } if (this._spaceChangePromise !== undefined && this._lastSpace !== this.space) { @@ -305,7 +305,7 @@ class OperationQueue { if (this._readableState === 'waiting') { this._readableState = 'readable'; - this._resolveReadableReadyPromise(); + this._resolveReadablePromise(); } return status; @@ -323,7 +323,7 @@ class OperationQueue { if (this._readableState === 'waiting') { this._readableState = 'readable'; - this._resolveReadableReadyPromise(); + this._resolveReadablePromise(); } return status; @@ -347,7 +347,7 @@ class OperationQueue { this._resolveErroredPromise(); if (this._writableState === 'waiting') { - this._resolveWritableReadyPromise(); + this._resolveWritablePromise(); } this._writableState = 'aborted'; @@ -355,7 +355,7 @@ class OperationQueue { this._abortOperation = new Operation('abort', reason, status); if (this._readableState === 'waiting') { - this._resolveReadableReadyPromise(); + this._resolveReadablePromise(); } this._readableState = 'aborted'; @@ -367,8 +367,8 @@ class OperationQueue { get readableState() { return this._readableState; } - get readableReady() { - return this._readableReadyPromise; + get readable() { + return this._readablePromise; } get abortOperation() { @@ -425,7 +425,7 @@ class OperationQueue { this._readableState = 'drained'; } else { this._readableState = 'waiting'; - this._initReadableReadyPromise(); + this._initReadablePromise(); } } @@ -448,12 +448,12 @@ class OperationQueue { this._cancelOperation = new Operation('cancel', reason, status); if (this._writableState === 'waiting') { - this._resolveWritableReadyPromise(); + this._resolveWritablePromise(); } this._writableState = 'cancelled'; if (this._readableState === 'waiting') { - this._resolveReadableReadyPromise(); + this._resolveReadablePromise(); } this._readableState = 'cancelled'; @@ -470,8 +470,8 @@ class OperationQueueWritableSide { get state() { return this._stream.writableState; } - get ready() { - return this._stream.writableReady; + get writable() { + return this._stream.writable; } get cancelOperation() { @@ -515,8 +515,8 @@ class OperationQueueReadableSide { get state() { return this._stream.readableState; } - get ready() { - return this._stream.readableReady; + get readable() { + return this._stream.readable; } get abortOperation() { diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 2225cb2cc..96f164b1b 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -100,7 +100,7 @@ test('Asynchronous write, read and completion of the operation', t => { var calledComplete = false; - ros.ready.then(() => { + ros.readable.then(() => { t.equals(ros.state, 'readable'); const op = ros.readOperation(); @@ -256,7 +256,7 @@ test('pipeOperationStreams()', t => { var helloStatus; t.equals(wos0.state, 'waiting'); // Check that wos0 becomes writable. - wos0.ready.then(() => { + wos0.writable.then(() => { t.equals(wos0.state, 'writable'); helloStatus = wos0.write('hello'); @@ -278,7 +278,7 @@ test('pipeOperationStreams()', t => { t.equals(ros1.state, 'waiting'); - ros1.ready.then(() => { + ros1.readable.then(() => { t.equals(ros1.state, 'readable'); const op0 = ros1.readOperation(); t.equals(op0.type, 'data'); @@ -482,7 +482,7 @@ test('Transformation example: Byte counting', t => { ros1.window = 1; - ros1.ready.then(() => { + ros1.readable.then(() => { const v = ros1.read(); t.equals(v, 17); t.end(); @@ -538,7 +538,7 @@ class FakeFileBackedByteSource { if (ws.state === 'writable') { promises.push(ws.errored); } else if (ws.state === 'waiting') { - promises.push(ws.ready); + promises.push(ws.writable); } if (this._fileReadPromise !== undefined) { @@ -697,7 +697,7 @@ class FakeFileBackedByteSource { if (rs.state === 'readable') { promises.push(rs.errored); } else if (rs.state === 'waiting') { - promises.push(rs.ready); + promises.push(rs.readable); } if (this._fileReadStatus !== undefined && this._fileReadStatus.state === 'waiting') { @@ -808,7 +808,7 @@ class BytesSetToOneExpectingByteSinkInternalWriter { this._sink._error(op.type); } } else if (rs.state === 'waiting') { - rs.ready + rs.readable .then(this._loop.bind(this)) .catch(this._sink._error.bind(this._sink)); } else if (rs.state === 'aborted') { @@ -886,7 +886,7 @@ class BytesSetToOneExpectingByteSink { if (ws.state === 'writable') { promises.push(ws.errored); } else if (ws.state === 'waiting') { - promises.push(ws.ready); + promises.push(ws.writable); } if (this._currentReadStatus !== undefined && this._currentReadStatus.state === 'waiting') { From c9fdd2c113be8cc72c72bb92362946f945880b88 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Fri, 27 Feb 2015 16:07:36 +0900 Subject: [PATCH 55/93] Add writer and reader --- .../lib/experimental/operation-stream.js | 104 ++++++++++++++---- 1 file changed, 85 insertions(+), 19 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index f2a7ecf05..ac3eb396c 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -210,26 +210,35 @@ class OperationQueue { }); } - // Common interfaces. + // Writable side interfaces - get errored() { - return this._erroredPromise; + _throwIfWritableLocked() { + if (this._writer !== undefined) { + throw new TypeError('locked'); + } } - // Writable side interfaces - get writableState() { + this._throwIfWritableLocked(); return this._writableState; } get writable() { + this._throwIfWritableLocked(); return this._writablePromise; } + get writableErrored() { + this._throwIfWritableLocked(); + return this._erroredPromise; + } get cancelOperation() { + this._throwIfWritableLocked(); return this._cancelOperation; } get space() { + this._throwIfWritableLocked(); + if (this._writableState === 'closed' || this._writableState === 'aborted' || this._writableState === 'cancelled') { @@ -242,7 +251,7 @@ class OperationQueue { return undefined; } - waitSpaceChange() { + _waitSpaceChangeInternal() { if (this._spaceChangePromise !== undefined) { return this._spaceChangePromise; } @@ -254,6 +263,10 @@ class OperationQueue { return this._spaceChangePromise; } + waitSpaceChange() { + this._throwIfWritableLocked(); + return this._waitSpaceChangeInternal(); + } _checkWritableState() { if (this._writableState === 'closed') { @@ -289,7 +302,7 @@ class OperationQueue { } } - write(argument) { + _writeInternal(argument) { this._checkWritableState(); var size = 1; @@ -310,8 +323,12 @@ class OperationQueue { return status; } + write(argument) { + this._throwIfWritableLocked(); + return this._writeInternal(argument); + } - close() { + _closeInternal() { this._checkWritableState(); this._strategy = undefined; @@ -328,8 +345,12 @@ class OperationQueue { return status; } + close() { + this._throwIfWritableLocked(); + return this._closeInternal(); + } - abort(reason) { + _abortInternal(reason) { if (this._writableState === 'aborted') { throw new TypeError('already aborted'); } @@ -361,24 +382,51 @@ class OperationQueue { return status; } + abort(reason) { + this._throwIfWritableLocked(); + return this._abortInternal(reason); + } + + getWriter() { + this._throwIfWritableLocked(); + this._writer = new ExclusiveOperationQueueWriter(this); + return this._writer; + } // Readable side interfaces. + _throwIfReadableLocked() { + if (this._reader !== undefined) { + throw new TypeError('locked'); + } + } + get readableState() { + this._throwIfReadableLocked(); return this._readableState; } get readable() { + this._throwIfReadableLocked(); return this._readablePromise; } + get readableErrored() { + this._throwIfReadableLocked(); + return this._erroredPromise; + } get abortOperation() { + this._throwIfReadableLocked(); return this._abortOperation; } - get window() { + get _windowInternal() { return this._window; } - set window(v) { + get window() { + this._throwIfReadableLocked(); + return this._windowInternal; + } + set _windowInternal(v) { this._window = v; if (this._writableState === 'closed' || @@ -392,6 +440,10 @@ class OperationQueue { } this._updateWritableState(); } + set window(v) { + this._throwIfReadableLocked(); + this._windowInternal = v; + } _checkReadableState() { if (this._readableState === 'drained') { @@ -405,7 +457,7 @@ class OperationQueue { } } - read() { + _readInternal() { this._checkReadableState(); if (this._queue.length === 0) { @@ -431,8 +483,12 @@ class OperationQueue { return entry.value; } + read() { + this._throwIfReadableLocked(); + return this._readInternal(); + } - cancel(reason) { + _cancelInternal(reason) { this._checkReadableState(); for (var i = 0; i < this._queue.length; ++i) { @@ -459,6 +515,16 @@ class OperationQueue { return status; } + cancel(reason) { + this._throwIfReadableLocked(); + return this._cancelInternal(reason); + } + + getReader() { + this._throwIfReadableLocked(); + this._reader = new ExclusiveOperationQueueWriter(this); + return this._reader; + } } // A wrapper to expose only the interfaces of writable side implementing the WritableOperationStream interface. @@ -473,13 +539,13 @@ class OperationQueueWritableSide { get writable() { return this._stream.writable; } + get errored() { + return this._stream.writableErrored; + } get cancelOperation() { return this._stream.cancelOperation; } - get errored() { - return this._stream.errored; - } get window() { return this._stream.window; @@ -518,13 +584,13 @@ class OperationQueueReadableSide { get readable() { return this._stream.readable; } + get errored() { + return this._stream.readableErrored; + } get abortOperation() { return this._stream.abortOperation; } - get errored() { - return this._stream.errored; - } get window() { return this._stream.window; From 3f6cd32bc379fb8b698dc9f736a67c351c8c47d0 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Fri, 27 Feb 2015 18:17:24 +0900 Subject: [PATCH 56/93] Propagate more errors in pipe --- .../lib/experimental/operation-stream.js | 153 +++++++++++++----- 1 file changed, 111 insertions(+), 42 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index ac3eb396c..d4ca9b936 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -33,9 +33,9 @@ export function selectOperationStreams(readable, writable) { if (writable.state === 'writable') { promises.push(writable.errored); + promises.push(writable.waitSpaceChange()); } else { promises.push(writable.writable); - promises.push(writable.waitSpaceChange()); } return Promise.race(promises); @@ -48,7 +48,9 @@ export function pipeOperationStreams(source, dest) { const oldWindow = source.window; function restoreWindowAndReject(e) { + console.log('ssssv'); source.window = oldWindow; + console.log('ssssvz'); reject(e); } @@ -59,23 +61,69 @@ export function pipeOperationStreams(source, dest) { if (source.state !== 'aborted') { source.abort(error); } - restoreWindowAndReject(error); + reject(error); + } + + function writableAcceptsAbort(state) { + return state === 'waiting' || state === 'writable' || state === 'closed'; + } + + function readableAcceptsCancel(state) { + return state === 'waiting' || state === 'readable'; } function loop() { for (;;) { + // Handle interrupt. + if (source.state === 'drained') { + reject(new TypeError('source is drained')); + return; + } + if (source.state === 'cancelled') { + reject(new TypeError('source is cancelled')); + return; + } + + if (dest.state === 'closed') { + reject(new TypeError('dest is closed')); + return; + } + if (dest.state === 'aborted') { + reject(new TypeError('dest is aborted')); + return; + } + + // Propagate errors. + if (source.state === 'aborted') { - if (dest.state !== 'cancelled') { + if (writableAcceptsAbort(dest.state)) { jointOps(source.abortOperation, dest.abort(source.abortOperation.argument)); } - restoreWindowAndReject(new TypeError('aborted')); + reject(new TypeError('aborted')); + return; + } + if (source.state === 'errored') { + const error = new TypeError('source is errored'); + if (writableAcceptsAbort(dest.state)) { + dest.abort(error); + } + reject(error); return; } + if (dest.state === 'cancelled') { - if (source.state !== 'aborted') { + if (readableAcceptsCancel(source.state)) { jointOps(dest.cancelOperation, source.cancel(dest.cancelOperation.argument)); } - restoreWindowAndReject(new TypeError('dest is cancelled')); + reject(new TypeError('dest is cancelled')); + return; + } + if (source.state === 'errored') { + const error = new TypeError('dest is errored'); + if (readableAcceptsCancel(source.state)) { + source.cancel(error); + } + reject(error); return; } @@ -231,19 +279,34 @@ class OperationQueue { return this._erroredPromise; } - get cancelOperation() { - this._throwIfWritableLocked(); + get _cancelOperationInternal() { + if (this._writableState !== 'cancelled') { + throw new TypeError('not cancelled'); + } return this._cancelOperation; } - - get space() { + get cancelOperation() { this._throwIfWritableLocked(); + return this._cancelOperationInternal; + } - if (this._writableState === 'closed' || - this._writableState === 'aborted' || - this._writableState === 'cancelled') { - return undefined; + _checkWritableState() { + if (this._writableState === 'closed') { + throw new TypeError('already closed'); } + if (this._writableState === 'aborted') { + throw new TypeError('already aborted'); + } + if (this._writableState === 'cancelled') { + throw new TypeError('already cancelled'); + } + if (this._writableState === 'errored') { + throw new TypeError('already errored'); + } + } + + get _spaceInternal() { + this._checkWritableState(); if (this._strategy.space !== undefined) { return this._strategy.space(this._queueSize); @@ -251,7 +314,13 @@ class OperationQueue { return undefined; } + get space() { + this._throwIfWritableLocked(); + return this._spaceInternal; + } _waitSpaceChangeInternal() { + this._checkWritableState(); + if (this._spaceChangePromise !== undefined) { return this._spaceChangePromise; } @@ -268,18 +337,6 @@ class OperationQueue { return this._waitSpaceChangeInternal(); } - _checkWritableState() { - if (this._writableState === 'closed') { - throw new TypeError('already closed'); - } - if (this._writableState === 'aborted') { - throw new TypeError('already aborted'); - } - if (this._writableState === 'cancelled') { - throw new TypeError('already cancelled'); - } - } - _updateWritableState() { let shouldApplyBackpressure = false; if (this._strategy.shouldApplyBackpressure !== undefined) { @@ -357,6 +414,9 @@ class OperationQueue { if (this._writableState === 'cancelled') { throw new TypeError('already cancelled'); } + if (this._writableState === 'errored') { + throw new TypeError('already errored'); + } for (var i = this._queue.length - 1; i >= 0; --i) { const op = this._queue[i].value; @@ -414,9 +474,30 @@ class OperationQueue { return this._erroredPromise; } + get _abortOperationInternal() { + if (this._writableState !== 'aborted') { + throw new TypeError('not aborted'); + } + return this._abortOperation; + } get abortOperation() { this._throwIfReadableLocked(); - return this._abortOperation; + return this._abortOperationInternal; + } + + _checkReadableState() { + if (this._readableState === 'drained') { + throw new TypeError('already drained'); + } + if (this._readableState === 'cancelled') { + throw new TypeError('already cancelled'); + } + if (this._readableState === 'aborted') { + throw new TypeError('already aborted'); + } + if (this._readableState === 'errored') { + throw new TypeError('already errored'); + } } get _windowInternal() { @@ -427,11 +508,11 @@ class OperationQueue { return this._windowInternal; } set _windowInternal(v) { + this._checkReadableState(); + this._window = v; - if (this._writableState === 'closed' || - this._writableState === 'aborted' || - this._writableState === 'cancelled') { + if (this._writableState === 'closed') { return; } @@ -445,18 +526,6 @@ class OperationQueue { this._windowInternal = v; } - _checkReadableState() { - if (this._readableState === 'drained') { - throw new TypeError('already drained'); - } - if (this._readableState === 'cancelled') { - throw new TypeError('already cancelled'); - } - if (this._readableState === 'aborted') { - throw new TypeError('already aborted'); - } - } - _readInternal() { this._checkReadableState(); From 28eb32abbe9298f974aa2dfed1f03ea225b8bc8b Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Fri, 27 Feb 2015 18:17:54 +0900 Subject: [PATCH 57/93] Remove restoreWindowAndReject --- .../lib/experimental/operation-stream.js | 7 ------- 1 file changed, 7 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index d4ca9b936..bda3f1e5b 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -47,13 +47,6 @@ export function pipeOperationStreams(source, dest) { return new Promise((resolve, reject) => { const oldWindow = source.window; - function restoreWindowAndReject(e) { - console.log('ssssv'); - source.window = oldWindow; - console.log('ssssvz'); - reject(e); - } - function disposeStreams(error) { if (dest.state !== 'cancelled') { dest.cancel(error); From 35ba1e309feadcf12d80d4a7b8de52ab1da434ca Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Sat, 28 Feb 2015 00:11:17 +0900 Subject: [PATCH 58/93] Drastic redesign --- .../exclusive-operation-stream-writer.js | 106 ++++ .../lib/experimental/operation-stream.js | 516 +----------------- .../experimental/readable-operation-stream.js | 171 ++++++ .../experimental/writable-operation-stream.js | 200 +++++++ .../test/experimental/operation-stream.js | 2 +- 5 files changed, 496 insertions(+), 499 deletions(-) create mode 100644 reference-implementation/lib/experimental/exclusive-operation-stream-writer.js create mode 100644 reference-implementation/lib/experimental/readable-operation-stream.js create mode 100644 reference-implementation/lib/experimental/writable-operation-stream.js diff --git a/reference-implementation/lib/experimental/exclusive-operation-stream-writer.js b/reference-implementation/lib/experimental/exclusive-operation-stream-writer.js new file mode 100644 index 000000000..c10f812b0 --- /dev/null +++ b/reference-implementation/lib/experimental/exclusive-operation-stream-writer.js @@ -0,0 +1,106 @@ +class ExclusiveOperationStreamWriter { + _initWritablePromise() { + this._writablePromise = new Promise((resolve, reject) => { + this._resolveWritablePromise = resolve; + }); + } + + constructor(parent) { + this._parent = parent; + + const state = this._parent._state; + + if (state === 'writable') { + this._writablePromise = Promise.resolve(); + } else { + this._initWritablePromise(); + } + + if (state === 'errored' || state === 'cancelled') { + this._erroredPromise = Promise.resolve(); + } else { + this._erroredPromise = new Promise((resolve, reject) => { + this._resolveErroredPromise = resolve; + }); + } + + this._lastSpace = undefined; + this._spaceChangePromise = undefined; + } + + _throwIfReleased() { + if (this._parent === undefined) { + throw new TypeError('already released'); + } + } + + get state() { + this._throwIfReleased(); + return this._parent._state; + } + get writable() { + this._throwIfReleased(); + return this._writablePromise; + } + get errored() { + this._throwIfReleased(); + return this._erroredPromise; + } + + get cancelOperation() { + this._throwIfReleased(); + return this._parent._cancelOperationIgnoringLock; + } + + _onSpaceChange() { + if (this._spaceChangePromise !== undefined && this._lastSpace !== this.space) { + this._resolveSpaceChangePromise(); + + this._lastSpace = undefined; + this._spaceChangePromise = undefined; + this._resolveSpaceChangePromise = undefined; + } + } + + get space() { + this._throwIfReleased(); + return this._parent._spaceIgnoringLock; + } + waitSpaceChange() { + this._throwIfReleased(); + + if (this._spaceChangePromise !== undefined) { + return this._spaceChangePromise; + } + + this._spaceChangePromise = new Promise((resolve, reject) => { + this._resolveSpaceChangePromise = resolve; + }); + this._lastSpace = this.space; + + return this._spaceChangePromise; + } + + write(argument) { + this._throwIfReleased(); + return this._parent._writeIgnoringLock(argument); + } + close() { + this._throwIfReleased(); + return this._parent._closeIgnoringLock(); + } + abort(reason) { + this._throwIfReleased(); + return this._parent._abortIgnoringLock(reason); + } + + release() { + this._parent._onWriterRelease(); + + this._parent = undefined; + + this._writablePromise = undefined; + this._erroredPromise = undefined; + this._spaceChangePromise = undefined; + } +} diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index bda3f1e5b..c86cf56fa 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -1,13 +1,3 @@ -// Creates a pair of WritableOperationStream implementation and ReadableOperationStream implementation that are -// connected with a queue. This can be used for creating queue-backed operation streams. -export function createOperationQueue(strategy) { - const queue = new OperationQueue(strategy); - return { - writable: new OperationQueueWritableSide(queue), - readable: new OperationQueueReadableSide(queue) - }; -} - export function jointOps(op, status) { function forward() { if (status.state === 'waiting') { @@ -25,22 +15,33 @@ export function jointOps(op, status) { export function selectOperationStreams(readable, writable) { const promises = []; - if (readable.state === 'readable') { - promises.push(readable.errored); - } else { + promises.push(readable.errored); + if (readable.state === 'waiting') { promises.push(readable.readable); } + promises.push(writable.errored); if (writable.state === 'writable') { - promises.push(writable.errored); promises.push(writable.waitSpaceChange()); - } else { + } else if (writable.state === 'waiting') { promises.push(writable.writable); } return Promise.race(promises); } +function writableAcceptsWriteAndClose(state) { + return state === 'waiting' || state === 'writable'; +} + +function writableAcceptsAbort(state) { + return state === 'waiting' || state === 'writable' || state === 'closed'; +} + +function readableAcceptsReadAndCancel(state) { + return state === 'waiting' || state === 'readable'; +} + // Pipes data from source to dest with no transformation. Abort signal, cancel signal and space are also propagated // between source and dest. export function pipeOperationStreams(source, dest) { @@ -57,14 +58,6 @@ export function pipeOperationStreams(source, dest) { reject(error); } - function writableAcceptsAbort(state) { - return state === 'waiting' || state === 'writable' || state === 'closed'; - } - - function readableAcceptsCancel(state) { - return state === 'waiting' || state === 'readable'; - } - function loop() { for (;;) { // Handle interrupt. @@ -105,7 +98,7 @@ export function pipeOperationStreams(source, dest) { } if (dest.state === 'cancelled') { - if (readableAcceptsCancel(source.state)) { + if (readableAcceptsReadAndCancel(source.state)) { jointOps(dest.cancelOperation, source.cancel(dest.cancelOperation.argument)); } reject(new TypeError('dest is cancelled')); @@ -113,7 +106,7 @@ export function pipeOperationStreams(source, dest) { } if (source.state === 'errored') { const error = new TypeError('dest is errored'); - if (readableAcceptsCancel(source.state)) { + if (readableAcceptsReadAndCancel(source.state)) { source.cancel(error); } reject(error); @@ -208,476 +201,3 @@ class Operation { this._status._onError(reason); } } - -class OperationQueue { - constructor(strategy) { - this._queue = []; - this._queueSize = 0; - - this._strategy = strategy; - - this._lastSpace = undefined; - this._spaceChangePromise = undefined; - - this._window = 0; - - this._writableState = 'waiting'; - this._initWritablePromise(); - - this._erroredPromise = new Promise((resolve, reject) => { - this._resolveErroredPromise = resolve; - }); - - this._updateWritableState(); - - this._cancelOperation = undefined; - - this._abortOperation = undefined; - - this._readableState = 'waiting'; - this._initReadablePromise(); - - } - - _initWritablePromise() { - this._writablePromise = new Promise((resolve, reject) => { - this._resolveWritablePromise = resolve; - }); - } - - _initReadablePromise() { - this._readablePromise = new Promise((resolve, reject) => { - this._resolveReadablePromise = resolve; - }); - } - - // Writable side interfaces - - _throwIfWritableLocked() { - if (this._writer !== undefined) { - throw new TypeError('locked'); - } - } - - get writableState() { - this._throwIfWritableLocked(); - return this._writableState; - } - get writable() { - this._throwIfWritableLocked(); - return this._writablePromise; - } - get writableErrored() { - this._throwIfWritableLocked(); - return this._erroredPromise; - } - - get _cancelOperationInternal() { - if (this._writableState !== 'cancelled') { - throw new TypeError('not cancelled'); - } - return this._cancelOperation; - } - get cancelOperation() { - this._throwIfWritableLocked(); - return this._cancelOperationInternal; - } - - _checkWritableState() { - if (this._writableState === 'closed') { - throw new TypeError('already closed'); - } - if (this._writableState === 'aborted') { - throw new TypeError('already aborted'); - } - if (this._writableState === 'cancelled') { - throw new TypeError('already cancelled'); - } - if (this._writableState === 'errored') { - throw new TypeError('already errored'); - } - } - - get _spaceInternal() { - this._checkWritableState(); - - if (this._strategy.space !== undefined) { - return this._strategy.space(this._queueSize); - } - - return undefined; - } - get space() { - this._throwIfWritableLocked(); - return this._spaceInternal; - } - _waitSpaceChangeInternal() { - this._checkWritableState(); - - if (this._spaceChangePromise !== undefined) { - return this._spaceChangePromise; - } - - this._spaceChangePromise = new Promise((resolve, reject) => { - this._resolveSpaceChangePromise = resolve; - }); - this._lastSpace = this.space; - - return this._spaceChangePromise; - } - waitSpaceChange() { - this._throwIfWritableLocked(); - return this._waitSpaceChangeInternal(); - } - - _updateWritableState() { - let shouldApplyBackpressure = false; - if (this._strategy.shouldApplyBackpressure !== undefined) { - shouldApplyBackpressure = this._strategy.shouldApplyBackpressure(this._queueSize); - } - if (shouldApplyBackpressure && this._writableState === 'writable') { - this._writableState = 'waiting'; - this._initWritablePromise(); - } else if (!shouldApplyBackpressure && this._writableState === 'waiting') { - this._writableState = 'writable'; - this._resolveWritablePromise(); - } - - if (this._spaceChangePromise !== undefined && this._lastSpace !== this.space) { - this._resolveSpaceChangePromise(); - - this._lastSpace = undefined; - this._spaceChangePromise = undefined; - this._resolveSpaceChangePromise = undefined; - } - } - - _writeInternal(argument) { - this._checkWritableState(); - - var size = 1; - if (this._strategy.size !== undefined) { - size = this._strategy.size(argument); - } - - const status = new OperationStatus(); - this._queue.push({value: new Operation('data', argument, status), size}); - this._queueSize += size; - - this._updateWritableState(); - - if (this._readableState === 'waiting') { - this._readableState = 'readable'; - this._resolveReadablePromise(); - } - - return status; - } - write(argument) { - this._throwIfWritableLocked(); - return this._writeInternal(argument); - } - - _closeInternal() { - this._checkWritableState(); - - this._strategy = undefined; - - const status = new OperationStatus(); - this._queue.push({value: new Operation('close', undefined, status), size: 0}); - - this._writableState = 'closed'; - - if (this._readableState === 'waiting') { - this._readableState = 'readable'; - this._resolveReadablePromise(); - } - - return status; - } - close() { - this._throwIfWritableLocked(); - return this._closeInternal(); - } - - _abortInternal(reason) { - if (this._writableState === 'aborted') { - throw new TypeError('already aborted'); - } - if (this._writableState === 'cancelled') { - throw new TypeError('already cancelled'); - } - if (this._writableState === 'errored') { - throw new TypeError('already errored'); - } - - for (var i = this._queue.length - 1; i >= 0; --i) { - const op = this._queue[i].value; - op.error(new TypeError('aborted')); - } - this._queue = []; - this._strategy = undefined; - - this._resolveErroredPromise(); - - if (this._writableState === 'waiting') { - this._resolveWritablePromise(); - } - this._writableState = 'aborted'; - - const status = new OperationStatus(); - this._abortOperation = new Operation('abort', reason, status); - - if (this._readableState === 'waiting') { - this._resolveReadablePromise(); - } - this._readableState = 'aborted'; - - return status; - } - abort(reason) { - this._throwIfWritableLocked(); - return this._abortInternal(reason); - } - - getWriter() { - this._throwIfWritableLocked(); - this._writer = new ExclusiveOperationQueueWriter(this); - return this._writer; - } - - // Readable side interfaces. - - _throwIfReadableLocked() { - if (this._reader !== undefined) { - throw new TypeError('locked'); - } - } - - get readableState() { - this._throwIfReadableLocked(); - return this._readableState; - } - get readable() { - this._throwIfReadableLocked(); - return this._readablePromise; - } - get readableErrored() { - this._throwIfReadableLocked(); - return this._erroredPromise; - } - - get _abortOperationInternal() { - if (this._writableState !== 'aborted') { - throw new TypeError('not aborted'); - } - return this._abortOperation; - } - get abortOperation() { - this._throwIfReadableLocked(); - return this._abortOperationInternal; - } - - _checkReadableState() { - if (this._readableState === 'drained') { - throw new TypeError('already drained'); - } - if (this._readableState === 'cancelled') { - throw new TypeError('already cancelled'); - } - if (this._readableState === 'aborted') { - throw new TypeError('already aborted'); - } - if (this._readableState === 'errored') { - throw new TypeError('already errored'); - } - } - - get _windowInternal() { - return this._window; - } - get window() { - this._throwIfReadableLocked(); - return this._windowInternal; - } - set _windowInternal(v) { - this._checkReadableState(); - - this._window = v; - - if (this._writableState === 'closed') { - return; - } - - if (this._strategy.onWindowUpdate !== undefined) { - this._strategy.onWindowUpdate(v); - } - this._updateWritableState(); - } - set window(v) { - this._throwIfReadableLocked(); - this._windowInternal = v; - } - - _readInternal() { - this._checkReadableState(); - - if (this._queue.length === 0) { - throw new TypeError('not readable'); - } - - const entry = this._queue.shift(); - this._queueSize -= entry.size; - - if (this._writableState === 'writable' || - this._writableState === 'waiting') { - this._updateWritableState(); - } - - if (this._queue.length === 0) { - if (entry.type === 'close') { - this._readableState = 'drained'; - } else { - this._readableState = 'waiting'; - this._initReadablePromise(); - } - } - - return entry.value; - } - read() { - this._throwIfReadableLocked(); - return this._readInternal(); - } - - _cancelInternal(reason) { - this._checkReadableState(); - - for (var i = 0; i < this._queue.length; ++i) { - const op = this._queue[i].value; - op.error(reason); - } - this._queue = []; - this._strategy = undefined; - - this._resolveErroredPromise(); - - const status = new OperationStatus(); - this._cancelOperation = new Operation('cancel', reason, status); - - if (this._writableState === 'waiting') { - this._resolveWritablePromise(); - } - this._writableState = 'cancelled'; - - if (this._readableState === 'waiting') { - this._resolveReadablePromise(); - } - this._readableState = 'cancelled'; - - return status; - } - cancel(reason) { - this._throwIfReadableLocked(); - return this._cancelInternal(reason); - } - - getReader() { - this._throwIfReadableLocked(); - this._reader = new ExclusiveOperationQueueWriter(this); - return this._reader; - } -} - -// A wrapper to expose only the interfaces of writable side implementing the WritableOperationStream interface. -class OperationQueueWritableSide { - constructor(stream) { - this._stream = stream; - } - - get state() { - return this._stream.writableState; - } - get writable() { - return this._stream.writable; - } - get errored() { - return this._stream.writableErrored; - } - - get cancelOperation() { - return this._stream.cancelOperation; - } - - get window() { - return this._stream.window; - } - set window(v) { - this._stream.window = v; - } - - get space() { - return this._stream.space; - } - waitSpaceChange() { - return this._stream.waitSpaceChange(); - } - - write(value) { - return this._stream.write(value); - } - close() { - return this._stream.close(); - } - abort(reason) { - return this._stream.abort(reason); - } -} - -// A wrapper to expose only the interfaces of readable side implementing the ReadableOperationStream interface. -class OperationQueueReadableSide { - constructor(stream) { - this._stream = stream; - } - - get state() { - return this._stream.readableState; - } - get readable() { - return this._stream.readable; - } - get errored() { - return this._stream.readableErrored; - } - - get abortOperation() { - return this._stream.abortOperation; - } - - get window() { - return this._stream.window; - } - set window(v) { - this._stream.window = v; - } - - read() { - const op = this._stream.read(); - if (op.type === 'data') { - return op.argument; - } else if (op.type === 'close') { - return ReadableOperationStream.EOS; - } - } - readOperation() { - return this._stream.read(); - } - cancel(reason) { - return this._stream.cancel(reason); - } -} - -class ReadableOperationStream { -} - -ReadableOperationStream.EOS = {}; diff --git a/reference-implementation/lib/experimental/readable-operation-stream.js b/reference-implementation/lib/experimental/readable-operation-stream.js new file mode 100644 index 000000000..50206d7ee --- /dev/null +++ b/reference-implementation/lib/experimental/readable-operation-stream.js @@ -0,0 +1,171 @@ +import { Operation, OperationStatus, readableAcceptsReadAndCancel } from './operation-stream.js'; + +export class ReadableOperationStream { + _initReadablePromise() { + this._readablePromise = new Promise((resolve, reject) => { + this._resolveReadablePromise = resolve; + }); + } + + constructor() { + this._state = 'waiting'; + + this._initReadablePromise(); + + this._erroredPromise = new Promise((resolve, reject) => { + this._resolveErroredPromise = resolve; + }); + + this._abortOperation = undefined; + + this._window = 0; + + this._reader = undefined; + } + + _markDrained() { + this._state = 'drained'; + } + + _markWaiting() { + if (this._reader === undefined) { + this._initReadablePromise(); + } else { + this._reader._initReadablePromise(); + } + + this._state = 'waiting'; + } + + _markReadable() { + if (this._state !== 'waiting') { + return; + } + + if (this._reader === undefined) { + this._resolveReadablePromise(); + } else { + this._reader._resolveReadablePromise(); + } + + this._state = 'readable'; + } + + _markAborted(operation) { + if (this._reader === undefined) { + this._resolveErroredPromise(); + } else { + this._writer._resolveErroredPromise(); + } + + this._state = 'aborted'; + + this._abortOperation = operation; + } + + _throwIfLocked() { + if (this._reader !== undefined) { + throw new TypeError('locked'); + } + } + + get state() { + this._throwIfLocked(); + return this._state; + } + get readable() { + this._throwIfLocked(); + return this._readablePromise; + } + get errored() { + this._throwIfLocked(); + return this._erroredPromise; + } + + get _abortOperationIgnoringLock() { + if (this._state !== 'aborted') { + throw new TypeError('not aborted'); + } + return this._abortOperation; + } + get abortOperation() { + this._throwIfLocked(); + return this._abortOperationIgnoringLock; + } + + _checkState() { + if (!readableAcceptsReadAndCancel(this._state)) { + throw new TypeError('already ' + this._state); + } + } + + get _windowIgnoringLock() { + return this._window; + } + get window() { + this._throwIfLocked(); + return this._windowIgnoringLock; + } + + set _windowIgnoringLock(v) { + this._checkState(); + + this._window = v; + + this._onWindowUpdate(v); + } + set window(v) { + this._throwIfLocked(); + this._windowIgnoringLock = v; + } + + _readOperationIgnoringLock() { + this._checkState(); + return this._readOperationInternal(); + } + readOperation() { + this._throwIfLocked(); + return this._readOperationIgnoringLock(); + } + read() { + const op = this.readOperation(); + if (op.type === 'data') { + return op.argument; + } else if (op.type === 'close') { + return ReadableOperationStream.EOS; + } else { + // error + } + } + + _cancelIgnoringLock(reason) { + this._checkState(); + + const operationStatus = new OperationStatus(); + const operation = new Operation('cancel', reason, operationStatus); + + this._cancelInternal(operation); + + if (this._reader !== undefined) { + this._reader._resolveErroredPromise(); + } else { + this._resolveErroredPromise(); + } + + this._state = 'cancelled'; + + return operationStatus; + } + cancel(reason) { + this._throwIfLocked(); + return this._cancelIgnoringLock(reason); + } + + getReader() { + this._throwIfLocked(); + this._reader = new ExclusiveOperationStreamWriter(this); + return this._reader; + } +} + +ReadableOperationStream.EOS = {}; diff --git a/reference-implementation/lib/experimental/writable-operation-stream.js b/reference-implementation/lib/experimental/writable-operation-stream.js new file mode 100644 index 000000000..ffc069a67 --- /dev/null +++ b/reference-implementation/lib/experimental/writable-operation-stream.js @@ -0,0 +1,200 @@ +import { Operation, OperationStatus, writableAcceptsWriteAndClose, writableAcceptsAbort } from './operation-stream.js'; +import { ExclusiveOperationStreamWriter } from './exclusive-operation-stream-writer.js'; + +class WritableOperationStream { + _initWritablePromise() { + this._writablePromise = new Promise((resolve, reject) => { + this._resolveWritablePromise = resolve; + }); + } + + constructor() { + this._state = 'waiting'; + + this._initWritablePromise(); + + this._erroredPromise = new Promise((resolve, reject) => { + this._resolveErroredPromise = resolve; + }); + + this._lastSpace = undefined; + this._spaceChangePromise = undefined; + + this._cancelOperation = undefined; + + this._writer = undefined; + } + + _throwIfLocked() { + if (this._writer !== undefined) { + throw new TypeError('locked'); + } + } + + _markWaiting() { + if (this._state !== 'writable') { + return; + } + + if (this._writer === undefined) { + this._initWritablePromise(); + } else { + this._writer._initWritablePromise(); + } + + this._state = 'waiting'; + } + + _onSpaceChange() { + if (this._writer === undefined) { + if (this._spaceChangePromise !== undefined && this._lastSpace !== this.space) { + this._resolveSpaceChangePromise(); + + this._lastSpace = undefined; + this._spaceChangePromise = undefined; + this._resolveSpaceChangePromise = undefined; + } + } else { + this._writer._onSpaceChange(); + } + } + + _markCancelled(operation) { + this._state = 'cancelled'; + + if (this._writer === undefined) { + this._resolveErroredPromise(); + } else { + this._writer._resolveErroredPromise(); + } + + this._cancelOperation = operation; + } + + _markWritable() { + if (this._state === 'waiting') { + if (this._writer === undefined) { + this._resolveWritablePromise(); + } else { + this._writer._resolveWritablePromise(); + } + + this._state = 'writable'; + } + } + + get state() { + this._throwIfLocked(); + return this._state; + } + get writable() { + this._throwIfLocked(); + return this._writablePromise; + } + get errored() { + this._throwIfLocked(); + return this._erroredPromise; + } + + get _cancelOperationIgnoringLock() { + if (this._state !== 'cancelled') { + throw new TypeError('not cancelled'); + } + return this._cancelOperation; + } + get cancelOperation() { + this._throwIfLocked(); + return this._cancelOperationIgnoringLock; + } + + _checkState() { + if (!writableAcceptsWriteAndClose(this._state)) { + throw new TypeError('already ' + this._state); + } + } + + get _spaceIgnoringLock() { + this._checkState(); + return this._spaceInternal(); + } + get space() { + this._throwIfLocked(); + return this._spaceIgnoringLock; + } + + _waitSpaceChangeIgnoringLock() { + this._checkState(); + + if (this._spaceChangePromise !== undefined) { + return this._spaceChangePromise; + } + + this._spaceChangePromise = new Promise((resolve, reject) => { + this._resolveSpaceChangePromise = resolve; + }); + this._lastSpace = this.space; + + return this._spaceChangePromise; + } + waitSpaceChange() { + this._throwIfLocked(); + return this._waitSpaceChangeIgnoringLock(); + } + + _writeIgnoringLock(argument) { + this._checkState(); + + const operationStatus = new OperationStatus(); + const operation = new Operation('data', argument, operationStatus); + + this._writeInternal(operation); + + return operationStatus; + } + write(argument) { + this._throwIfLocked(); + return this._writeIgnoringLock(argument); + } + + _closeIgnoringLock() { + this._checkState(); + + const operationStatus = new OperationStatus(); + const operation = new Operation('close', undefined, operationStatus); + + this._closeInternal(operation); + + this._state = 'closed'; + + return operationStatus; + } + close() { + this._throwIfLocked(); + return this._closeIgnoringLock(); + } + + _abortIgnoringLock(reason) { + if (!writableAcceptsAbort(this._state)) { + throw new TypeError('already ' + this._state); + } + + const operationStatus = new OperationStatus(); + const operation = new Operation('abort', reason, operationStatus); + + this._abortInternal(operation); + + this._state = 'aborted'; + + return operationStatus; + } + abort(reason) { + this._throwIfLocked(); + return this._abortIgnoringLock(reason); + } + + getWriter() { + this._throwIfLocked(); + this._writer = new ExclusiveOperationStreamWriter(this); + return this._writer; + } +} diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 96f164b1b..dec7363fc 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -773,6 +773,7 @@ class FakeFileBackedByteSource { class BytesSetToOneExpectingByteSinkInternalWriter { constructor(sink, readableStream) { this._readableStream = readableStream; + this._readableStream.window = 64; this._sink = sink; @@ -973,7 +974,6 @@ test('Piping from a source with a buffer pool to a buffer taking sink', t => { const sink = new BytesSetToOneExpectingByteSink(); const bufferConsumingStream = sink.createBufferConsumingStream(); - bufferConsumingStream.window = 64; // pipeOperationStreams automatically adjusts window of the readable side. const pipePromise = pipeOperationStreams(file.createBufferProducingStreamWithPool(pool), bufferConsumingStream); From d94a04051f1723dbaafecab29b3720ea75707020 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Sat, 28 Feb 2015 00:12:36 +0900 Subject: [PATCH 59/93] Add operation queue file --- .../lib/experimental/operation-queue.js | 177 ++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 reference-implementation/lib/experimental/operation-queue.js diff --git a/reference-implementation/lib/experimental/operation-queue.js b/reference-implementation/lib/experimental/operation-queue.js new file mode 100644 index 000000000..60fe874e1 --- /dev/null +++ b/reference-implementation/lib/experimental/operation-queue.js @@ -0,0 +1,177 @@ +// Creates a pair of WritableOperationStream implementation and ReadableOperationStream implementation that are +// connected with a queue. This can be used for creating queue-backed operation streams. +export function createOperationQueue(strategy) { + return new OperationQueue(strategy); +} + +class OperationQueue { + _updateWritableSide() { + if (this._strategy === undefined) { + return; + } + + let shouldApplyBackpressure = false; + if (this._strategy.shouldApplyBackpressure !== undefined) { + shouldApplyBackpressure = this._strategy.shouldApplyBackpressure(this._queueSize); + } + if (shouldApplyBackpressure) { + this._writableSide._markWaiting(); + } else { + this._writableSide._markWritable(); + } + + this._writableSide._onSpaceChange(); + } + + constructor(strategy) { + this._queue = []; + this._queueSize = 0; + + this._strategy = strategy; + + this._writableSide = new OperationQueueWritableSide(this); + this._readableSide = new OperationQueueReadableSide(this); + + this._updateWritableSide(); + } + + get writable() { + return this._writableSide; + } + + get readable() { + return this._readableSide; + } + + get space() { + if (this._strategy.space !== undefined) { + return this._strategy.space(this._queueSize); + } + + return undefined; + } + + write(operation) { + var size = 1; + if (this._strategy.size !== undefined) { + size = this._strategy.size(operation.argument); + } + + this._queue.push({value: operation, size}); + this._queueSize += size; + + this._updateWritableSide(); + + this._readableSide._markReadable(); + } + + close(operation) { + this._queue.push({value: operation, size: 0}); + + // No longer necessary. + this._strategy = undefined; + + this._readableSide._markReadable(); + } + + abort(operation) { + for (var i = this._queue.length - 1; i >= 0; --i) { + const op = this._queue[i].value; + op.error(new TypeError('aborted')); + } + this._queue = []; + + this._strategy = undefined; + + this._readableSide._markAborted(operation); + } + + onWindowUpdate(v) { + if (this._writableSide._state === 'closed') { + return; + } + + if (this._strategy.onWindowUpdate !== undefined) { + this._strategy.onWindowUpdate(v); + } + + this._updateWritableSide(); + } + + readOperation() { + if (this._queue.length === 0) { + throw new TypeError('not readable'); + } + + const entry = this._queue.shift(); + this._queueSize -= entry.size; + + if (this._queue.length === 0) { + if (entry.type === 'close') { + this._readableSide._markDrained(); + } else { + this._readableSide._markWaiting(); + } + } + + this._updateWritableSide(); + + return entry.value; + } + + cancel(operation) { + for (var i = 0; i < this._queue.length; ++i) { + const op = this._queue[i].value; + op.error(operation.argument); + } + this._queue = []; + + this._strategy = undefined; + + this._writableSide._markCancelled(operation); + } +} + +export class OperationQueueWritableSide extends WritableOperationStream { + constructor(queue) { + super(); + + this._queue = queue; + } + + _spaceInternal() { + return this._queue.space; + } + + _writeInternal(operation) { + this._queue.write(operation); + } + + _closeInternal(operation) { + this._queue.close(operation); + } + + _abortInternal(operation) { + this._queue.abort(operation); + } +} + +export class OperationQueueReadableSide extends ReadableOperationStream { + constructor(queue) { + super(); + + this._queue = queue; + } + + _onWindowUpdate(v) { + this._queue.onWindowUpdate(v); + } + + _readOperationInternal() { + return this._queue.readOperation(); + } + + _cancelInternal(operation) { + this._queue.cancel(operation); + } +} From a9f45b969b3439b4a367d25369db52e4d895aa31 Mon Sep 17 00:00:00 2001 From: rieszgithub Date: Sat, 28 Feb 2015 01:48:34 +0900 Subject: [PATCH 60/93] Make things work after file reorganization --- .../lib/experimental/operation-queue.js | 3 +++ .../lib/experimental/operation-stream.js | 10 +++++----- .../lib/experimental/readable-operation-stream.js | 2 +- .../lib/experimental/writable-operation-stream.js | 6 +++--- reference-implementation/run-tests.js | 5 ++++- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-queue.js b/reference-implementation/lib/experimental/operation-queue.js index 60fe874e1..ed81b8244 100644 --- a/reference-implementation/lib/experimental/operation-queue.js +++ b/reference-implementation/lib/experimental/operation-queue.js @@ -1,3 +1,6 @@ +import { WritableOperationStream } from './writable-operation-stream'; +import { ReadableOperationStream } from './readable-operation-stream'; + // Creates a pair of WritableOperationStream implementation and ReadableOperationStream implementation that are // connected with a queue. This can be used for creating queue-backed operation streams. export function createOperationQueue(strategy) { diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index c86cf56fa..ff7dd0402 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -30,15 +30,15 @@ export function selectOperationStreams(readable, writable) { return Promise.race(promises); } -function writableAcceptsWriteAndClose(state) { +export function writableAcceptsWriteAndClose(state) { return state === 'waiting' || state === 'writable'; } -function writableAcceptsAbort(state) { +export function writableAcceptsAbort(state) { return state === 'waiting' || state === 'writable' || state === 'closed'; } -function readableAcceptsReadAndCancel(state) { +export function readableAcceptsReadAndCancel(state) { return state === 'waiting' || state === 'readable'; } @@ -147,7 +147,7 @@ export function pipeOperationStreams(source, dest) { }); } -class OperationStatus { +export class OperationStatus { constructor() { this._state = 'waiting'; this._result = undefined; @@ -180,7 +180,7 @@ class OperationStatus { } } -class Operation { +export class Operation { constructor(type, argument, status) { this._type = type; this._argument = argument; diff --git a/reference-implementation/lib/experimental/readable-operation-stream.js b/reference-implementation/lib/experimental/readable-operation-stream.js index 50206d7ee..3e4d7d1ba 100644 --- a/reference-implementation/lib/experimental/readable-operation-stream.js +++ b/reference-implementation/lib/experimental/readable-operation-stream.js @@ -1,4 +1,4 @@ -import { Operation, OperationStatus, readableAcceptsReadAndCancel } from './operation-stream.js'; +import { Operation, OperationStatus, readableAcceptsReadAndCancel } from './operation-stream'; export class ReadableOperationStream { _initReadablePromise() { diff --git a/reference-implementation/lib/experimental/writable-operation-stream.js b/reference-implementation/lib/experimental/writable-operation-stream.js index ffc069a67..a724027ec 100644 --- a/reference-implementation/lib/experimental/writable-operation-stream.js +++ b/reference-implementation/lib/experimental/writable-operation-stream.js @@ -1,7 +1,7 @@ -import { Operation, OperationStatus, writableAcceptsWriteAndClose, writableAcceptsAbort } from './operation-stream.js'; -import { ExclusiveOperationStreamWriter } from './exclusive-operation-stream-writer.js'; +import { Operation, OperationStatus, writableAcceptsWriteAndClose, writableAcceptsAbort } from './operation-stream'; +import { ExclusiveOperationStreamWriter } from './exclusive-operation-stream-writer'; -class WritableOperationStream { +export class WritableOperationStream { _initWritablePromise() { this._writablePromise = new Promise((resolve, reject) => { this._resolveWritablePromise = resolve; diff --git a/reference-implementation/run-tests.js b/reference-implementation/run-tests.js index a9ca8bccc..25057ac99 100644 --- a/reference-implementation/run-tests.js +++ b/reference-implementation/run-tests.js @@ -4,18 +4,21 @@ const path = require('path'); import ReadableStream from './lib/readable-stream'; import WritableStream from './lib/writable-stream'; import ReadableByteStream from './lib/experimental/readable-byte-stream'; -import { createOperationQueue, jointOps, pipeOperationStreams, selectOperationStreams +import { jointOps, pipeOperationStreams, selectOperationStreams } from './lib/experimental/operation-stream'; +import { createOperationQueue } from './lib/experimental/operation-queue'; import ByteLengthQueuingStrategy from './lib/byte-length-queuing-strategy'; import CountQueuingStrategy from './lib/count-queuing-strategy'; import TransformStream from './lib/transform-stream'; global.ReadableStream = ReadableStream; global.WritableStream = WritableStream; + global.createOperationQueue = createOperationQueue; global.jointOps = jointOps; global.pipeOperationStreams = pipeOperationStreams; global.selectOperationStreams = selectOperationStreams; + global.ReadableByteStream = ReadableByteStream; global.ByteLengthQueuingStrategy = ByteLengthQueuingStrategy; global.CountQueuingStrategy = CountQueuingStrategy; From 57a3c918e2c1ea074aa09454017d06fab7b44d09 Mon Sep 17 00:00:00 2001 From: rieszgithub Date: Sat, 28 Feb 2015 01:55:58 +0900 Subject: [PATCH 61/93] Reoreder methods --- .../experimental/readable-operation-stream.js | 84 +++++++------- .../experimental/writable-operation-stream.js | 108 +++++++++--------- 2 files changed, 100 insertions(+), 92 deletions(-) diff --git a/reference-implementation/lib/experimental/readable-operation-stream.js b/reference-implementation/lib/experimental/readable-operation-stream.js index 3e4d7d1ba..c5b997fb2 100644 --- a/reference-implementation/lib/experimental/readable-operation-stream.js +++ b/reference-implementation/lib/experimental/readable-operation-stream.js @@ -1,6 +1,8 @@ import { Operation, OperationStatus, readableAcceptsReadAndCancel } from './operation-stream'; export class ReadableOperationStream { + // Public members and internal methods. + _initReadablePromise() { this._readablePromise = new Promise((resolve, reject) => { this._resolveReadablePromise = resolve; @@ -23,46 +25,6 @@ export class ReadableOperationStream { this._reader = undefined; } - _markDrained() { - this._state = 'drained'; - } - - _markWaiting() { - if (this._reader === undefined) { - this._initReadablePromise(); - } else { - this._reader._initReadablePromise(); - } - - this._state = 'waiting'; - } - - _markReadable() { - if (this._state !== 'waiting') { - return; - } - - if (this._reader === undefined) { - this._resolveReadablePromise(); - } else { - this._reader._resolveReadablePromise(); - } - - this._state = 'readable'; - } - - _markAborted(operation) { - if (this._reader === undefined) { - this._resolveErroredPromise(); - } else { - this._writer._resolveErroredPromise(); - } - - this._state = 'aborted'; - - this._abortOperation = operation; - } - _throwIfLocked() { if (this._reader !== undefined) { throw new TypeError('locked'); @@ -166,6 +128,48 @@ export class ReadableOperationStream { this._reader = new ExclusiveOperationStreamWriter(this); return this._reader; } + + // Methods exposed only to the underlying source. + + _markWaiting() { + if (this._reader === undefined) { + this._initReadablePromise(); + } else { + this._reader._initReadablePromise(); + } + + this._state = 'waiting'; + } + + _markReadable() { + if (this._state !== 'waiting') { + return; + } + + if (this._reader === undefined) { + this._resolveReadablePromise(); + } else { + this._reader._resolveReadablePromise(); + } + + this._state = 'readable'; + } + + _markDrained() { + this._state = 'drained'; + } + + _markAborted(operation) { + if (this._reader === undefined) { + this._resolveErroredPromise(); + } else { + this._writer._resolveErroredPromise(); + } + + this._state = 'aborted'; + + this._abortOperation = operation; + } } ReadableOperationStream.EOS = {}; diff --git a/reference-implementation/lib/experimental/writable-operation-stream.js b/reference-implementation/lib/experimental/writable-operation-stream.js index a724027ec..aaf76c86e 100644 --- a/reference-implementation/lib/experimental/writable-operation-stream.js +++ b/reference-implementation/lib/experimental/writable-operation-stream.js @@ -2,6 +2,8 @@ import { Operation, OperationStatus, writableAcceptsWriteAndClose, writableAccep import { ExclusiveOperationStreamWriter } from './exclusive-operation-stream-writer'; export class WritableOperationStream { + // Public members and internal methods. + _initWritablePromise() { this._writablePromise = new Promise((resolve, reject) => { this._resolveWritablePromise = resolve; @@ -31,58 +33,6 @@ export class WritableOperationStream { } } - _markWaiting() { - if (this._state !== 'writable') { - return; - } - - if (this._writer === undefined) { - this._initWritablePromise(); - } else { - this._writer._initWritablePromise(); - } - - this._state = 'waiting'; - } - - _onSpaceChange() { - if (this._writer === undefined) { - if (this._spaceChangePromise !== undefined && this._lastSpace !== this.space) { - this._resolveSpaceChangePromise(); - - this._lastSpace = undefined; - this._spaceChangePromise = undefined; - this._resolveSpaceChangePromise = undefined; - } - } else { - this._writer._onSpaceChange(); - } - } - - _markCancelled(operation) { - this._state = 'cancelled'; - - if (this._writer === undefined) { - this._resolveErroredPromise(); - } else { - this._writer._resolveErroredPromise(); - } - - this._cancelOperation = operation; - } - - _markWritable() { - if (this._state === 'waiting') { - if (this._writer === undefined) { - this._resolveWritablePromise(); - } else { - this._writer._resolveWritablePromise(); - } - - this._state = 'writable'; - } - } - get state() { this._throwIfLocked(); return this._state; @@ -197,4 +147,58 @@ export class WritableOperationStream { this._writer = new ExclusiveOperationStreamWriter(this); return this._writer; } + + // Methods exposed only to the underlying sink. + + _markWaiting() { + if (this._state !== 'writable') { + return; + } + + if (this._writer === undefined) { + this._initWritablePromise(); + } else { + this._writer._initWritablePromise(); + } + + this._state = 'waiting'; + } + + _markWritable() { + if (this._state === 'waiting') { + if (this._writer === undefined) { + this._resolveWritablePromise(); + } else { + this._writer._resolveWritablePromise(); + } + + this._state = 'writable'; + } + } + + _markCancelled(operation) { + this._state = 'cancelled'; + + if (this._writer === undefined) { + this._resolveErroredPromise(); + } else { + this._writer._resolveErroredPromise(); + } + + this._cancelOperation = operation; + } + + _onSpaceChange() { + if (this._writer === undefined) { + if (this._spaceChangePromise !== undefined && this._lastSpace !== this.space) { + this._resolveSpaceChangePromise(); + + this._lastSpace = undefined; + this._spaceChangePromise = undefined; + this._resolveSpaceChangePromise = undefined; + } + } else { + this._writer._onSpaceChange(); + } + } } From 76df4ca46da435b4619d892a3f3b55b2acaaf904 Mon Sep 17 00:00:00 2001 From: rieszgithub Date: Sat, 28 Feb 2015 02:00:58 +0900 Subject: [PATCH 62/93] Use underlying source/sink approach --- .../lib/experimental/operation-queue.js | 48 +------------------ .../experimental/readable-operation-stream.js | 10 ++-- .../experimental/writable-operation-stream.js | 12 +++-- 3 files changed, 15 insertions(+), 55 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-queue.js b/reference-implementation/lib/experimental/operation-queue.js index ed81b8244..b1673fc43 100644 --- a/reference-implementation/lib/experimental/operation-queue.js +++ b/reference-implementation/lib/experimental/operation-queue.js @@ -32,8 +32,8 @@ class OperationQueue { this._strategy = strategy; - this._writableSide = new OperationQueueWritableSide(this); - this._readableSide = new OperationQueueReadableSide(this); + this._writableSide = new WritableOperationStream(this); + this._readableSide = new ReadableOperationStream(this); this._updateWritableSide(); } @@ -134,47 +134,3 @@ class OperationQueue { this._writableSide._markCancelled(operation); } } - -export class OperationQueueWritableSide extends WritableOperationStream { - constructor(queue) { - super(); - - this._queue = queue; - } - - _spaceInternal() { - return this._queue.space; - } - - _writeInternal(operation) { - this._queue.write(operation); - } - - _closeInternal(operation) { - this._queue.close(operation); - } - - _abortInternal(operation) { - this._queue.abort(operation); - } -} - -export class OperationQueueReadableSide extends ReadableOperationStream { - constructor(queue) { - super(); - - this._queue = queue; - } - - _onWindowUpdate(v) { - this._queue.onWindowUpdate(v); - } - - _readOperationInternal() { - return this._queue.readOperation(); - } - - _cancelInternal(operation) { - this._queue.cancel(operation); - } -} diff --git a/reference-implementation/lib/experimental/readable-operation-stream.js b/reference-implementation/lib/experimental/readable-operation-stream.js index c5b997fb2..87b6a9939 100644 --- a/reference-implementation/lib/experimental/readable-operation-stream.js +++ b/reference-implementation/lib/experimental/readable-operation-stream.js @@ -9,7 +9,9 @@ export class ReadableOperationStream { }); } - constructor() { + constructor(source) { + this._source = source; + this._state = 'waiting'; this._initReadablePromise(); @@ -74,7 +76,7 @@ export class ReadableOperationStream { this._window = v; - this._onWindowUpdate(v); + this._source.onWindowUpdate(v); } set window(v) { this._throwIfLocked(); @@ -83,7 +85,7 @@ export class ReadableOperationStream { _readOperationIgnoringLock() { this._checkState(); - return this._readOperationInternal(); + return this._source.readOperation(); } readOperation() { this._throwIfLocked(); @@ -106,7 +108,7 @@ export class ReadableOperationStream { const operationStatus = new OperationStatus(); const operation = new Operation('cancel', reason, operationStatus); - this._cancelInternal(operation); + this._source.cancel(operation); if (this._reader !== undefined) { this._reader._resolveErroredPromise(); diff --git a/reference-implementation/lib/experimental/writable-operation-stream.js b/reference-implementation/lib/experimental/writable-operation-stream.js index aaf76c86e..1e6c90093 100644 --- a/reference-implementation/lib/experimental/writable-operation-stream.js +++ b/reference-implementation/lib/experimental/writable-operation-stream.js @@ -10,7 +10,9 @@ export class WritableOperationStream { }); } - constructor() { + constructor(sink) { + this._sink = sink; + this._state = 'waiting'; this._initWritablePromise(); @@ -65,7 +67,7 @@ export class WritableOperationStream { get _spaceIgnoringLock() { this._checkState(); - return this._spaceInternal(); + return this._sink.space; } get space() { this._throwIfLocked(); @@ -97,7 +99,7 @@ export class WritableOperationStream { const operationStatus = new OperationStatus(); const operation = new Operation('data', argument, operationStatus); - this._writeInternal(operation); + this._sink.write(operation); return operationStatus; } @@ -112,7 +114,7 @@ export class WritableOperationStream { const operationStatus = new OperationStatus(); const operation = new Operation('close', undefined, operationStatus); - this._closeInternal(operation); + this._sink.close(operation); this._state = 'closed'; @@ -131,7 +133,7 @@ export class WritableOperationStream { const operationStatus = new OperationStatus(); const operation = new Operation('abort', reason, operationStatus); - this._abortInternal(operation); + this._sink.abort(operation); this._state = 'aborted'; From ba4d9f1a5148cf78c83921d8c611a7777a6cabd3 Mon Sep 17 00:00:00 2001 From: rieszgithub Date: Sat, 28 Feb 2015 02:02:00 +0900 Subject: [PATCH 63/93] Hide details of OperationQueue --- reference-implementation/lib/experimental/operation-queue.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/reference-implementation/lib/experimental/operation-queue.js b/reference-implementation/lib/experimental/operation-queue.js index b1673fc43..b00370e2b 100644 --- a/reference-implementation/lib/experimental/operation-queue.js +++ b/reference-implementation/lib/experimental/operation-queue.js @@ -4,7 +4,8 @@ import { ReadableOperationStream } from './readable-operation-stream'; // Creates a pair of WritableOperationStream implementation and ReadableOperationStream implementation that are // connected with a queue. This can be used for creating queue-backed operation streams. export function createOperationQueue(strategy) { - return new OperationQueue(strategy); + const queue = new OperationQueue(strategy); + return { writable: queue.writable, readable: queue.readable }; } class OperationQueue { From 2adb0cffb80e107ac54c6c55733124be4f551ebd Mon Sep 17 00:00:00 2001 From: rieszgithub Date: Sat, 28 Feb 2015 02:26:45 +0900 Subject: [PATCH 64/93] Fix transition to drain --- .../lib/experimental/operation-queue.js | 2 +- .../lib/experimental/operation-stream.js | 7 +-- .../test/experimental/operation-stream.js | 55 ++++++++++++------- 3 files changed, 40 insertions(+), 24 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-queue.js b/reference-implementation/lib/experimental/operation-queue.js index b00370e2b..1437737e9 100644 --- a/reference-implementation/lib/experimental/operation-queue.js +++ b/reference-implementation/lib/experimental/operation-queue.js @@ -111,7 +111,7 @@ class OperationQueue { this._queueSize -= entry.size; if (this._queue.length === 0) { - if (entry.type === 'close') { + if (entry.value.type === 'close') { this._readableSide._markDrained(); } else { this._readableSide._markWaiting(); diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index ff7dd0402..2922b6942 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -49,10 +49,10 @@ export function pipeOperationStreams(source, dest) { const oldWindow = source.window; function disposeStreams(error) { - if (dest.state !== 'cancelled') { - dest.cancel(error); + if (writableAcceptsAbort(dest.state)) { + dest.abort(error); } - if (source.state !== 'aborted') { + if (readableAcceptsReadAndCancel(source.state)) { source.abort(error); } reject(error); @@ -121,7 +121,6 @@ export function pipeOperationStreams(source, dest) { } else if (op.type === 'close') { jointOps(op, dest.close()); - source.window = oldWindow; resolve(); return; diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index dec7363fc..e8f519980 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -172,6 +172,25 @@ test('Asynchronous write, read and completion of the operation', t => { t.end(); }); +test('close()', t => { + const pair = createOperationQueue(new AdjustableArrayBufferStrategy()); + const wos = pair.writable; + const ros = pair.readable; + + wos.write(new ArrayBuffer(10)); + wos.close(); + t.equals(wos.state, 'closed'); + + t.equals(ros.state, 'readable'); + const v0 = ros.read(); + t.equals(ros.state, 'readable'); + const v1 = ros.read(); + t.equals(v1, ros.constructor.EOS); + t.equals(ros.state, 'drained'); + + t.end(); +}); + test('abort()', t => { t.plan(9); @@ -535,9 +554,8 @@ class FakeFileBackedByteSource { const ws = this._writableStream; - if (ws.state === 'writable') { - promises.push(ws.errored); - } else if (ws.state === 'waiting') { + promises.push(ws.errored); + if (ws.state === 'waiting') { promises.push(ws.writable); } @@ -670,14 +688,7 @@ class FakeFileBackedByteSource { this._fileReadStatus.result = e; } - _readFromFile() { - const op = this._readableStream.readOperation(); - - if (op.type === 'close') { - return; - } - // Assert: op.type === 'data'. - + _readFromFile(op) { this._currentRequest = op; const status = {}; @@ -694,9 +705,8 @@ class FakeFileBackedByteSource { const rs = this._readableStream; - if (rs.state === 'readable') { - promises.push(rs.errored); - } else if (rs.state === 'waiting') { + promises.push(rs.errored); + if (rs.state === 'waiting') { promises.push(rs.readable); } @@ -731,8 +741,16 @@ class FakeFileBackedByteSource { } if (rs.state === 'readable' && this._currentRequest === undefined) { - this._readFromFile(); - continue; + const op = this._readableStream.readOperation(); + + if (op.type === 'close') { + return; + } else if (op.type === 'data') { + this._readFromFile(op); + continue; + } else { + // error + } } this._select(); @@ -884,9 +902,8 @@ class BytesSetToOneExpectingByteSink { const ws = this._writableStream; - if (ws.state === 'writable') { - promises.push(ws.errored); - } else if (ws.state === 'waiting') { + promises.push(ws.errored); + if (ws.state === 'waiting') { promises.push(ws.writable); } From 9918c769ada8212662732a5ff2c2b7333a7a000f Mon Sep 17 00:00:00 2001 From: rieszgithub Date: Sat, 28 Feb 2015 02:28:46 +0900 Subject: [PATCH 65/93] More comments --- reference-implementation/lib/experimental/operation-queue.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/reference-implementation/lib/experimental/operation-queue.js b/reference-implementation/lib/experimental/operation-queue.js index 1437737e9..2a513e7f8 100644 --- a/reference-implementation/lib/experimental/operation-queue.js +++ b/reference-implementation/lib/experimental/operation-queue.js @@ -47,6 +47,8 @@ class OperationQueue { return this._readableSide; } + // Underlying sink implementation. + get space() { if (this._strategy.space !== undefined) { return this._strategy.space(this._queueSize); @@ -90,6 +92,8 @@ class OperationQueue { this._readableSide._markAborted(operation); } + // Underlying source implementation. + onWindowUpdate(v) { if (this._writableSide._state === 'closed') { return; From 6e43b6ee76c52472937d80409cd083f61ac5fab6 Mon Sep 17 00:00:00 2001 From: rieszgithub Date: Sat, 28 Feb 2015 02:31:47 +0900 Subject: [PATCH 66/93] Rename --- .../lib/experimental/operation-queue.js | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-queue.js b/reference-implementation/lib/experimental/operation-queue.js index 2a513e7f8..3928459cd 100644 --- a/reference-implementation/lib/experimental/operation-queue.js +++ b/reference-implementation/lib/experimental/operation-queue.js @@ -9,7 +9,7 @@ export function createOperationQueue(strategy) { } class OperationQueue { - _updateWritableSide() { + _updateWritableStream() { if (this._strategy === undefined) { return; } @@ -19,12 +19,12 @@ class OperationQueue { shouldApplyBackpressure = this._strategy.shouldApplyBackpressure(this._queueSize); } if (shouldApplyBackpressure) { - this._writableSide._markWaiting(); + this._writableStream._markWaiting(); } else { - this._writableSide._markWritable(); + this._writableStream._markWritable(); } - this._writableSide._onSpaceChange(); + this._writableStream._onSpaceChange(); } constructor(strategy) { @@ -33,18 +33,18 @@ class OperationQueue { this._strategy = strategy; - this._writableSide = new WritableOperationStream(this); - this._readableSide = new ReadableOperationStream(this); + this._writableStream = new WritableOperationStream(this); + this._readableStream = new ReadableOperationStream(this); - this._updateWritableSide(); + this._updateWritableStream(); } get writable() { - return this._writableSide; + return this._writableStream; } get readable() { - return this._readableSide; + return this._readableStream; } // Underlying sink implementation. @@ -66,9 +66,9 @@ class OperationQueue { this._queue.push({value: operation, size}); this._queueSize += size; - this._updateWritableSide(); + this._updateWritableStream(); - this._readableSide._markReadable(); + this._readableStream._markReadable(); } close(operation) { @@ -77,7 +77,7 @@ class OperationQueue { // No longer necessary. this._strategy = undefined; - this._readableSide._markReadable(); + this._readableStream._markReadable(); } abort(operation) { @@ -89,13 +89,13 @@ class OperationQueue { this._strategy = undefined; - this._readableSide._markAborted(operation); + this._readableStream._markAborted(operation); } // Underlying source implementation. onWindowUpdate(v) { - if (this._writableSide._state === 'closed') { + if (this._writableStream._state === 'closed') { return; } @@ -103,7 +103,7 @@ class OperationQueue { this._strategy.onWindowUpdate(v); } - this._updateWritableSide(); + this._updateWritableStream(); } readOperation() { @@ -116,13 +116,13 @@ class OperationQueue { if (this._queue.length === 0) { if (entry.value.type === 'close') { - this._readableSide._markDrained(); + this._readableStream._markDrained(); } else { - this._readableSide._markWaiting(); + this._readableStream._markWaiting(); } } - this._updateWritableSide(); + this._updateWritableStream(); return entry.value; } @@ -136,6 +136,6 @@ class OperationQueue { this._strategy = undefined; - this._writableSide._markCancelled(operation); + this._writableStream._markCancelled(operation); } } From 43dcaaeaaadb74fc74bb215a8f069fa39ef11b9c Mon Sep 17 00:00:00 2001 From: rieszgithub Date: Sat, 28 Feb 2015 02:35:14 +0900 Subject: [PATCH 67/93] Tweak --- reference-implementation/lib/experimental/operation-queue.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reference-implementation/lib/experimental/operation-queue.js b/reference-implementation/lib/experimental/operation-queue.js index 3928459cd..960eec523 100644 --- a/reference-implementation/lib/experimental/operation-queue.js +++ b/reference-implementation/lib/experimental/operation-queue.js @@ -95,7 +95,7 @@ class OperationQueue { // Underlying source implementation. onWindowUpdate(v) { - if (this._writableStream._state === 'closed') { + if (this._strategy === undefined) { return; } From aac7c91caa4fea6320c806c2c515e09e8e638af9 Mon Sep 17 00:00:00 2001 From: rieszgithub Date: Sat, 28 Feb 2015 08:53:10 +0900 Subject: [PATCH 68/93] Implemented Writer --- .../exclusive-operation-stream-writer.js | 20 +++- .../experimental/writable-operation-stream.js | 48 +++++--- .../test/experimental/operation-stream.js | 107 ++++++++++++++++++ 3 files changed, 157 insertions(+), 18 deletions(-) diff --git a/reference-implementation/lib/experimental/exclusive-operation-stream-writer.js b/reference-implementation/lib/experimental/exclusive-operation-stream-writer.js index c10f812b0..fd532c64d 100644 --- a/reference-implementation/lib/experimental/exclusive-operation-stream-writer.js +++ b/reference-implementation/lib/experimental/exclusive-operation-stream-writer.js @@ -1,4 +1,4 @@ -class ExclusiveOperationStreamWriter { +export class ExclusiveOperationStreamWriter { _initWritablePromise() { this._writablePromise = new Promise((resolve, reject) => { this._resolveWritablePromise = resolve; @@ -52,6 +52,22 @@ class ExclusiveOperationStreamWriter { return this._parent._cancelOperationIgnoringLock; } + _syncStateAndPromises() { + const state = this._parent._state; + if (state === 'waiting') { + if (this._resolveWritablePromise === undefined) { + this._initWritablePromise(); + } + } else if (state === 'writable') { + if (this._resolveWritablePromise !== undefined) { + this._resolveWritablePromise(); + this._resolveWritablePromise = undefined; + } + } else if (state === 'cancelled' || state === 'errored') { + this._resolveErroredPromise(); + } + } + _onSpaceChange() { if (this._spaceChangePromise !== undefined && this._lastSpace !== this.space) { this._resolveSpaceChangePromise(); @@ -95,7 +111,7 @@ class ExclusiveOperationStreamWriter { } release() { - this._parent._onWriterRelease(); + this._parent._releaseWriter(); this._parent = undefined; diff --git a/reference-implementation/lib/experimental/writable-operation-stream.js b/reference-implementation/lib/experimental/writable-operation-stream.js index 1e6c90093..fa8a1e9c4 100644 --- a/reference-implementation/lib/experimental/writable-operation-stream.js +++ b/reference-implementation/lib/experimental/writable-operation-stream.js @@ -144,6 +144,28 @@ export class WritableOperationStream { return this._abortIgnoringLock(reason); } + _syncStateAndPromises() { + if (this._state === 'waiting') { + if (this._resolveWritablePromise === undefined) { + this._initWritablePromise(); + } + } else if (this._state === 'writable') { + if (this._resolveWritablePromise !== undefined) { + this._resolveWritablePromise(); + this._resolveWritablePromise = undefined; + } + } else if (this._state === 'cancelled' || this._state === 'errored') { + this._resolveErroredPromise(); + } + } + + _releaseWriter() { + this._writer = undefined; + + this._syncStateAndPromises(); + this._onSpaceChange(); + } + getWriter() { this._throwIfLocked(); this._writer = new ExclusiveOperationStreamWriter(this); @@ -153,28 +175,22 @@ export class WritableOperationStream { // Methods exposed only to the underlying sink. _markWaiting() { - if (this._state !== 'writable') { - return; - } + this._state = 'waiting'; if (this._writer === undefined) { - this._initWritablePromise(); + this._syncStateAndPromises(); } else { - this._writer._initWritablePromise(); + this._writer._syncStateAndPromises(); } - - this._state = 'waiting'; } _markWritable() { - if (this._state === 'waiting') { - if (this._writer === undefined) { - this._resolveWritablePromise(); - } else { - this._writer._resolveWritablePromise(); - } + this._state = 'writable'; - this._state = 'writable'; + if (this._writer === undefined) { + this._syncStateAndPromises(); + } else { + this._writer._syncStateAndPromises(); } } @@ -182,9 +198,9 @@ export class WritableOperationStream { this._state = 'cancelled'; if (this._writer === undefined) { - this._resolveErroredPromise(); + this._syncStateAndPromises(); } else { - this._writer._resolveErroredPromise(); + this._writer._syncStateAndPromises(); } this._cancelOperation = operation; diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index e8f519980..4bdf4d75d 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -1069,3 +1069,110 @@ test('Piping from a buffer taking source to a sink with buffer', t => { t.end(); }); }); + +test('getWriter()', t => { + const pair = createOperationQueue(new AdjustableArrayBufferStrategy()); + const wos = pair.writable; + const ros = pair.readable; + + const writer = wos.getWriter(); + + t.throws(() => wos.state, /TypeError/); + t.throws(() => wos.writable, /TypeError/); + t.throws(() => wos.errored, /TypeError/); + t.throws(() => wos.cancelOperation, /TypeError/); + t.throws(() => wos.space, /TypeError/); + t.throws(() => wos.waitSpaceChange(), /TypeError/); + t.throws(() => wos.write(new ArrayBuffer(10)), /TypeError/); + t.throws(() => wos.close(), /TypeError/); + t.throws(() => wos.abort(), /TypeError/); + t.throws(() => wos.getWriter(), /TypeError/); + + t.equals(writer.state, 'waiting'); + + ros.window = 1; + + t.equals(writer.state, 'writable'); + t.ok(writer.writable); + t.ok(writer.errored); + t.ok(writer.space); + t.ok(writer.waitSpaceChange()); + const writeStatus = writer.write(new ArrayBuffer(10)); + t.equals(writer.state, 'waiting'); + const closeStatus = writer.close(); + t.equals(writer.state, 'closed'); + const abortStatus = writer.abort(); + t.equals(writer.state, 'aborted'); + + writer.release(); + + t.equals(wos.state, 'aborted'); + t.ok(wos.writable); + t.ok(wos.errored); + + t.end(); +}); + +test('writable and waitSpaceChange() on the stream obtained before getWriter() call', t => { + t.plan(2); + + const pair = createOperationQueue(new AdjustableArrayBufferStrategy()); + const wos = pair.writable; + const ros = pair.readable; + + let released = false; + + wos.writable.then(r => { + if (released) { + t.pass(); + } else { + t.fail(r); + } + }); + + wos.waitSpaceChange().then(r => { + if (released) { + t.pass(); + } else { + t.fail(r); + } + }); + + const writer = wos.getWriter(); + + ros.window = 1; + + setTimeout(() => { + writer.release(); + + released = true; + }, 10); +}); + +test('errored on the stream obtained before getWriter() call', t => { + t.plan(1); + + const pair = createOperationQueue(new AdjustableArrayBufferStrategy()); + const wos = pair.writable; + const ros = pair.readable; + + let released = false; + + wos.errored.then(r => { + if (released) { + t.pass(); + } else { + t.fail(r); + } + }); + + const writer = wos.getWriter(); + + ros.cancel(); + + setTimeout(() => { + writer.release(); + + released = true; + }, 10); +}); From b48944f829592aa4834dd0c1eb8fcfca17d49745 Mon Sep 17 00:00:00 2001 From: tyoshino Date: Sat, 28 Feb 2015 10:46:32 +0900 Subject: [PATCH 69/93] Use revealing constructor pattern --- .../lib/experimental/operation-queue.js | 22 +++++++++---------- .../experimental/readable-operation-stream.js | 11 +++++++++- .../experimental/writable-operation-stream.js | 11 +++++++++- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-queue.js b/reference-implementation/lib/experimental/operation-queue.js index 960eec523..4005ee30f 100644 --- a/reference-implementation/lib/experimental/operation-queue.js +++ b/reference-implementation/lib/experimental/operation-queue.js @@ -19,12 +19,12 @@ class OperationQueue { shouldApplyBackpressure = this._strategy.shouldApplyBackpressure(this._queueSize); } if (shouldApplyBackpressure) { - this._writableStream._markWaiting(); + this._writableStreamDelegate.markWaiting(); } else { - this._writableStream._markWritable(); + this._writableStreamDelegate.markWritable(); } - this._writableStream._onSpaceChange(); + this._writableStreamDelegate.onSpaceChange(); } constructor(strategy) { @@ -33,8 +33,8 @@ class OperationQueue { this._strategy = strategy; - this._writableStream = new WritableOperationStream(this); - this._readableStream = new ReadableOperationStream(this); + this._writableStream = new WritableOperationStream(this, delegate => this._writableStreamDelegate = delegate); + this._readableStream = new ReadableOperationStream(this, delegate => this._readableStreamDelegate = delegate); this._updateWritableStream(); } @@ -68,7 +68,7 @@ class OperationQueue { this._updateWritableStream(); - this._readableStream._markReadable(); + this._readableStreamDelegate.markReadable(); } close(operation) { @@ -77,7 +77,7 @@ class OperationQueue { // No longer necessary. this._strategy = undefined; - this._readableStream._markReadable(); + this._readableStreamDelegate.markReadable(); } abort(operation) { @@ -89,7 +89,7 @@ class OperationQueue { this._strategy = undefined; - this._readableStream._markAborted(operation); + this._readableStreamDelegate.markAborted(operation); } // Underlying source implementation. @@ -116,9 +116,9 @@ class OperationQueue { if (this._queue.length === 0) { if (entry.value.type === 'close') { - this._readableStream._markDrained(); + this._readableStreamDelegate.markDrained(); } else { - this._readableStream._markWaiting(); + this._readableStreamDelegate.markWaiting(); } } @@ -136,6 +136,6 @@ class OperationQueue { this._strategy = undefined; - this._writableStream._markCancelled(operation); + this._writableStreamDelegate.markCancelled(operation); } } diff --git a/reference-implementation/lib/experimental/readable-operation-stream.js b/reference-implementation/lib/experimental/readable-operation-stream.js index 87b6a9939..b1d97bd34 100644 --- a/reference-implementation/lib/experimental/readable-operation-stream.js +++ b/reference-implementation/lib/experimental/readable-operation-stream.js @@ -9,7 +9,7 @@ export class ReadableOperationStream { }); } - constructor(source) { + constructor(source, f) { this._source = source; this._state = 'waiting'; @@ -25,6 +25,15 @@ export class ReadableOperationStream { this._window = 0; this._reader = undefined; + + const delegate = { + markWaiting: this._markWaiting.bind(this), + markReadable: this._markReadable.bind(this), + markDrained: this._markDrained.bind(this), + markAborted: this._markAborted.bind(this) + }; + + f(delegate); } _throwIfLocked() { diff --git a/reference-implementation/lib/experimental/writable-operation-stream.js b/reference-implementation/lib/experimental/writable-operation-stream.js index fa8a1e9c4..cccf3d41c 100644 --- a/reference-implementation/lib/experimental/writable-operation-stream.js +++ b/reference-implementation/lib/experimental/writable-operation-stream.js @@ -10,7 +10,7 @@ export class WritableOperationStream { }); } - constructor(sink) { + constructor(sink, f) { this._sink = sink; this._state = 'waiting'; @@ -27,6 +27,15 @@ export class WritableOperationStream { this._cancelOperation = undefined; this._writer = undefined; + + const delegate = { + markWaiting: this._markWaiting.bind(this), + markWritable: this._markWritable.bind(this), + markCancelled: this._markCancelled.bind(this), + onSpaceChange: this._onSpaceChange.bind(this) + }; + + f(delegate); } _throwIfLocked() { From f86d33423e20afc6fb2d95ae578613dd64e4dc64 Mon Sep 17 00:00:00 2001 From: tyoshino Date: Sun, 1 Mar 2015 22:05:54 +0900 Subject: [PATCH 70/93] Add unstoppable source example --- .../test/experimental/operation-stream.js | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 4bdf4d75d..9142af36f 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -1,5 +1,8 @@ const test = require('tape-catch'); +import { Operation, OperationStatus } from '../../lib/experimental/operation-stream'; +import { ReadableOperationStream } from '../../lib/experimental/readable-operation-stream'; + test('Operation stream pair is constructed', t => { const pair = createOperationQueue({ size() { @@ -1176,3 +1179,141 @@ test('errored on the stream obtained before getWriter() call', t => { released = true; }, 10); }); + +class FakeUnstoppablePushSource { + constructor(count) { + this._count = count; + + setTimeout(this._push.bind(this), 0); + } + + _push() { + if (this._count == 0) { + this.onend(); + return; + } + + this.ondata('foo'); + --this._count; + + setTimeout(this._push.bind(this), 0); + } +} + +test.only('Adapting unstoppable push source', t => { + class Source { + constructor() { + this._pushSource = new FakeUnstoppablePushSource(10); + + this._queue = []; + + this._readableStream = new ReadableOperationStream(this, delegate => { + t.equal(typeof delegate.markWaiting, 'function', 'markWaiting is a function'); + t.equal(typeof delegate.markReadable, 'function', 'markReadable is a function'); + t.equal(typeof delegate.markDrained, 'function', 'markDrained is a function'); + t.equal(typeof delegate.markAborted, 'function', 'markAborted is a function'); + + this._readableStreamDelegate = delegate; + + this._pushSource.ondata = chunk => { + const status = new OperationStatus(); + this._queue.push(new Operation('data', chunk, status)); + delegate.markReadable(); + }; + + this._pushSource.onend = () => { + const status = new OperationStatus(); + this._queue.push(new Operation('close', undefined, status)); + delegate.markReadable(); + }; + + this._pushSource.onerror = () => { + this._queue = []; + delegate.markAborted(); + }; + }); + } + + get readableStream() { + return this._readableStream; + } + + onWindowUpdate(v) { + } + + readOperation() { + if (this._queue.length === 0) { + throw new TypeError('not readable'); + } + + const entry = this._queue.shift(); + + if (this._queue.length === 0) { + if (entry.type === 'close') { + this._readableStreamDelegate.markDrained(); + } else { + this._readableStreamDelegate.markWaiting(); + } + } + + return entry; + } + + cancel() { + this._queue = []; + + this._pushSource.close(); + } + } + + const source = new Source(); + const readableStream = source.readableStream; + + let count = 0; + function pump() { + for (;;) { + if (readableStream.state === 'waiting') { + Promise.race([readableStream.readable, readableStream.errored]) + .then(pump) + .catch(e => { + t.fail(e); + t.end(); + }); + return; + } else if (readableStream.state === 'readable') { + const data = readableStream.read(); + if (count === 10) { + t.equals(data, ReadableOperationStream.EOS); + t.end(); + return; + } else { + t.equals(data, 'foo'); + console.log(data); + ++count; + } + } else if (readableStream.state === 'drained') { + t.fail(); + t.end(); + return; + } else if (readableStream.state === 'aborted') { + t.fail(); + t.end(); + return; + } else if (readableStream.state === 'cancelled') { + t.fail(); + t.end(); + return; + } else if (readableStream.state === 'errored') { + t.fail(); + t.end(); + return; + } else { + t.fail(readableStream.state); + t.end(); + return; + } + } + } + pump(); +}); + From f63d2debf6e495da2c68aa25b6ce32857c28709d Mon Sep 17 00:00:00 2001 From: tyoshino Date: Sun, 1 Mar 2015 22:26:50 +0900 Subject: [PATCH 71/93] Move all stuff for implementing operation concept to operation-queue --- .../lib/experimental/operation-queue.js | 31 +++++++++++++--- .../lib/experimental/operation-stream.js | 2 +- .../experimental/readable-operation-stream.js | 26 ++++---------- .../experimental/writable-operation-stream.js | 27 +++++--------- .../test/experimental/operation-stream.js | 36 +++++++++---------- 5 files changed, 59 insertions(+), 63 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-queue.js b/reference-implementation/lib/experimental/operation-queue.js index 4005ee30f..c6935fbd6 100644 --- a/reference-implementation/lib/experimental/operation-queue.js +++ b/reference-implementation/lib/experimental/operation-queue.js @@ -1,3 +1,4 @@ +import { Operation, OperationStatus } from './operation-stream'; import { WritableOperationStream } from './writable-operation-stream'; import { ReadableOperationStream } from './readable-operation-stream'; @@ -57,7 +58,10 @@ class OperationQueue { return undefined; } - write(operation) { + write(value) { + const operationStatus = new OperationStatus(); + const operation = new Operation('data', value, operationStatus); + var size = 1; if (this._strategy.size !== undefined) { size = this._strategy.size(operation.argument); @@ -69,18 +73,28 @@ class OperationQueue { this._updateWritableStream(); this._readableStreamDelegate.markReadable(); + + return operationStatus; } - close(operation) { + close() { + const operationStatus = new OperationStatus(); + const operation = new Operation('close', ReadableOperationStream.EOS, operationStatus); + this._queue.push({value: operation, size: 0}); // No longer necessary. this._strategy = undefined; this._readableStreamDelegate.markReadable(); + + return operationStatus; } - abort(operation) { + abort(reason) { + const operationStatus = new OperationStatus(); + const operation = new Operation('abort', reason, operationStatus); + for (var i = this._queue.length - 1; i >= 0; --i) { const op = this._queue[i].value; op.error(new TypeError('aborted')); @@ -90,6 +104,8 @@ class OperationQueue { this._strategy = undefined; this._readableStreamDelegate.markAborted(operation); + + return operationStatus; } // Underlying source implementation. @@ -106,7 +122,7 @@ class OperationQueue { this._updateWritableStream(); } - readOperation() { + read() { if (this._queue.length === 0) { throw new TypeError('not readable'); } @@ -127,7 +143,10 @@ class OperationQueue { return entry.value; } - cancel(operation) { + cancel(reason) { + const operationStatus = new OperationStatus(); + const operation = new Operation('cancel', reason, operationStatus); + for (var i = 0; i < this._queue.length; ++i) { const op = this._queue[i].value; op.error(operation.argument); @@ -137,5 +156,7 @@ class OperationQueue { this._strategy = undefined; this._writableStreamDelegate.markCancelled(operation); + + return operationStatus; } } diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 2922b6942..64b751d10 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -115,7 +115,7 @@ export function pipeOperationStreams(source, dest) { if (dest.state === 'writable') { if (source.state === 'readable') { - const op = source.readOperation(); + const op = source.read(); if (op.type === 'data') { jointOps(op, dest.write(op.argument)); } else if (op.type === 'close') { diff --git a/reference-implementation/lib/experimental/readable-operation-stream.js b/reference-implementation/lib/experimental/readable-operation-stream.js index b1d97bd34..37d033f54 100644 --- a/reference-implementation/lib/experimental/readable-operation-stream.js +++ b/reference-implementation/lib/experimental/readable-operation-stream.js @@ -92,32 +92,20 @@ export class ReadableOperationStream { this._windowIgnoringLock = v; } - _readOperationIgnoringLock() { + _readIgnoringLock() { this._checkState(); - return this._source.readOperation(); - } - readOperation() { - this._throwIfLocked(); - return this._readOperationIgnoringLock(); + + return this._source.read(); } read() { - const op = this.readOperation(); - if (op.type === 'data') { - return op.argument; - } else if (op.type === 'close') { - return ReadableOperationStream.EOS; - } else { - // error - } + this._throwIfLocked(); + return this._readIgnoringLock(); } _cancelIgnoringLock(reason) { this._checkState(); - const operationStatus = new OperationStatus(); - const operation = new Operation('cancel', reason, operationStatus); - - this._source.cancel(operation); + const result = this._source.cancel(reason); if (this._reader !== undefined) { this._reader._resolveErroredPromise(); @@ -127,7 +115,7 @@ export class ReadableOperationStream { this._state = 'cancelled'; - return operationStatus; + return result; } cancel(reason) { this._throwIfLocked(); diff --git a/reference-implementation/lib/experimental/writable-operation-stream.js b/reference-implementation/lib/experimental/writable-operation-stream.js index cccf3d41c..570801997 100644 --- a/reference-implementation/lib/experimental/writable-operation-stream.js +++ b/reference-implementation/lib/experimental/writable-operation-stream.js @@ -102,32 +102,24 @@ export class WritableOperationStream { return this._waitSpaceChangeIgnoringLock(); } - _writeIgnoringLock(argument) { + _writeIgnoringLock(value) { this._checkState(); - const operationStatus = new OperationStatus(); - const operation = new Operation('data', argument, operationStatus); - - this._sink.write(operation); - - return operationStatus; + return this._sink.write(value); } - write(argument) { + write(value) { this._throwIfLocked(); - return this._writeIgnoringLock(argument); + return this._writeIgnoringLock(value); } _closeIgnoringLock() { this._checkState(); - const operationStatus = new OperationStatus(); - const operation = new Operation('close', undefined, operationStatus); - - this._sink.close(operation); + const result = this._sink.close(); this._state = 'closed'; - return operationStatus; + return result; } close() { this._throwIfLocked(); @@ -139,14 +131,11 @@ export class WritableOperationStream { throw new TypeError('already ' + this._state); } - const operationStatus = new OperationStatus(); - const operation = new Operation('abort', reason, operationStatus); - - this._sink.abort(operation); + const result = this._sink.abort(reason); this._state = 'aborted'; - return operationStatus; + return result; } abort(reason) { this._throwIfLocked(); diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 9142af36f..4dc5aa00b 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -1,6 +1,5 @@ const test = require('tape-catch'); -import { Operation, OperationStatus } from '../../lib/experimental/operation-stream'; import { ReadableOperationStream } from '../../lib/experimental/readable-operation-stream'; test('Operation stream pair is constructed', t => { @@ -78,7 +77,7 @@ test('Synchronous write, read and completion of the operation', t => { t.equals(status.state, 'waiting'); - const op = ros.readOperation(); + const op = ros.read(); t.equals(op.argument, 'hello'); t.equals(ros.state, 'waiting'); @@ -106,7 +105,7 @@ test('Asynchronous write, read and completion of the operation', t => { ros.readable.then(() => { t.equals(ros.state, 'readable'); - const op = ros.readOperation(); + const op = ros.read(); t.equals(op.argument, 'hello'); t.equals(ros.state, 'waiting'); @@ -185,9 +184,9 @@ test('close()', t => { t.equals(wos.state, 'closed'); t.equals(ros.state, 'readable'); - const v0 = ros.read(); + const v0 = ros.read().argument; t.equals(ros.state, 'readable'); - const v1 = ros.read(); + const v1 = ros.read().argument; t.equals(v1, ros.constructor.EOS); t.equals(ros.state, 'drained'); @@ -302,19 +301,19 @@ test('pipeOperationStreams()', t => { ros1.readable.then(() => { t.equals(ros1.state, 'readable'); - const op0 = ros1.readOperation(); + const op0 = ros1.read(); t.equals(op0.type, 'data'); t.equals(op0.argument, 'hello'); op0.complete('hi'); t.equals(ros1.state, 'readable'); - const op1 = ros1.readOperation(); + const op1 = ros1.read(); t.equals(op1.type, 'data'); t.equals(op1.argument, 'world'); t.equals(ros1.state, 'readable'); - const op2 = ros1.readOperation(); + const op2 = ros1.read(); t.equals(op2.type, 'close'); t.equals(helloStatus.state, 'waiting'); @@ -450,7 +449,7 @@ test('Transformation example: Byte counting', t => { if (dest.state === 'writable') { if (source.state === 'readable') { - const op = source.readOperation(); + const op = source.read(); if (op.type === 'data') { count += op.argument.length; op.complete(); @@ -505,7 +504,7 @@ test('Transformation example: Byte counting', t => { ros1.window = 1; ros1.readable.then(() => { - const v = ros1.read(); + const v = ros1.read().argument; t.equals(v, 17); t.end(); }).catch(e => { @@ -744,7 +743,7 @@ class FakeFileBackedByteSource { } if (rs.state === 'readable' && this._currentRequest === undefined) { - const op = this._readableStream.readOperation(); + const op = this._readableStream.read(); if (op.type === 'close') { return; @@ -806,7 +805,7 @@ class BytesSetToOneExpectingByteSinkInternalWriter { for (;;) { if (rs.state === 'readable') { - const op = rs.readOperation(); + const op = rs.read(); if (op.type === 'data') { const view = op.argument; @@ -1200,7 +1199,7 @@ class FakeUnstoppablePushSource { } } -test.only('Adapting unstoppable push source', t => { +test('Adapting unstoppable push source', t => { class Source { constructor() { this._pushSource = new FakeUnstoppablePushSource(10); @@ -1216,14 +1215,12 @@ test.only('Adapting unstoppable push source', t => { this._readableStreamDelegate = delegate; this._pushSource.ondata = chunk => { - const status = new OperationStatus(); - this._queue.push(new Operation('data', chunk, status)); + this._queue.push({type: 'data', data: chunk}); delegate.markReadable(); }; this._pushSource.onend = () => { - const status = new OperationStatus(); - this._queue.push(new Operation('close', undefined, status)); + this._queue.push({type: 'close'}); delegate.markReadable(); }; @@ -1241,7 +1238,7 @@ test.only('Adapting unstoppable push source', t => { onWindowUpdate(v) { } - readOperation() { + read() { if (this._queue.length === 0) { throw new TypeError('not readable'); } @@ -1251,8 +1248,10 @@ test.only('Adapting unstoppable push source', t => { if (this._queue.length === 0) { if (entry.type === 'close') { this._readableStreamDelegate.markDrained(); + return ReadableOperationStream.EOS; } else { this._readableStreamDelegate.markWaiting(); + return entry.data; } } @@ -1288,7 +1287,6 @@ test.only('Adapting unstoppable push source', t => { return; } else { t.equals(data, 'foo'); - console.log(data); ++count; } } else if (readableStream.state === 'drained') { From 8a19445eaca753865e9a8860bc9924419c2b3f2e Mon Sep 17 00:00:00 2001 From: tyoshino Date: Sun, 1 Mar 2015 22:31:34 +0900 Subject: [PATCH 72/93] Tweak --- .../experimental/writable-operation-stream.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/reference-implementation/lib/experimental/writable-operation-stream.js b/reference-implementation/lib/experimental/writable-operation-stream.js index 570801997..e9c3b5e96 100644 --- a/reference-implementation/lib/experimental/writable-operation-stream.js +++ b/reference-implementation/lib/experimental/writable-operation-stream.js @@ -68,14 +68,11 @@ export class WritableOperationStream { return this._cancelOperationIgnoringLock; } - _checkState() { + get _spaceIgnoringLock() { if (!writableAcceptsWriteAndClose(this._state)) { throw new TypeError('already ' + this._state); } - } - get _spaceIgnoringLock() { - this._checkState(); return this._sink.space; } get space() { @@ -84,7 +81,9 @@ export class WritableOperationStream { } _waitSpaceChangeIgnoringLock() { - this._checkState(); + if (!writableAcceptsWriteAndClose(this._state)) { + throw new TypeError('already ' + this._state); + } if (this._spaceChangePromise !== undefined) { return this._spaceChangePromise; @@ -103,7 +102,9 @@ export class WritableOperationStream { } _writeIgnoringLock(value) { - this._checkState(); + if (!writableAcceptsWriteAndClose(this._state)) { + throw new TypeError('already ' + this._state); + } return this._sink.write(value); } @@ -113,7 +114,9 @@ export class WritableOperationStream { } _closeIgnoringLock() { - this._checkState(); + if (!writableAcceptsWriteAndClose(this._state)) { + throw new TypeError('already ' + this._state); + } const result = this._sink.close(); From 5eb050b7842adadb5df26182e876614e133dd3e5 Mon Sep 17 00:00:00 2001 From: tyoshino Date: Sun, 1 Mar 2015 23:09:12 +0900 Subject: [PATCH 73/93] Reorganize promise sync methods --- .../exclusive-operation-stream-writer.js | 46 +++++++-------- .../experimental/writable-operation-stream.js | 58 +++++++++++-------- 2 files changed, 56 insertions(+), 48 deletions(-) diff --git a/reference-implementation/lib/experimental/exclusive-operation-stream-writer.js b/reference-implementation/lib/experimental/exclusive-operation-stream-writer.js index fd532c64d..9929da113 100644 --- a/reference-implementation/lib/experimental/exclusive-operation-stream-writer.js +++ b/reference-implementation/lib/experimental/exclusive-operation-stream-writer.js @@ -8,24 +8,16 @@ export class ExclusiveOperationStreamWriter { constructor(parent) { this._parent = parent; - const state = this._parent._state; - - if (state === 'writable') { - this._writablePromise = Promise.resolve(); - } else { - this._initWritablePromise(); - } - - if (state === 'errored' || state === 'cancelled') { - this._erroredPromise = Promise.resolve(); - } else { - this._erroredPromise = new Promise((resolve, reject) => { - this._resolveErroredPromise = resolve; - }); - } + this._initWritablePromise(); + this._erroredPromise = new Promise((resolve, reject) => { + this._resolveErroredPromise = resolve; + }); this._lastSpace = undefined; this._spaceChangePromise = undefined; + + this._syncStateAndWritablePromise(); + this._syncStateAndErroredPromise(); } _throwIfReleased() { @@ -52,29 +44,33 @@ export class ExclusiveOperationStreamWriter { return this._parent._cancelOperationIgnoringLock; } - _syncStateAndPromises() { - const state = this._parent._state; - if (state === 'waiting') { - if (this._resolveWritablePromise === undefined) { - this._initWritablePromise(); - } - } else if (state === 'writable') { + _syncStateAndWritablePromise() { + if (this._parent._state === 'writable') { if (this._resolveWritablePromise !== undefined) { this._resolveWritablePromise(); this._resolveWritablePromise = undefined; } - } else if (state === 'cancelled' || state === 'errored') { + } else { + if (this._resolveWritablePromise === undefined) { + this._initWritablePromise(); + } + } + } + + _syncStateAndErroredPromise() { + if (this._parent._state === 'cancelled' || this._parent._state === 'errored') { this._resolveErroredPromise(); + this._resolveErroredPromise = undefined; } } - _onSpaceChange() { + _syncSpaceAndSpaceChangePromise() { if (this._spaceChangePromise !== undefined && this._lastSpace !== this.space) { this._resolveSpaceChangePromise(); + this._resolveSpaceChangePromise = undefined; this._lastSpace = undefined; this._spaceChangePromise = undefined; - this._resolveSpaceChangePromise = undefined; } } diff --git a/reference-implementation/lib/experimental/writable-operation-stream.js b/reference-implementation/lib/experimental/writable-operation-stream.js index e9c3b5e96..c26c7fddb 100644 --- a/reference-implementation/lib/experimental/writable-operation-stream.js +++ b/reference-implementation/lib/experimental/writable-operation-stream.js @@ -145,26 +145,42 @@ export class WritableOperationStream { return this._abortIgnoringLock(reason); } - _syncStateAndPromises() { - if (this._state === 'waiting') { - if (this._resolveWritablePromise === undefined) { - this._initWritablePromise(); - } - } else if (this._state === 'writable') { + _syncStateAndWritablePromise() { + if (this._state === 'writable') { if (this._resolveWritablePromise !== undefined) { this._resolveWritablePromise(); this._resolveWritablePromise = undefined; } - } else if (this._state === 'cancelled' || this._state === 'errored') { + } else { + if (this._resolveWritablePromise === undefined) { + this._initWritablePromise(); + } + } + } + + _syncStateAndErroredPromise() { + if (this._resolveErroredPromise !== undefined) { this._resolveErroredPromise(); + this._resolveErroredPromise = undefined; + } + } + + _syncSpaceAndSpaceChangePromise() { + if (this._spaceChangePromise !== undefined && this._lastSpace !== this.space) { + this._resolveSpaceChangePromise(); + this._resolveSpaceChangePromise = undefined; + + this._lastSpace = undefined; + this._spaceChangePromise = undefined; } } _releaseWriter() { this._writer = undefined; - this._syncStateAndPromises(); - this._onSpaceChange(); + this._syncStateAndWritablePromise(); + this._syncStateAndErroredPromise(); + this._syncSpaceAndSpaceChangePromise(); } getWriter() { @@ -179,9 +195,9 @@ export class WritableOperationStream { this._state = 'waiting'; if (this._writer === undefined) { - this._syncStateAndPromises(); + this._syncStateAndWritablePromise(); } else { - this._writer._syncStateAndPromises(); + this._writer._syncStateAndWritablePromise(); } } @@ -189,9 +205,9 @@ export class WritableOperationStream { this._state = 'writable'; if (this._writer === undefined) { - this._syncStateAndPromises(); + this._syncStateAndWritablePromise(); } else { - this._writer._syncStateAndPromises(); + this._writer._syncStateAndWritablePromise(); } } @@ -199,9 +215,11 @@ export class WritableOperationStream { this._state = 'cancelled'; if (this._writer === undefined) { - this._syncStateAndPromises(); + this._syncStateAndWritablePromise(); + this._syncStateAndErroredPromise(); } else { - this._writer._syncStateAndPromises(); + this._writer._syncStateAndWritablePromise(); + this._writer._syncStateAndErroredPromise(); } this._cancelOperation = operation; @@ -209,15 +227,9 @@ export class WritableOperationStream { _onSpaceChange() { if (this._writer === undefined) { - if (this._spaceChangePromise !== undefined && this._lastSpace !== this.space) { - this._resolveSpaceChangePromise(); - - this._lastSpace = undefined; - this._spaceChangePromise = undefined; - this._resolveSpaceChangePromise = undefined; - } + this._syncSpaceAndSpaceChangePromise(); } else { - this._writer._onSpaceChange(); + this._writer._syncSpaceAndSpaceChangePromise(); } } } From 9604a12dc7647dfa88dc0001f7b471579f992416 Mon Sep 17 00:00:00 2001 From: tyoshino Date: Sun, 1 Mar 2015 23:34:47 +0900 Subject: [PATCH 74/93] Reorganize methods based on their purpose --- .../exclusive-operation-stream-writer.js | 109 ++++++++------ .../experimental/writable-operation-stream.js | 137 ++++++++++-------- 2 files changed, 135 insertions(+), 111 deletions(-) diff --git a/reference-implementation/lib/experimental/exclusive-operation-stream-writer.js b/reference-implementation/lib/experimental/exclusive-operation-stream-writer.js index 9929da113..65bb48b50 100644 --- a/reference-implementation/lib/experimental/exclusive-operation-stream-writer.js +++ b/reference-implementation/lib/experimental/exclusive-operation-stream-writer.js @@ -5,14 +5,44 @@ export class ExclusiveOperationStreamWriter { }); } + _syncStateAndWritablePromise() { + if (this._parent._state === 'writable') { + if (this._resolveWritablePromise !== undefined) { + this._resolveWritablePromise(); + this._resolveWritablePromise = undefined; + } + } else { + if (this._resolveWritablePromise === undefined) { + this._initWritablePromise(); + } + } + } + + _syncStateAndErroredPromise() { + if (this._parent._state === 'cancelled' || this._parent._state === 'errored') { + this._resolveErroredPromise(); + this._resolveErroredPromise = undefined; + } + } + + _syncSpaceAndSpaceChangePromise() { + if (this._spaceChangePromise !== undefined && this._lastSpace !== this.space) { + this._resolveSpaceChangePromise(); + this._resolveSpaceChangePromise = undefined; + + this._lastSpace = undefined; + this._spaceChangePromise = undefined; + } + } + constructor(parent) { this._parent = parent; - this._initWritablePromise(); this._erroredPromise = new Promise((resolve, reject) => { this._resolveErroredPromise = resolve; }); + this._initWritablePromise(); this._lastSpace = undefined; this._spaceChangePromise = undefined; @@ -20,20 +50,28 @@ export class ExclusiveOperationStreamWriter { this._syncStateAndErroredPromise(); } - _throwIfReleased() { - if (this._parent === undefined) { - throw new TypeError('already released'); - } - } - get state() { this._throwIfReleased(); return this._parent._state; } - get writable() { + + // Main interfaces. + + write(argument) { this._throwIfReleased(); - return this._writablePromise; + return this._parent._writeIgnoringLock(argument); + } + close() { + this._throwIfReleased(); + return this._parent._closeIgnoringLock(); + } + abort(reason) { + this._throwIfReleased(); + return this._parent._abortIgnoringLock(reason); } + + // Error receiving interfaces. + get errored() { this._throwIfReleased(); return this._erroredPromise; @@ -44,34 +82,11 @@ export class ExclusiveOperationStreamWriter { return this._parent._cancelOperationIgnoringLock; } - _syncStateAndWritablePromise() { - if (this._parent._state === 'writable') { - if (this._resolveWritablePromise !== undefined) { - this._resolveWritablePromise(); - this._resolveWritablePromise = undefined; - } - } else { - if (this._resolveWritablePromise === undefined) { - this._initWritablePromise(); - } - } - } - - _syncStateAndErroredPromise() { - if (this._parent._state === 'cancelled' || this._parent._state === 'errored') { - this._resolveErroredPromise(); - this._resolveErroredPromise = undefined; - } - } - - _syncSpaceAndSpaceChangePromise() { - if (this._spaceChangePromise !== undefined && this._lastSpace !== this.space) { - this._resolveSpaceChangePromise(); - this._resolveSpaceChangePromise = undefined; + // Flow control interfaces. - this._lastSpace = undefined; - this._spaceChangePromise = undefined; - } + get writable() { + this._throwIfReleased(); + return this._writablePromise; } get space() { @@ -93,17 +108,12 @@ export class ExclusiveOperationStreamWriter { return this._spaceChangePromise; } - write(argument) { - this._throwIfReleased(); - return this._parent._writeIgnoringLock(argument); - } - close() { - this._throwIfReleased(); - return this._parent._closeIgnoringLock(); - } - abort(reason) { - this._throwIfReleased(); - return this._parent._abortIgnoringLock(reason); + // Locking interfaces. + + _throwIfReleased() { + if (this._parent === undefined) { + throw new TypeError('already released'); + } } release() { @@ -111,8 +121,11 @@ export class ExclusiveOperationStreamWriter { this._parent = undefined; - this._writablePromise = undefined; + // Make promises collectable. + this._erroredPromise = undefined; + + this._writablePromise = undefined; this._spaceChangePromise = undefined; } } diff --git a/reference-implementation/lib/experimental/writable-operation-stream.js b/reference-implementation/lib/experimental/writable-operation-stream.js index c26c7fddb..f83132977 100644 --- a/reference-implementation/lib/experimental/writable-operation-stream.js +++ b/reference-implementation/lib/experimental/writable-operation-stream.js @@ -1,31 +1,62 @@ -import { Operation, OperationStatus, writableAcceptsWriteAndClose, writableAcceptsAbort } from './operation-stream'; +import { writableAcceptsWriteAndClose, writableAcceptsAbort } from './operation-stream'; import { ExclusiveOperationStreamWriter } from './exclusive-operation-stream-writer'; export class WritableOperationStream { - // Public members and internal methods. - _initWritablePromise() { this._writablePromise = new Promise((resolve, reject) => { this._resolveWritablePromise = resolve; }); } + _syncStateAndWritablePromise() { + if (this._state === 'writable') { + if (this._resolveWritablePromise !== undefined) { + this._resolveWritablePromise(); + this._resolveWritablePromise = undefined; + } + } else { + if (this._resolveWritablePromise === undefined) { + this._initWritablePromise(); + } + } + } + + _syncStateAndErroredPromise() { + if (this._state === 'cancelled' || this._state === 'errored') { + // erroredPromise may be already fulfilled if this method is called on release of a writer. + if (this._resolveErroredPromise !== undefined) { + this._resolveErroredPromise(); + this._resolveErroredPromise = undefined; + } + } + } + + _syncSpaceAndSpaceChangePromise() { + if (this._spaceChangePromise !== undefined && this._lastSpace !== this.space) { + this._resolveSpaceChangePromise(); + this._resolveSpaceChangePromise = undefined; + + this._lastSpace = undefined; + this._spaceChangePromise = undefined; + } + } + + // Public members and internal methods. + constructor(sink, f) { this._sink = sink; this._state = 'waiting'; - this._initWritablePromise(); - this._erroredPromise = new Promise((resolve, reject) => { this._resolveErroredPromise = resolve; }); + this._cancelOperation = undefined; + this._initWritablePromise(); this._lastSpace = undefined; this._spaceChangePromise = undefined; - this._cancelOperation = undefined; - this._writer = undefined; const delegate = { @@ -38,47 +69,12 @@ export class WritableOperationStream { f(delegate); } - _throwIfLocked() { - if (this._writer !== undefined) { - throw new TypeError('locked'); - } - } - get state() { this._throwIfLocked(); return this._state; } - get writable() { - this._throwIfLocked(); - return this._writablePromise; - } - get errored() { - this._throwIfLocked(); - return this._erroredPromise; - } - get _cancelOperationIgnoringLock() { - if (this._state !== 'cancelled') { - throw new TypeError('not cancelled'); - } - return this._cancelOperation; - } - get cancelOperation() { - this._throwIfLocked(); - return this._cancelOperationIgnoringLock; - } - - get _spaceIgnoringLock() { - if (!writableAcceptsWriteAndClose(this._state)) { - throw new TypeError('already ' + this._state); - } - - return this._sink.space; - } - get space() { - this._throwIfLocked(); - return this._spaceIgnoringLock; - } + // Main interfaces. _waitSpaceChangeIgnoringLock() { if (!writableAcceptsWriteAndClose(this._state)) { @@ -145,33 +141,48 @@ export class WritableOperationStream { return this._abortIgnoringLock(reason); } - _syncStateAndWritablePromise() { - if (this._state === 'writable') { - if (this._resolveWritablePromise !== undefined) { - this._resolveWritablePromise(); - this._resolveWritablePromise = undefined; - } - } else { - if (this._resolveWritablePromise === undefined) { - this._initWritablePromise(); - } + // Error receiving interfaces. + + get errored() { + this._throwIfLocked(); + return this._erroredPromise; + } + + get _cancelOperationIgnoringLock() { + if (this._state !== 'cancelled') { + throw new TypeError('not cancelled'); } + return this._cancelOperation; + } + get cancelOperation() { + this._throwIfLocked(); + return this._cancelOperationIgnoringLock; } - _syncStateAndErroredPromise() { - if (this._resolveErroredPromise !== undefined) { - this._resolveErroredPromise(); - this._resolveErroredPromise = undefined; + // Flow control interfaces. + + get writable() { + this._throwIfLocked(); + return this._writablePromise; + } + + get _spaceIgnoringLock() { + if (!writableAcceptsWriteAndClose(this._state)) { + throw new TypeError('already ' + this._state); } + + return this._sink.space; + } + get space() { + this._throwIfLocked(); + return this._spaceIgnoringLock; } - _syncSpaceAndSpaceChangePromise() { - if (this._spaceChangePromise !== undefined && this._lastSpace !== this.space) { - this._resolveSpaceChangePromise(); - this._resolveSpaceChangePromise = undefined; + // Locking interfaces. - this._lastSpace = undefined; - this._spaceChangePromise = undefined; + _throwIfLocked() { + if (this._writer !== undefined) { + throw new TypeError('locked'); } } From 27a72de799da1a7b8db8f2097e03f66dbf45099d Mon Sep 17 00:00:00 2001 From: tyoshino Date: Sun, 1 Mar 2015 23:38:41 +0900 Subject: [PATCH 75/93] Cont'd --- .../experimental/readable-operation-stream.js | 86 ++++++++++--------- .../experimental/writable-operation-stream.js | 2 - 2 files changed, 47 insertions(+), 41 deletions(-) diff --git a/reference-implementation/lib/experimental/readable-operation-stream.js b/reference-implementation/lib/experimental/readable-operation-stream.js index 37d033f54..a7434adac 100644 --- a/reference-implementation/lib/experimental/readable-operation-stream.js +++ b/reference-implementation/lib/experimental/readable-operation-stream.js @@ -1,8 +1,6 @@ import { Operation, OperationStatus, readableAcceptsReadAndCancel } from './operation-stream'; export class ReadableOperationStream { - // Public members and internal methods. - _initReadablePromise() { this._readablePromise = new Promise((resolve, reject) => { this._resolveReadablePromise = resolve; @@ -36,20 +34,54 @@ export class ReadableOperationStream { f(delegate); } - _throwIfLocked() { - if (this._reader !== undefined) { - throw new TypeError('locked'); - } - } - get state() { this._throwIfLocked(); return this._state; } + + // Main interfaces. + get readable() { this._throwIfLocked(); return this._readablePromise; } + + _readIgnoringLock() { + if (!readableAcceptsReadAndCancel(this._state)) { + throw new TypeError('already ' + this._state); + } + + return this._source.read(); + } + read() { + this._throwIfLocked(); + return this._readIgnoringLock(); + } + + _cancelIgnoringLock(reason) { + if (!readableAcceptsReadAndCancel(this._state)) { + throw new TypeError('already ' + this._state); + } + + const result = this._source.cancel(reason); + + if (this._reader !== undefined) { + this._reader._resolveErroredPromise(); + } else { + this._resolveErroredPromise(); + } + + this._state = 'cancelled'; + + return result; + } + cancel(reason) { + this._throwIfLocked(); + return this._cancelIgnoringLock(reason); + } + + // Error receiving interfaces. + get errored() { this._throwIfLocked(); return this._erroredPromise; @@ -66,11 +98,7 @@ export class ReadableOperationStream { return this._abortOperationIgnoringLock; } - _checkState() { - if (!readableAcceptsReadAndCancel(this._state)) { - throw new TypeError('already ' + this._state); - } - } + // Flow control interfaces. get _windowIgnoringLock() { return this._window; @@ -81,7 +109,9 @@ export class ReadableOperationStream { } set _windowIgnoringLock(v) { - this._checkState(); + if (!readableAcceptsReadAndCancel(this._state)) { + throw new TypeError('already ' + this._state); + } this._window = v; @@ -92,34 +122,12 @@ export class ReadableOperationStream { this._windowIgnoringLock = v; } - _readIgnoringLock() { - this._checkState(); - - return this._source.read(); - } - read() { - this._throwIfLocked(); - return this._readIgnoringLock(); - } - - _cancelIgnoringLock(reason) { - this._checkState(); - - const result = this._source.cancel(reason); + // Locking interfaces. + _throwIfLocked() { if (this._reader !== undefined) { - this._reader._resolveErroredPromise(); - } else { - this._resolveErroredPromise(); + throw new TypeError('locked'); } - - this._state = 'cancelled'; - - return result; - } - cancel(reason) { - this._throwIfLocked(); - return this._cancelIgnoringLock(reason); } getReader() { diff --git a/reference-implementation/lib/experimental/writable-operation-stream.js b/reference-implementation/lib/experimental/writable-operation-stream.js index f83132977..8d60a8b3e 100644 --- a/reference-implementation/lib/experimental/writable-operation-stream.js +++ b/reference-implementation/lib/experimental/writable-operation-stream.js @@ -41,8 +41,6 @@ export class WritableOperationStream { } } - // Public members and internal methods. - constructor(sink, f) { this._sink = sink; From ce301896f98a5c51a0d7630bd292d189502bbc73 Mon Sep 17 00:00:00 2001 From: tyoshino Date: Mon, 2 Mar 2015 00:09:56 +0900 Subject: [PATCH 76/93] Split OperationQueue --- .../lib/experimental/operation-queue.js | 93 +++++++++++++------ .../experimental/readable-operation-stream.js | 4 +- .../experimental/writable-operation-stream.js | 4 +- .../test/experimental/operation-stream.js | 46 +++++---- 4 files changed, 90 insertions(+), 57 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-queue.js b/reference-implementation/lib/experimental/operation-queue.js index c6935fbd6..eddf3a375 100644 --- a/reference-implementation/lib/experimental/operation-queue.js +++ b/reference-implementation/lib/experimental/operation-queue.js @@ -5,8 +5,52 @@ import { ReadableOperationStream } from './readable-operation-stream'; // Creates a pair of WritableOperationStream implementation and ReadableOperationStream implementation that are // connected with a queue. This can be used for creating queue-backed operation streams. export function createOperationQueue(strategy) { - const queue = new OperationQueue(strategy); - return { writable: queue.writable, readable: queue.readable }; + const source = new OperationQueueUnderlyingSource(); + const sink = new OperationQueue(strategy); + + source.setSink(sink); + sink.setSource(source); + + return { writable: new WritableOperationStream(sink), readable: new ReadableOperationStream(source) }; +} + +class OperationQueueUnderlyingSource { + setSink(sink) { + this._sink = sink; + } + + init(delegate) { + this._delegate = delegate; + } + + abort(reason) { + this._delegate.markAborted(reason); + } + + onWindowUpdate(v) { + this._sink.onWindowUpdate(v); + } + + read() { + return this._sink.read(); + } + + cancel(reason) { + return this._sink.cancel(reason); + } + + markWaiting() { + this._delegate.markWaiting(); + } + markReadable() { + this._delegate.markReadable(); + } + markDrained() { + this._delegate.markDrained(); + } + markAborted(reason) { + this._delegate.markAborted(reason); + } } class OperationQueue { @@ -20,12 +64,12 @@ class OperationQueue { shouldApplyBackpressure = this._strategy.shouldApplyBackpressure(this._queueSize); } if (shouldApplyBackpressure) { - this._writableStreamDelegate.markWaiting(); + this._delegate.markWaiting(); } else { - this._writableStreamDelegate.markWritable(); + this._delegate.markWritable(); } - this._writableStreamDelegate.onSpaceChange(); + this._delegate.onSpaceChange(); } constructor(strategy) { @@ -33,23 +77,8 @@ class OperationQueue { this._queueSize = 0; this._strategy = strategy; - - this._writableStream = new WritableOperationStream(this, delegate => this._writableStreamDelegate = delegate); - this._readableStream = new ReadableOperationStream(this, delegate => this._readableStreamDelegate = delegate); - - this._updateWritableStream(); } - get writable() { - return this._writableStream; - } - - get readable() { - return this._readableStream; - } - - // Underlying sink implementation. - get space() { if (this._strategy.space !== undefined) { return this._strategy.space(this._queueSize); @@ -58,6 +87,16 @@ class OperationQueue { return undefined; } + setSource(source) { + this._source = source; + } + + init(delegate) { + this._delegate = delegate; + + this._updateWritableStream(); + } + write(value) { const operationStatus = new OperationStatus(); const operation = new Operation('data', value, operationStatus); @@ -72,7 +111,7 @@ class OperationQueue { this._updateWritableStream(); - this._readableStreamDelegate.markReadable(); + this._source.markReadable(); return operationStatus; } @@ -86,7 +125,7 @@ class OperationQueue { // No longer necessary. this._strategy = undefined; - this._readableStreamDelegate.markReadable(); + this._source.markReadable(); return operationStatus; } @@ -103,13 +142,11 @@ class OperationQueue { this._strategy = undefined; - this._readableStreamDelegate.markAborted(operation); + this._source.abort(operation); return operationStatus; } - // Underlying source implementation. - onWindowUpdate(v) { if (this._strategy === undefined) { return; @@ -132,9 +169,9 @@ class OperationQueue { if (this._queue.length === 0) { if (entry.value.type === 'close') { - this._readableStreamDelegate.markDrained(); + this._source.markDrained(); } else { - this._readableStreamDelegate.markWaiting(); + this._source.markWaiting(); } } @@ -155,7 +192,7 @@ class OperationQueue { this._strategy = undefined; - this._writableStreamDelegate.markCancelled(operation); + this._delegate.markCancelled(operation); return operationStatus; } diff --git a/reference-implementation/lib/experimental/readable-operation-stream.js b/reference-implementation/lib/experimental/readable-operation-stream.js index a7434adac..1c6c022c8 100644 --- a/reference-implementation/lib/experimental/readable-operation-stream.js +++ b/reference-implementation/lib/experimental/readable-operation-stream.js @@ -7,7 +7,7 @@ export class ReadableOperationStream { }); } - constructor(source, f) { + constructor(source) { this._source = source; this._state = 'waiting'; @@ -31,7 +31,7 @@ export class ReadableOperationStream { markAborted: this._markAborted.bind(this) }; - f(delegate); + this._source.init(delegate); } get state() { diff --git a/reference-implementation/lib/experimental/writable-operation-stream.js b/reference-implementation/lib/experimental/writable-operation-stream.js index 8d60a8b3e..240ec492b 100644 --- a/reference-implementation/lib/experimental/writable-operation-stream.js +++ b/reference-implementation/lib/experimental/writable-operation-stream.js @@ -41,7 +41,7 @@ export class WritableOperationStream { } } - constructor(sink, f) { + constructor(sink) { this._sink = sink; this._state = 'waiting'; @@ -64,7 +64,7 @@ export class WritableOperationStream { onSpaceChange: this._onSpaceChange.bind(this) }; - f(delegate); + this._sink.init(delegate); } get state() { diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 4dc5aa00b..c92aee7a1 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -1202,37 +1202,33 @@ class FakeUnstoppablePushSource { test('Adapting unstoppable push source', t => { class Source { constructor() { - this._pushSource = new FakeUnstoppablePushSource(10); - this._queue = []; + } - this._readableStream = new ReadableOperationStream(this, delegate => { - t.equal(typeof delegate.markWaiting, 'function', 'markWaiting is a function'); - t.equal(typeof delegate.markReadable, 'function', 'markReadable is a function'); - t.equal(typeof delegate.markDrained, 'function', 'markDrained is a function'); - t.equal(typeof delegate.markAborted, 'function', 'markAborted is a function'); + init(delegate) { + this._pushSource = new FakeUnstoppablePushSource(10); - this._readableStreamDelegate = delegate; + t.equal(typeof delegate.markWaiting, 'function', 'markWaiting is a function'); + t.equal(typeof delegate.markReadable, 'function', 'markReadable is a function'); + t.equal(typeof delegate.markDrained, 'function', 'markDrained is a function'); + t.equal(typeof delegate.markAborted, 'function', 'markAborted is a function'); - this._pushSource.ondata = chunk => { - this._queue.push({type: 'data', data: chunk}); - delegate.markReadable(); - }; + this._readableStreamDelegate = delegate; - this._pushSource.onend = () => { - this._queue.push({type: 'close'}); - delegate.markReadable(); - }; + this._pushSource.ondata = chunk => { + this._queue.push({type: 'data', data: chunk}); + delegate.markReadable(); + }; - this._pushSource.onerror = () => { - this._queue = []; - delegate.markAborted(); - }; - }); - } + this._pushSource.onend = () => { + this._queue.push({type: 'close'}); + delegate.markReadable(); + }; - get readableStream() { - return this._readableStream; + this._pushSource.onerror = () => { + this._queue = []; + delegate.markAborted(); + }; } onWindowUpdate(v) { @@ -1266,7 +1262,7 @@ test('Adapting unstoppable push source', t => { } const source = new Source(); - const readableStream = source.readableStream; + const readableStream = new ReadableOperationStream(source); let count = 0; function pump() { From 03e57d91b439a83fcb03c2f59fb13686134b5330 Mon Sep 17 00:00:00 2001 From: tyoshino Date: Mon, 2 Mar 2015 00:11:58 +0900 Subject: [PATCH 77/93] More reorg --- .../lib/experimental/operation-queue.js | 40 +++++++++---------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-queue.js b/reference-implementation/lib/experimental/operation-queue.js index eddf3a375..7cfc67bd2 100644 --- a/reference-implementation/lib/experimental/operation-queue.js +++ b/reference-implementation/lib/experimental/operation-queue.js @@ -23,6 +23,10 @@ class OperationQueueUnderlyingSource { this._delegate = delegate; } + markReadable() { + this._delegate.markReadable(); + } + abort(reason) { this._delegate.markAborted(reason); } @@ -31,25 +35,19 @@ class OperationQueueUnderlyingSource { this._sink.onWindowUpdate(v); } - read() { - return this._sink.read(); - } - - cancel(reason) { - return this._sink.cancel(reason); - } - markWaiting() { this._delegate.markWaiting(); } - markReadable() { - this._delegate.markReadable(); - } markDrained() { this._delegate.markDrained(); } - markAborted(reason) { - this._delegate.markAborted(reason); + + read() { + return this._sink.read(); + } + + cancel(reason) { + return this._sink.cancel(reason); } } @@ -79,14 +77,6 @@ class OperationQueue { this._strategy = strategy; } - get space() { - if (this._strategy.space !== undefined) { - return this._strategy.space(this._queueSize); - } - - return undefined; - } - setSource(source) { this._source = source; } @@ -97,6 +87,14 @@ class OperationQueue { this._updateWritableStream(); } + get space() { + if (this._strategy.space !== undefined) { + return this._strategy.space(this._queueSize); + } + + return undefined; + } + write(value) { const operationStatus = new OperationStatus(); const operation = new Operation('data', value, operationStatus); From 7f252bac9771595ee55f550a2e03578160c50a20 Mon Sep 17 00:00:00 2001 From: tyoshino Date: Mon, 2 Mar 2015 01:04:56 +0900 Subject: [PATCH 78/93] More --- .../lib/experimental/operation-queue.js | 164 +++++++++--------- 1 file changed, 83 insertions(+), 81 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-queue.js b/reference-implementation/lib/experimental/operation-queue.js index 7cfc67bd2..c310f6b7e 100644 --- a/reference-implementation/lib/experimental/operation-queue.js +++ b/reference-implementation/lib/experimental/operation-queue.js @@ -5,61 +5,23 @@ import { ReadableOperationStream } from './readable-operation-stream'; // Creates a pair of WritableOperationStream implementation and ReadableOperationStream implementation that are // connected with a queue. This can be used for creating queue-backed operation streams. export function createOperationQueue(strategy) { - const source = new OperationQueueUnderlyingSource(); - const sink = new OperationQueue(strategy); - - source.setSink(sink); + const queue = new OperationQueue(strategy); + const source = new OperationQueueUnderlyingSource(queue); + const sink = new OperationQueueUnderlyingSink(queue); sink.setSource(source); - + source.setSink(sink); return { writable: new WritableOperationStream(sink), readable: new ReadableOperationStream(source) }; } -class OperationQueueUnderlyingSource { - setSink(sink) { - this._sink = sink; - } - - init(delegate) { - this._delegate = delegate; - } - - markReadable() { - this._delegate.markReadable(); - } - - abort(reason) { - this._delegate.markAborted(reason); - } - - onWindowUpdate(v) { - this._sink.onWindowUpdate(v); - } - - markWaiting() { - this._delegate.markWaiting(); - } - markDrained() { - this._delegate.markDrained(); - } - - read() { - return this._sink.read(); - } - - cancel(reason) { - return this._sink.cancel(reason); - } -} - -class OperationQueue { +class OperationQueueUnderlyingSink { _updateWritableStream() { - if (this._strategy === undefined) { + if (this._queue._strategy === undefined) { return; } let shouldApplyBackpressure = false; - if (this._strategy.shouldApplyBackpressure !== undefined) { - shouldApplyBackpressure = this._strategy.shouldApplyBackpressure(this._queueSize); + if (this._queue._strategy.shouldApplyBackpressure !== undefined) { + shouldApplyBackpressure = this._queue._strategy.shouldApplyBackpressure(this._queue._queueSize); } if (shouldApplyBackpressure) { this._delegate.markWaiting(); @@ -70,11 +32,8 @@ class OperationQueue { this._delegate.onSpaceChange(); } - constructor(strategy) { - this._queue = []; - this._queueSize = 0; - - this._strategy = strategy; + constructor(queue) { + this._queue = queue; } setSource(source) { @@ -88,8 +47,8 @@ class OperationQueue { } get space() { - if (this._strategy.space !== undefined) { - return this._strategy.space(this._queueSize); + if (this._queue._strategy.space !== undefined) { + return this._queue._strategy.space(this._queue._queueSize); } return undefined; @@ -100,16 +59,16 @@ class OperationQueue { const operation = new Operation('data', value, operationStatus); var size = 1; - if (this._strategy.size !== undefined) { - size = this._strategy.size(operation.argument); + if (this._queue._strategy.size !== undefined) { + size = this._queue._strategy.size(operation.argument); } - this._queue.push({value: operation, size}); - this._queueSize += size; + this._queue._queue.push({value: operation, size}); + this._queue._queueSize += size; this._updateWritableStream(); - this._source.markReadable(); + this._source.onQueueFill(); return operationStatus; } @@ -118,12 +77,12 @@ class OperationQueue { const operationStatus = new OperationStatus(); const operation = new Operation('close', ReadableOperationStream.EOS, operationStatus); - this._queue.push({value: operation, size: 0}); + this._queue._queue.push({value: operation, size: 0}); // No longer necessary. - this._strategy = undefined; + this._queue._strategy = undefined; - this._source.markReadable(); + this._source.onQueueFill(); return operationStatus; } @@ -132,48 +91,82 @@ class OperationQueue { const operationStatus = new OperationStatus(); const operation = new Operation('abort', reason, operationStatus); - for (var i = this._queue.length - 1; i >= 0; --i) { - const op = this._queue[i].value; + for (var i = this._queue._queue.length - 1; i >= 0; --i) { + const op = this._queue._queue[i].value; op.error(new TypeError('aborted')); } - this._queue = []; + this._queue._queue = []; - this._strategy = undefined; + this._queue._strategy = undefined; this._source.abort(operation); return operationStatus; } + onWindowUpdate() { + this._updateWritableStream(); + } + + onQueueConsume() { + this._updateWritableStream(); + } + + onCancel(reason) { + this._delegate.markCancelled(reason); + } +} + +class OperationQueueUnderlyingSource { + constructor(queue) { + this._queue = queue; + } + + setSink(sink) { + this._sink = sink; + } + + init(delegate) { + this._delegate = delegate; + } + + onQueueFill() { + this._delegate.markReadable(); + } + + abort(reason) { + this._delegate.markAborted(reason); + } + onWindowUpdate(v) { - if (this._strategy === undefined) { + if (this._queue._strategy === undefined) { return; } - if (this._strategy.onWindowUpdate !== undefined) { - this._strategy.onWindowUpdate(v); + if (this._queue._strategy.onWindowUpdate !== undefined) { + this._queue._strategy.onWindowUpdate(v); } - this._updateWritableStream(); + this._sink.onWindowUpdate(); } read() { - if (this._queue.length === 0) { + if (this._queue._queue.length === 0) { throw new TypeError('not readable'); } - const entry = this._queue.shift(); - this._queueSize -= entry.size; + const entry = this._queue._queue.shift(); + this._queue._queueSize -= entry.size; - if (this._queue.length === 0) { + if (this._queue._queue.length === 0) { if (entry.value.type === 'close') { - this._source.markDrained(); + this._delegate.markDrained(); } else { - this._source.markWaiting(); + this._delegate.markWaiting(); } } - this._updateWritableStream(); + this._sink.onQueueConsume(); return entry.value; } @@ -182,16 +175,25 @@ class OperationQueue { const operationStatus = new OperationStatus(); const operation = new Operation('cancel', reason, operationStatus); - for (var i = 0; i < this._queue.length; ++i) { - const op = this._queue[i].value; + for (var i = 0; i < this._queue._queue.length; ++i) { + const op = this._queue._queue[i].value; op.error(operation.argument); } - this._queue = []; + this._queue._queue = []; - this._strategy = undefined; + this._queue._strategy = undefined; - this._delegate.markCancelled(operation); + this._sink.onCancel(operation); return operationStatus; } } + +class OperationQueue { + constructor(strategy) { + this._queue = []; + this._queueSize = 0; + + this._strategy = strategy; + } +} From 19de1e09a60a9bb232b934375fbe85fc12ee2514 Mon Sep 17 00:00:00 2001 From: tyoshino Date: Mon, 2 Mar 2015 01:38:21 +0900 Subject: [PATCH 79/93] Remove cancelled state from Writable and aborted state from Readable --- .../lib/experimental/operation-queue.js | 4 +- .../lib/experimental/operation-stream.js | 34 ++++------- .../experimental/readable-operation-stream.js | 20 +++---- .../experimental/writable-operation-stream.js | 24 ++++---- .../test/experimental/operation-stream.js | 60 +++++++++---------- 5 files changed, 65 insertions(+), 77 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-queue.js b/reference-implementation/lib/experimental/operation-queue.js index c310f6b7e..67786e465 100644 --- a/reference-implementation/lib/experimental/operation-queue.js +++ b/reference-implementation/lib/experimental/operation-queue.js @@ -113,7 +113,7 @@ class OperationQueueUnderlyingSink { } onCancel(reason) { - this._delegate.markCancelled(reason); + this._delegate.markErrored(reason); } } @@ -135,7 +135,7 @@ class OperationQueueUnderlyingSource { } abort(reason) { - this._delegate.markAborted(reason); + this._delegate.markErrored(reason); } onWindowUpdate(v) { diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 64b751d10..c9ec0528d 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -81,35 +81,27 @@ export function pipeOperationStreams(source, dest) { // Propagate errors. - if (source.state === 'aborted') { - if (writableAcceptsAbort(dest.state)) { - jointOps(source.abortOperation, dest.abort(source.abortOperation.argument)); - } - reject(new TypeError('aborted')); - return; - } if (source.state === 'errored') { - const error = new TypeError('source is errored'); if (writableAcceptsAbort(dest.state)) { - dest.abort(error); + if (source.error.constructor === Operation) { + jointOps(source.error, dest.abort(source.error.argument)); + } else { + dest.abort(source.error); + } } - reject(error); + reject(new TypeError('source is errored')); return; } - if (dest.state === 'cancelled') { + if (dest.state === 'errored') { if (readableAcceptsReadAndCancel(source.state)) { - jointOps(dest.cancelOperation, source.cancel(dest.cancelOperation.argument)); - } - reject(new TypeError('dest is cancelled')); - return; - } - if (source.state === 'errored') { - const error = new TypeError('dest is errored'); - if (readableAcceptsReadAndCancel(source.state)) { - source.cancel(error); + if (dest.error.constructor === Operation) { + jointOps(dest.error, source.cancel(dest.error.argument)); + } else { + source.cancel(dest.error); + } } - reject(error); + reject(new TypeError('dest is errored')); return; } diff --git a/reference-implementation/lib/experimental/readable-operation-stream.js b/reference-implementation/lib/experimental/readable-operation-stream.js index 1c6c022c8..49404f441 100644 --- a/reference-implementation/lib/experimental/readable-operation-stream.js +++ b/reference-implementation/lib/experimental/readable-operation-stream.js @@ -28,7 +28,7 @@ export class ReadableOperationStream { markWaiting: this._markWaiting.bind(this), markReadable: this._markReadable.bind(this), markDrained: this._markDrained.bind(this), - markAborted: this._markAborted.bind(this) + markErrored: this._markErrored.bind(this) }; this._source.init(delegate); @@ -87,15 +87,15 @@ export class ReadableOperationStream { return this._erroredPromise; } - get _abortOperationIgnoringLock() { - if (this._state !== 'aborted') { - throw new TypeError('not aborted'); + get _errorIgnoringLock() { + if (this._state !== 'errored') { + throw new TypeError('not errored'); } - return this._abortOperation; + return this._error; } - get abortOperation() { + get error() { this._throwIfLocked(); - return this._abortOperationIgnoringLock; + return this._errorIgnoringLock; } // Flow control interfaces. @@ -166,16 +166,16 @@ export class ReadableOperationStream { this._state = 'drained'; } - _markAborted(operation) { + _markErrored(error) { if (this._reader === undefined) { this._resolveErroredPromise(); } else { this._writer._resolveErroredPromise(); } - this._state = 'aborted'; + this._state = 'errored'; - this._abortOperation = operation; + this._error = error; } } diff --git a/reference-implementation/lib/experimental/writable-operation-stream.js b/reference-implementation/lib/experimental/writable-operation-stream.js index 240ec492b..ed6fb9d78 100644 --- a/reference-implementation/lib/experimental/writable-operation-stream.js +++ b/reference-implementation/lib/experimental/writable-operation-stream.js @@ -22,7 +22,7 @@ export class WritableOperationStream { } _syncStateAndErroredPromise() { - if (this._state === 'cancelled' || this._state === 'errored') { + if (this._state === 'errored') { // erroredPromise may be already fulfilled if this method is called on release of a writer. if (this._resolveErroredPromise !== undefined) { this._resolveErroredPromise(); @@ -49,7 +49,7 @@ export class WritableOperationStream { this._erroredPromise = new Promise((resolve, reject) => { this._resolveErroredPromise = resolve; }); - this._cancelOperation = undefined; + this._error = undefined; this._initWritablePromise(); this._lastSpace = undefined; @@ -60,7 +60,7 @@ export class WritableOperationStream { const delegate = { markWaiting: this._markWaiting.bind(this), markWritable: this._markWritable.bind(this), - markCancelled: this._markCancelled.bind(this), + markErrored: this._markErrored.bind(this), onSpaceChange: this._onSpaceChange.bind(this) }; @@ -146,15 +146,15 @@ export class WritableOperationStream { return this._erroredPromise; } - get _cancelOperationIgnoringLock() { - if (this._state !== 'cancelled') { - throw new TypeError('not cancelled'); + get _errorIgnoringLock() { + if (this._state !== 'errored') { + throw new TypeError('not errored'); } - return this._cancelOperation; + return this._error; } - get cancelOperation() { + get error() { this._throwIfLocked(); - return this._cancelOperationIgnoringLock; + return this._errorIgnoringLock; } // Flow control interfaces. @@ -220,8 +220,8 @@ export class WritableOperationStream { } } - _markCancelled(operation) { - this._state = 'cancelled'; + _markErrored(error) { + this._state = 'errored'; if (this._writer === undefined) { this._syncStateAndWritablePromise(); @@ -231,7 +231,7 @@ export class WritableOperationStream { this._writer._syncStateAndErroredPromise(); } - this._cancelOperation = operation; + this._error = error; } _onSpaceChange() { diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index c92aee7a1..9bd29e75a 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -222,10 +222,10 @@ test('abort()', t => { t.equals(worldStatus.result.message, 'aborted', 'worldStatus.result'); }); ros.errored.then(() => { - t.equals(ros.state, 'aborted', 'ros.state'); - t.equals(ros.abortOperation.argument, testError, 'ros.abortOperation.argument'); + t.equals(ros.state, 'errored', 'ros.state'); + t.equals(ros.error.argument, testError, 'ros.error.argument'); - ros.abortOperation.complete(testCompletion); + ros.error.complete(testCompletion); }); }); @@ -258,10 +258,10 @@ test('cancel()', t => { t.equals(worldStatus.result, testError, 'worldStatus.result'); }); wos.errored.then(() => { - t.equals(wos.state, 'cancelled', 'wos.state'); - t.equals(wos.cancelOperation.argument, testError, 'wos.cancelOperation.argument'); + t.equals(wos.state, 'errored', 'wos.state'); + t.equals(wos.error.argument, testError, 'wos.error.argument'); - wos.cancelOperation.complete(testCompletion); + wos.error.complete(testCompletion); }); }); @@ -349,7 +349,7 @@ test('pipeOperationStreams(): abort() propagation', t => { pipePromise .then( v => t.fail('pipePromise is fulfilled with ' + v), - e => t.equals(e.message, 'aborted', 'rejection reason of pipePromise')); + e => t.equals(e.message, 'source is errored', 'rejection reason of pipePromise')); const status = wos0.abort(testError); status.ready.then(() => { @@ -366,10 +366,10 @@ test('pipeOperationStreams(): abort() propagation', t => { t.equals(worldStatus.result.message, 'aborted', 'worldStatus.result'); }); ros1.errored.then(() => { - t.equals(ros1.state, 'aborted', 'ros.state'); - t.equals(ros1.abortOperation.argument, testError, 'ros1.abortOperation.argument'); + t.equals(ros1.state, 'errored', 'ros.state'); + t.equals(ros1.error.argument, testError, 'ros1.error.argument'); - ros1.abortOperation.complete(testCompletion); + ros1.error.complete(testCompletion); }); }); @@ -394,7 +394,7 @@ test('pipeOperationStreams(): cancel() propagation', t => { pipePromise .then( v => t.fail('pipePromise is fulfilled with ' + v), - e => t.equals(e.message, 'dest is cancelled', 'rejection reason of pipePromise')); + e => t.equals(e.message, 'dest is errored', 'rejection reason of pipePromise')); const status = ros1.cancel(testError); status.ready.then(() => { @@ -411,10 +411,10 @@ test('pipeOperationStreams(): cancel() propagation', t => { t.equals(worldStatus.result, testError, 'worldStatus.result'); }); wos0.errored.then(() => { - t.equals(wos0.state, 'cancelled', 'wos0.state'); - t.equals(wos0.cancelOperation.argument, testError, 'wos0.cancelOperation.argument'); + t.equals(wos0.state, 'errored', 'wos0.state'); + t.equals(wos0.error.argument, testError, 'wos0.error.argument'); - wos0.cancelOperation.complete(testCompletion); + wos0.error.complete(testCompletion); }); }); @@ -434,15 +434,15 @@ test('Transformation example: Byte counting', t => { function loop() { for (;;) { - if (source.state === 'aborted') { + if (source.state === 'errored') { if (dest.state !== 'cancelled') { - jointOps(source.abortOperation, dest.cancel(source.abortOperation.argument)); + jointOps(source.error, dest.cancel(source.error.argument)); } return; } - if (dest.state === 'cancelled') { + if (dest.state === 'errored') { if (source.state !== 'aborted') { - jointOps(dest.cancelOperation, source.abort(dest.cancelOperation.argument)); + jointOps(dest.error, source.abort(dest.error.argument)); } return; } @@ -578,7 +578,7 @@ class FakeFileBackedByteSource { const ws = this._writableStream; for (;;) { - if (ws.state === 'cancelled') { + if (ws.state === 'errored') { return; } @@ -725,9 +725,9 @@ class FakeFileBackedByteSource { const rs = this._readableStream; for (;;) { - if (rs.state === 'aborted') { + if (rs.state === 'errored') { if (this._currentRequest !== undefined) { - this._currentRequest.error(rs.abortOperation.argument); + this._currentRequest.error(rs.error.argument); } return; @@ -832,8 +832,8 @@ class BytesSetToOneExpectingByteSinkInternalWriter { rs.readable .then(this._loop.bind(this)) .catch(this._sink._error.bind(this._sink)); - } else if (rs.state === 'aborted') { - this._sink._error(rs.abortOperation.argument); + } else if (rs.state === 'errored') { + this._sink._error(rs.error.argument); } else { this._sink._error(rs.state); } @@ -932,8 +932,8 @@ class BytesSetToOneExpectingByteSink { return; } - if (ws.state === 'cancelled') { - this._sink._error(ws.cancelOperation.argument); + if (ws.state === 'errored') { + this._sink._error(ws.error.argument); return; } @@ -1082,7 +1082,7 @@ test('getWriter()', t => { t.throws(() => wos.state, /TypeError/); t.throws(() => wos.writable, /TypeError/); t.throws(() => wos.errored, /TypeError/); - t.throws(() => wos.cancelOperation, /TypeError/); + t.throws(() => wos.error, /TypeError/); t.throws(() => wos.space, /TypeError/); t.throws(() => wos.waitSpaceChange(), /TypeError/); t.throws(() => wos.write(new ArrayBuffer(10)), /TypeError/); @@ -1211,7 +1211,7 @@ test('Adapting unstoppable push source', t => { t.equal(typeof delegate.markWaiting, 'function', 'markWaiting is a function'); t.equal(typeof delegate.markReadable, 'function', 'markReadable is a function'); t.equal(typeof delegate.markDrained, 'function', 'markDrained is a function'); - t.equal(typeof delegate.markAborted, 'function', 'markAborted is a function'); + t.equal(typeof delegate.markErrored, 'function', 'markErrored is a function'); this._readableStreamDelegate = delegate; @@ -1227,7 +1227,7 @@ test('Adapting unstoppable push source', t => { this._pushSource.onerror = () => { this._queue = []; - delegate.markAborted(); + delegate.markErrored(); }; } @@ -1289,10 +1289,6 @@ test('Adapting unstoppable push source', t => { t.fail(); t.end(); return; - } else if (readableStream.state === 'aborted') { - t.fail(); - t.end(); - return; } else if (readableStream.state === 'cancelled') { t.fail(); t.end(); From 9e9fa786e96478164ead7662b9b393f2b335d8e6 Mon Sep 17 00:00:00 2001 From: tyoshino Date: Mon, 2 Mar 2015 01:51:59 +0900 Subject: [PATCH 80/93] File reorganization --- ...m-writer.js => exclusive-stream-writer.js} | 2 +- .../lib/experimental/operation-queue.js | 10 +++--- .../lib/experimental/operation-stream.js | 36 +++---------------- ...operation-stream.js => readable-stream.js} | 10 +++--- ...operation-stream.js => writable-stream.js} | 8 ++--- reference-implementation/run-tests.js | 9 ----- .../test/experimental/operation-stream.js | 13 ++++--- 7 files changed, 27 insertions(+), 61 deletions(-) rename reference-implementation/lib/experimental/{exclusive-operation-stream-writer.js => exclusive-stream-writer.js} (98%) rename reference-implementation/lib/experimental/{readable-operation-stream.js => readable-stream.js} (92%) rename reference-implementation/lib/experimental/{writable-operation-stream.js => writable-stream.js} (96%) diff --git a/reference-implementation/lib/experimental/exclusive-operation-stream-writer.js b/reference-implementation/lib/experimental/exclusive-stream-writer.js similarity index 98% rename from reference-implementation/lib/experimental/exclusive-operation-stream-writer.js rename to reference-implementation/lib/experimental/exclusive-stream-writer.js index 65bb48b50..d405328f2 100644 --- a/reference-implementation/lib/experimental/exclusive-operation-stream-writer.js +++ b/reference-implementation/lib/experimental/exclusive-stream-writer.js @@ -1,4 +1,4 @@ -export class ExclusiveOperationStreamWriter { +export class ExclusiveStreamWriter { _initWritablePromise() { this._writablePromise = new Promise((resolve, reject) => { this._resolveWritablePromise = resolve; diff --git a/reference-implementation/lib/experimental/operation-queue.js b/reference-implementation/lib/experimental/operation-queue.js index 67786e465..84e5a0517 100644 --- a/reference-implementation/lib/experimental/operation-queue.js +++ b/reference-implementation/lib/experimental/operation-queue.js @@ -1,8 +1,8 @@ import { Operation, OperationStatus } from './operation-stream'; -import { WritableOperationStream } from './writable-operation-stream'; -import { ReadableOperationStream } from './readable-operation-stream'; +import { WritableStream } from './writable-stream'; +import { ReadableStream } from './readable-stream'; -// Creates a pair of WritableOperationStream implementation and ReadableOperationStream implementation that are +// Creates a pair of WritableStream implementation and ReadableStream implementation that are // connected with a queue. This can be used for creating queue-backed operation streams. export function createOperationQueue(strategy) { const queue = new OperationQueue(strategy); @@ -10,7 +10,7 @@ export function createOperationQueue(strategy) { const sink = new OperationQueueUnderlyingSink(queue); sink.setSource(source); source.setSink(sink); - return { writable: new WritableOperationStream(sink), readable: new ReadableOperationStream(source) }; + return { writable: new WritableStream(sink), readable: new ReadableStream(source) }; } class OperationQueueUnderlyingSink { @@ -75,7 +75,7 @@ class OperationQueueUnderlyingSink { close() { const operationStatus = new OperationStatus(); - const operation = new Operation('close', ReadableOperationStream.EOS, operationStatus); + const operation = new Operation('close', ReadableStream.EOS, operationStatus); this._queue._queue.push({value: operation, size: 0}); diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index c9ec0528d..0ee4c5da7 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -1,3 +1,6 @@ +import { selectStreams, writableAcceptsWriteAndClose, writableAcceptsAbort, readableAcceptsReadAndCancel + } from './stream-base'; + export function jointOps(op, status) { function forward() { if (status.state === 'waiting') { @@ -11,37 +14,6 @@ export function jointOps(op, status) { forward(); } -// Exported as a helper for building transformation. -export function selectOperationStreams(readable, writable) { - const promises = []; - - promises.push(readable.errored); - if (readable.state === 'waiting') { - promises.push(readable.readable); - } - - promises.push(writable.errored); - if (writable.state === 'writable') { - promises.push(writable.waitSpaceChange()); - } else if (writable.state === 'waiting') { - promises.push(writable.writable); - } - - return Promise.race(promises); -} - -export function writableAcceptsWriteAndClose(state) { - return state === 'waiting' || state === 'writable'; -} - -export function writableAcceptsAbort(state) { - return state === 'waiting' || state === 'writable' || state === 'closed'; -} - -export function readableAcceptsReadAndCancel(state) { - return state === 'waiting' || state === 'readable'; -} - // Pipes data from source to dest with no transformation. Abort signal, cancel signal and space are also propagated // between source and dest. export function pipeOperationStreams(source, dest) { @@ -128,7 +100,7 @@ export function pipeOperationStreams(source, dest) { } } - selectOperationStreams(source, dest) + selectStreams(source, dest) .then(loop) .catch(disposeStreams); return; diff --git a/reference-implementation/lib/experimental/readable-operation-stream.js b/reference-implementation/lib/experimental/readable-stream.js similarity index 92% rename from reference-implementation/lib/experimental/readable-operation-stream.js rename to reference-implementation/lib/experimental/readable-stream.js index 49404f441..89d635603 100644 --- a/reference-implementation/lib/experimental/readable-operation-stream.js +++ b/reference-implementation/lib/experimental/readable-stream.js @@ -1,6 +1,6 @@ -import { Operation, OperationStatus, readableAcceptsReadAndCancel } from './operation-stream'; +import { readableAcceptsReadAndCancel } from './stream-base'; -export class ReadableOperationStream { +export class ReadableStream { _initReadablePromise() { this._readablePromise = new Promise((resolve, reject) => { this._resolveReadablePromise = resolve; @@ -18,7 +18,7 @@ export class ReadableOperationStream { this._resolveErroredPromise = resolve; }); - this._abortOperation = undefined; + this._error = undefined; this._window = 0; @@ -132,7 +132,7 @@ export class ReadableOperationStream { getReader() { this._throwIfLocked(); - this._reader = new ExclusiveOperationStreamWriter(this); + this._reader = new ExclusiveStreamWriter(this); return this._reader; } @@ -179,4 +179,4 @@ export class ReadableOperationStream { } } -ReadableOperationStream.EOS = {}; +ReadableStream.EOS = {}; diff --git a/reference-implementation/lib/experimental/writable-operation-stream.js b/reference-implementation/lib/experimental/writable-stream.js similarity index 96% rename from reference-implementation/lib/experimental/writable-operation-stream.js rename to reference-implementation/lib/experimental/writable-stream.js index ed6fb9d78..be2494c90 100644 --- a/reference-implementation/lib/experimental/writable-operation-stream.js +++ b/reference-implementation/lib/experimental/writable-stream.js @@ -1,7 +1,7 @@ -import { writableAcceptsWriteAndClose, writableAcceptsAbort } from './operation-stream'; -import { ExclusiveOperationStreamWriter } from './exclusive-operation-stream-writer'; +import { writableAcceptsWriteAndClose, writableAcceptsAbort } from './stream-base'; +import { ExclusiveStreamWriter } from './exclusive-stream-writer'; -export class WritableOperationStream { +export class WritableStream { _initWritablePromise() { this._writablePromise = new Promise((resolve, reject) => { this._resolveWritablePromise = resolve; @@ -194,7 +194,7 @@ export class WritableOperationStream { getWriter() { this._throwIfLocked(); - this._writer = new ExclusiveOperationStreamWriter(this); + this._writer = new ExclusiveStreamWriter(this); return this._writer; } diff --git a/reference-implementation/run-tests.js b/reference-implementation/run-tests.js index 25057ac99..b810bf900 100644 --- a/reference-implementation/run-tests.js +++ b/reference-implementation/run-tests.js @@ -4,21 +4,12 @@ const path = require('path'); import ReadableStream from './lib/readable-stream'; import WritableStream from './lib/writable-stream'; import ReadableByteStream from './lib/experimental/readable-byte-stream'; -import { jointOps, pipeOperationStreams, selectOperationStreams - } from './lib/experimental/operation-stream'; -import { createOperationQueue } from './lib/experimental/operation-queue'; import ByteLengthQueuingStrategy from './lib/byte-length-queuing-strategy'; import CountQueuingStrategy from './lib/count-queuing-strategy'; import TransformStream from './lib/transform-stream'; global.ReadableStream = ReadableStream; global.WritableStream = WritableStream; - -global.createOperationQueue = createOperationQueue; -global.jointOps = jointOps; -global.pipeOperationStreams = pipeOperationStreams; -global.selectOperationStreams = selectOperationStreams; - global.ReadableByteStream = ReadableByteStream; global.ByteLengthQueuingStrategy = ByteLengthQueuingStrategy; global.CountQueuingStrategy = CountQueuingStrategy; diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 9bd29e75a..1f45f5ed0 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -1,6 +1,9 @@ const test = require('tape-catch'); -import { ReadableOperationStream } from '../../lib/experimental/readable-operation-stream'; +import { ReadableStream } from '../../lib/experimental/readable-stream'; +import { selectStreams } from '../../lib/experimental/stream-base'; +import { jointOps, pipeOperationStreams } from '../../lib/experimental/operation-stream'; +import { createOperationQueue } from '../../lib/experimental/operation-queue'; test('Operation stream pair is constructed', t => { const pair = createOperationQueue({ @@ -472,7 +475,7 @@ test('Transformation example: Byte counting', t => { } } - selectOperationStreams(source, dest) + selectStreams(source, dest) .then(loop) .catch(disposeStreams); return; @@ -1244,7 +1247,7 @@ test('Adapting unstoppable push source', t => { if (this._queue.length === 0) { if (entry.type === 'close') { this._readableStreamDelegate.markDrained(); - return ReadableOperationStream.EOS; + return ReadableStream.EOS; } else { this._readableStreamDelegate.markWaiting(); return entry.data; @@ -1262,7 +1265,7 @@ test('Adapting unstoppable push source', t => { } const source = new Source(); - const readableStream = new ReadableOperationStream(source); + const readableStream = new ReadableStream(source); let count = 0; function pump() { @@ -1278,7 +1281,7 @@ test('Adapting unstoppable push source', t => { } else if (readableStream.state === 'readable') { const data = readableStream.read(); if (count === 10) { - t.equals(data, ReadableOperationStream.EOS); + t.equals(data, ReadableStream.EOS); t.end(); return; } else { From 4527336a5ea1c1bf85c7ad6653413e966d1224fc Mon Sep 17 00:00:00 2001 From: tyoshino Date: Mon, 2 Mar 2015 01:52:14 +0900 Subject: [PATCH 81/93] Add stream-base.js --- .../lib/experimental/stream-base.js | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 reference-implementation/lib/experimental/stream-base.js diff --git a/reference-implementation/lib/experimental/stream-base.js b/reference-implementation/lib/experimental/stream-base.js new file mode 100644 index 000000000..970faa4a4 --- /dev/null +++ b/reference-implementation/lib/experimental/stream-base.js @@ -0,0 +1,30 @@ +export function writableAcceptsWriteAndClose(state) { + return state === 'waiting' || state === 'writable'; +} + +export function writableAcceptsAbort(state) { + return state === 'waiting' || state === 'writable' || state === 'closed'; +} + +export function readableAcceptsReadAndCancel(state) { + return state === 'waiting' || state === 'readable'; +} + +// Exported as a helper for building transformation. +export function selectStreams(readable, writable) { + const promises = []; + + promises.push(readable.errored); + if (readable.state === 'waiting') { + promises.push(readable.readable); + } + + promises.push(writable.errored); + if (writable.state === 'writable') { + promises.push(writable.waitSpaceChange()); + } else if (writable.state === 'waiting') { + promises.push(writable.writable); + } + + return Promise.race(promises); +} From b0540cec098ca81fc931f7fe6fbcd80e3c195a53 Mon Sep 17 00:00:00 2001 From: tyoshino Date: Mon, 2 Mar 2015 02:37:20 +0900 Subject: [PATCH 82/93] State check fix for read() --- .../lib/experimental/operation-stream.js | 6 +++--- .../lib/experimental/readable-stream.js | 10 +++++----- .../lib/experimental/stream-base.js | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js index 0ee4c5da7..bad0e00de 100644 --- a/reference-implementation/lib/experimental/operation-stream.js +++ b/reference-implementation/lib/experimental/operation-stream.js @@ -1,4 +1,4 @@ -import { selectStreams, writableAcceptsWriteAndClose, writableAcceptsAbort, readableAcceptsReadAndCancel +import { selectStreams, writableAcceptsWriteAndClose, writableAcceptsAbort, readableAcceptsCancel } from './stream-base'; export function jointOps(op, status) { @@ -24,7 +24,7 @@ export function pipeOperationStreams(source, dest) { if (writableAcceptsAbort(dest.state)) { dest.abort(error); } - if (readableAcceptsReadAndCancel(source.state)) { + if (readableAcceptsCancel(source.state)) { source.abort(error); } reject(error); @@ -66,7 +66,7 @@ export function pipeOperationStreams(source, dest) { } if (dest.state === 'errored') { - if (readableAcceptsReadAndCancel(source.state)) { + if (readableAcceptsCancel(source.state)) { if (dest.error.constructor === Operation) { jointOps(dest.error, source.cancel(dest.error.argument)); } else { diff --git a/reference-implementation/lib/experimental/readable-stream.js b/reference-implementation/lib/experimental/readable-stream.js index 89d635603..19f15e760 100644 --- a/reference-implementation/lib/experimental/readable-stream.js +++ b/reference-implementation/lib/experimental/readable-stream.js @@ -1,4 +1,4 @@ -import { readableAcceptsReadAndCancel } from './stream-base'; +import { readableAcceptsCancel } from './stream-base'; export class ReadableStream { _initReadablePromise() { @@ -47,8 +47,8 @@ export class ReadableStream { } _readIgnoringLock() { - if (!readableAcceptsReadAndCancel(this._state)) { - throw new TypeError('already ' + this._state); + if (this._state !== 'readable') { + throw new TypeError('not readable'); } return this._source.read(); @@ -59,7 +59,7 @@ export class ReadableStream { } _cancelIgnoringLock(reason) { - if (!readableAcceptsReadAndCancel(this._state)) { + if (!readableAcceptsCancel(this._state)) { throw new TypeError('already ' + this._state); } @@ -109,7 +109,7 @@ export class ReadableStream { } set _windowIgnoringLock(v) { - if (!readableAcceptsReadAndCancel(this._state)) { + if (!readableAcceptsCancel(this._state)) { throw new TypeError('already ' + this._state); } diff --git a/reference-implementation/lib/experimental/stream-base.js b/reference-implementation/lib/experimental/stream-base.js index 970faa4a4..b63766957 100644 --- a/reference-implementation/lib/experimental/stream-base.js +++ b/reference-implementation/lib/experimental/stream-base.js @@ -6,7 +6,7 @@ export function writableAcceptsAbort(state) { return state === 'waiting' || state === 'writable' || state === 'closed'; } -export function readableAcceptsReadAndCancel(state) { +export function readableAcceptsCancel(state) { return state === 'waiting' || state === 'readable'; } From 0d54e6e124b32a46009e4bec98ad7dce8d8889d2 Mon Sep 17 00:00:00 2001 From: tyoshino Date: Mon, 2 Mar 2015 02:37:53 +0900 Subject: [PATCH 83/93] Add ThinStream.md --- ThinStream.md | 173 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 ThinStream.md diff --git a/ThinStream.md b/ThinStream.md new file mode 100644 index 000000000..06260d351 --- /dev/null +++ b/ThinStream.md @@ -0,0 +1,173 @@ +```es6 +class UnderlyingSink { + // delegate contains: + // - markWaiting(): Tells the stream that backpressure is applied. + // - markWritable(): Tells the stream that backpressure is not applied. + // - markErrored(error): Tells the stream that the sink is errored. + // - onSpaceChange(): Tells the stream that get space() may return something new. + init(delegate) + + // May return something. If backpressure state is changed, should call appropriate + // delegate function to update the stream. + write(value) + // May return something. + abort() + + get space() +} + +// Once a writer is created, promises obtained from this stream are not fulfilled until the +// writer is released. +class WritableStream { + // At the end of construction, calls init() on underlyingSink with delegate functions for this + // stream. + constructor(underlyingSink) + + // List of states in the form of: + // X -> A, B, C, ... + // X is state name or a name of a group of states. + // A, B, C, ... are the states X may transit to. + // + // - "locked" -> available + // - available -> "locked" + // - (write()/close() are allowed) -> "closed", "aborted", "errored" + // - "waiting" (backpressure not applied) -> "writable" + // - "writable" (backpressure applied) -> "waiting" + // - "closed" -> "aborted", "errored" + // - "aborted" + // - "errored" + // + // Distinction between "waiting" and "writable" is a part of the flow control interfaces. + get state() + + // Main interfaces. + + // Passes value and returns something returned by the underlying sink. + // + // Available in "waiting" and "writable" state. + write(value) + // Tells the underlying sink that no more data will be write()-en and returns something returned + // by the underlying sink. + // + // Available in "waiting" and "writable" state. + close() + // Tells the underlying sink that no more data will be write()-en and returns something returned + // by the underlying sink. + // + // Available in "waiting", "writable" and "closed" state. + abort(reason) + + // Error receiving interfaces. + + // Returns a promise which gets fulfilled when this instance enters the "errored" state. + get errored() + // Returns an object representing the error when the state is "errored". + get error() + + // Flow control interfaces. + + // Returns a promise which gets fulfilled when this instance enters "writable" state. + get writable() + // Returns the space available for write. + // + // Available in "waiting" and "writable" state. + get space() + // Returns a promise which gets fulfilled when space() becomes different value than one at the last + // waitSpaceChange() call. + // + // Available in "waiting" and "writable" state. + waitSpaceChange() + + // Locking interfaces. + + // Creates and returns an ExclusiveStreamWriter instance. Once a writer is created, all methods + // and accessors on this stream throws until the writer is released. + // + // Available in all states if there is no existing active writer. + getWriter() +} + +// Once release() is called, all methods and accessors on this writer throw. +class ExclusiveStreamWriter { + // - "locked" -> available + // - available -> "locked" + // - (write()/close() are allowed) -> "closed", "aborted", "errored" + // - "waiting" (backpressure not applied) -> "writable" + // - "writable" (backpressure applied) -> "waiting" + // - "closed -> "aborted", "errored" + // - "aborted" + // - "errored" + get state() + + get writable() + get space() + waitSpaceChange() + + get errored() + get error() + + write(argument) + close() + abort(reason) + + release() +} + +class UnderlyingSource { + // delegate contains: + // - markWaiting(): Tells the stream that nothing is ready for synchronous reading. + // - markReadable(): Tells the stream that something is ready for synchronous reading. + // - markDrained(): Tells the stream that no more data will become available for read. + // - markErrored(error): Tells the stream that the source is errored. + init(delegate) + + // Returns something. If data availability is changed, should call appropriate + // delegate function to update the stream. + read() + // May return something. + cancel(reason) + + get onWindowUpdate(v) +} + +class ReadableStream { + constructor(underlyingSource) + + // - "locked" -> available + // - available -> "locked" + // - normal -> "cancelled", "errored" + // - "waiting" -> "readable" + // - "readable" -> "waiting", "drained" + // - "drained" + // - "cancelled" + // - "errored" + get state() + + // Returns a promise which gets fulfilled when this instance enters "readable" state. + get readable() + // Returns something returned by the underlying source. + // + // Available in "readable" state. + read() + // Passes reason and returns something returned by the underlying source. + // + // Available in "waiting" and "readable" state. + cancel(reason) + + get errored() + get error() + + // Flow control interfaces. + + // Available in "waiting" and "readable" state. + get window() + // Passes v to the underlying source. v indicates how much data should be pulled. + // + // Available in "waiting" and "readable" state. + set window(v) + + // Locking interfaces. + + getReader() +} +``` From 3931c1bdca6f06ad7747eea161f0814a4f6c8a2b Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Mon, 2 Mar 2015 03:04:16 +0900 Subject: [PATCH 84/93] Add close() to ThinStream --- ThinStream.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ThinStream.md b/ThinStream.md index 06260d351..83e06e932 100644 --- a/ThinStream.md +++ b/ThinStream.md @@ -11,6 +11,8 @@ class UnderlyingSink { // delegate function to update the stream. write(value) // May return something. + close() + // May return something. abort() get space() From 06939ad5d3119386b4c8a07ecd7e266f2d6d5df6 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Wed, 4 Mar 2015 23:14:21 +0900 Subject: [PATCH 85/93] Thin Stream --- .../lib/experimental/byte-stream-queue.js | 65 + .../experimental/exclusive-stream-writer.js | 131 -- .../fake-file-backed-byte-source.js | 230 +++ .../fake-unstoppable-byte-source.js | 127 ++ .../lib/experimental/mock-byte-sink.js | 109 ++ .../lib/experimental/operation-queue.js | 89 +- .../lib/experimental/readable-stream.js | 182 --- .../lib/experimental/stream-base.js | 30 - .../lib/experimental/stream-queue.js | 179 +++ .../experimental/thin-readable-byte-stream.js | 149 ++ .../lib/experimental/thin-readable-stream.js | 124 ++ .../lib/experimental/thin-stream-base.js | 92 ++ .../lib/experimental/thin-stream-utils.js | 8 + .../experimental/thin-writable-byte-stream.js | 201 +++ .../lib/experimental/thin-writable-stream.js | 152 ++ .../lib/experimental/writable-stream.js | 244 ---- reference-implementation/run-tests.js | 2 +- .../test/experimental/operation-stream.js | 1244 +---------------- .../test/experimental/thin-stream.js | 409 ++++++ 19 files changed, 1904 insertions(+), 1863 deletions(-) create mode 100644 reference-implementation/lib/experimental/byte-stream-queue.js delete mode 100644 reference-implementation/lib/experimental/exclusive-stream-writer.js create mode 100644 reference-implementation/lib/experimental/fake-file-backed-byte-source.js create mode 100644 reference-implementation/lib/experimental/fake-unstoppable-byte-source.js create mode 100644 reference-implementation/lib/experimental/mock-byte-sink.js delete mode 100644 reference-implementation/lib/experimental/readable-stream.js create mode 100644 reference-implementation/lib/experimental/stream-queue.js create mode 100644 reference-implementation/lib/experimental/thin-readable-byte-stream.js create mode 100644 reference-implementation/lib/experimental/thin-readable-stream.js create mode 100644 reference-implementation/lib/experimental/thin-stream-base.js create mode 100644 reference-implementation/lib/experimental/thin-stream-utils.js create mode 100644 reference-implementation/lib/experimental/thin-writable-byte-stream.js create mode 100644 reference-implementation/lib/experimental/thin-writable-stream.js delete mode 100644 reference-implementation/lib/experimental/writable-stream.js create mode 100644 reference-implementation/test/experimental/thin-stream.js diff --git a/reference-implementation/lib/experimental/byte-stream-queue.js b/reference-implementation/lib/experimental/byte-stream-queue.js new file mode 100644 index 000000000..6a48ffa84 --- /dev/null +++ b/reference-implementation/lib/experimental/byte-stream-queue.js @@ -0,0 +1,65 @@ + pull(container) { + if (container === undefined) { + throw new TypeError('container is undefined'); + } + if (container.constructor !== Uint8Array) { + throw new TypeError('the only supported container is Uint8Array'); + } + + this._pendingPulls.push(container); + this._resolvePendingPulls(); + } + + _resolvePendingPulls() { + let bytesConsumed = 0; + let enqueued = false; + + while (this._pendingPulls.length > 0) { + const destView = this._pendingPulls.shift(); + let destViewPosition = 0; + + while (this._shared._queue.length > 0) { + if (destViewPosition === destView.byteLength) { + this._readableValueQueue.push(destView); + enqueued = true; + break; + } + + const entry = this._shared._queue[0]; + + const srcView = entry.value; + if (srcView === undefined || srcView.constructor !== Uint8Array) { + console.log('not reached'); + } + + const bytesToCopy = Math.min(destView.byteLength - destViewPosition, srcView.byteLength); + destView.set(srcView.subarray(0, bytesToCopy), destViewPosition); + destViewPosition += bytesToCopy; + + if (bytesToCopy === srcView.byteLength) { + this._shared._queue.shift(); + } else { + this._shared[0] = srcView.subarray(bytesToCopy); + } + bytesConsumed += bytesToCopy; + } + + if (this._shared._queue.length === 0) { + this._readableValueQueue.push(destView.subarray(0, destViewPosition)); + enqueued = true; + + if (this._shared._draining) { + while (this._pendingPulls.length > 0) { + const destView = this._pendingPulls.shift(); + this._readableValueQueue.push(destView.subarray()); + } + } + break; + } + } + + this._delegate.markReadable(); + + this._shared._queueSize -= bytesConsumed; + this._sink.onQueueConsume(); + } diff --git a/reference-implementation/lib/experimental/exclusive-stream-writer.js b/reference-implementation/lib/experimental/exclusive-stream-writer.js deleted file mode 100644 index d405328f2..000000000 --- a/reference-implementation/lib/experimental/exclusive-stream-writer.js +++ /dev/null @@ -1,131 +0,0 @@ -export class ExclusiveStreamWriter { - _initWritablePromise() { - this._writablePromise = new Promise((resolve, reject) => { - this._resolveWritablePromise = resolve; - }); - } - - _syncStateAndWritablePromise() { - if (this._parent._state === 'writable') { - if (this._resolveWritablePromise !== undefined) { - this._resolveWritablePromise(); - this._resolveWritablePromise = undefined; - } - } else { - if (this._resolveWritablePromise === undefined) { - this._initWritablePromise(); - } - } - } - - _syncStateAndErroredPromise() { - if (this._parent._state === 'cancelled' || this._parent._state === 'errored') { - this._resolveErroredPromise(); - this._resolveErroredPromise = undefined; - } - } - - _syncSpaceAndSpaceChangePromise() { - if (this._spaceChangePromise !== undefined && this._lastSpace !== this.space) { - this._resolveSpaceChangePromise(); - this._resolveSpaceChangePromise = undefined; - - this._lastSpace = undefined; - this._spaceChangePromise = undefined; - } - } - - constructor(parent) { - this._parent = parent; - - this._erroredPromise = new Promise((resolve, reject) => { - this._resolveErroredPromise = resolve; - }); - - this._initWritablePromise(); - this._lastSpace = undefined; - this._spaceChangePromise = undefined; - - this._syncStateAndWritablePromise(); - this._syncStateAndErroredPromise(); - } - - get state() { - this._throwIfReleased(); - return this._parent._state; - } - - // Main interfaces. - - write(argument) { - this._throwIfReleased(); - return this._parent._writeIgnoringLock(argument); - } - close() { - this._throwIfReleased(); - return this._parent._closeIgnoringLock(); - } - abort(reason) { - this._throwIfReleased(); - return this._parent._abortIgnoringLock(reason); - } - - // Error receiving interfaces. - - get errored() { - this._throwIfReleased(); - return this._erroredPromise; - } - - get cancelOperation() { - this._throwIfReleased(); - return this._parent._cancelOperationIgnoringLock; - } - - // Flow control interfaces. - - get writable() { - this._throwIfReleased(); - return this._writablePromise; - } - - get space() { - this._throwIfReleased(); - return this._parent._spaceIgnoringLock; - } - waitSpaceChange() { - this._throwIfReleased(); - - if (this._spaceChangePromise !== undefined) { - return this._spaceChangePromise; - } - - this._spaceChangePromise = new Promise((resolve, reject) => { - this._resolveSpaceChangePromise = resolve; - }); - this._lastSpace = this.space; - - return this._spaceChangePromise; - } - - // Locking interfaces. - - _throwIfReleased() { - if (this._parent === undefined) { - throw new TypeError('already released'); - } - } - - release() { - this._parent._releaseWriter(); - - this._parent = undefined; - - // Make promises collectable. - - this._erroredPromise = undefined; - - this._writablePromise = undefined; - this._spaceChangePromise = undefined; - } -} diff --git a/reference-implementation/lib/experimental/fake-file-backed-byte-source.js b/reference-implementation/lib/experimental/fake-file-backed-byte-source.js new file mode 100644 index 000000000..57d5e4dc6 --- /dev/null +++ b/reference-implementation/lib/experimental/fake-file-backed-byte-source.js @@ -0,0 +1,230 @@ +import { NewReadableStream } from './new-readable-stream'; +import { NewReadableByteStream } from './new-readable-byte-stream'; + +import { fillArrayBufferView } from './new-stream-utils'; + +class FileBackedUnderlyingSource { + constructor(readFileInto, strategy) { + this._readFileInto = readFileInto; + + this._strategy = strategy; + + this._queue = []; + this._queueSize = 0; + + this._draining = false; + + this._cancelled = false; + + this._fileReadPromise = undefined; + } + + _handleFileReadResult(result) { + if (this._cancelled) { + return; + } + + this._fileReadPromise = undefined; + + this._queue.push(result.writtenRegion); + this._queueSize += result.writtenRegion.byteLength; + + console.log('queue' + this._queueSize); + + this._delegate.markReadable(); + + if (result.closed) { + this._strategy = undefined; + this._draining = true; + return; + } + + this._pull(); + } + + _pull() { + console.log('222'); + + if (this._cancelled) { + return; + } + + console.log('223'); + + if (this._strategy.shouldApplyBackpressure === undefined || + this._strategy.shouldApplyBackpressure(this._queueSize)) { + return; + } + + console.log('224'); + + if (this._fileReadPromise !== undefined) { + return; + } + + let size = 1024; + if (this._strategy.space !== undefined) { + size = this._strategy.space(this._queueSize); + } + if (size === 0) { + console.log('full'); + return; + } + console.log('ssss' + size); + + const view = new Uint8Array(size); + this._fileReadPromise = this._readFileInto(view) + .then(this._handleFileReadResult.bind(this)) + .catch(this._delegate.markErrored.bind(this._delegate)); + } + + start(delegate) { + this._delegate = delegate; + + this._pull(); + } + + onWindowUpdate(v) { + if (this._strategy === undefined) { + return; + } + + if (this._strategy.onWindowUpdate !== undefined) { + this._strategy.onWindowUpdate(v); + } + + this._pull(); + } + + read() { + const view = this._queue.shift(); + this._queueSize -= view.byteLength; + + if (this._queue.length === 0) { + if (this._draining) { + this._delegate.markClosed(); + } else { + this._delegate.markWaiting(); + } + } + + if (!this._draining) { + this._pull(); + } + + return view; + } + + cancel() { + this._queue = []; + + this._cancelled = true; + } +} + +class FileBackedManualPullUnderlyingSource { + constructor(readFileInto) { + this._readFileInto = readFileInto; + + this._currentResult = undefined; + + this._draining = false; + + this._cancelled = false; + + this._fileReadPromise = undefined; + } + + _handleFileReadResult(result) { + if (this._cancelled) { + return; + } + + this._fileReadPromise = undefined; + this._delegate.markPullable(); + + this._currentResult = result.writtenRegion; + this._delegate.markReadable(); + + if (result.closed) { + this._draining = true; + } + } + + start(delegate) { + this._delegate = delegate; + + this._delegate.markPullable(); + } + + pull(view) { + this._fileReadPromise = this._readFileInto(view) + .then(this._handleFileReadResult.bind(this)) + .catch(this._delegate.markErrored.bind(this._delegate)); + + this._delegate.markNotPullable(); + } + + read() { + if (this._draining) { + this._delegate.markClosed(); + } else { + this._delegate.markWaiting(); + } + + const result = this._currentResult; + this._currentResult = undefined; + return result; + } + + cancel() { + this._cancelled = true; + } +} + +export class FakeFile { + constructor(size) { + this._bytesRemaining = size; + } + + _readFileInto(view) { + return new Promise((resolve, reject) => { + setTimeout(() => { + try { + const bytesToWrite = Math.min(this._bytesRemaining, view.byteLength); + + fillArrayBufferView(view, 1, bytesToWrite); + + this._bytesRemaining -= bytesToWrite; + + const writtenRegion = view.subarray(0, bytesToWrite); + console.log('zzzz ' + writtenRegion.byteLength); + if (this._bytesRemaining === 0) { + resolve({closed: true, writtenRegion}); + } else { + resolve({closed: false, writtenRegion}); + } + } catch (e) { + reject(e); + } + }, 0); + }); + } + + createStream(strategy) { + return new NewReadableStream(new FileBackedUnderlyingSource(this._readFileInto.bind(this), strategy)); + } + + // Returns a manual pull readable stream. + // + // Example semantics: + // + // POSIX socket: + // - The stream becomes pullable when epoll(7) returns, and stays to be pullable until read(2) returns EAGAIN. + // + // Blocking or async I/O interfaces which takes a buffer on reading function call: + // - The stream is always pullable. + createManualPullStream() { + return new NewReadableByteStream(new FileBackedManualPullUnderlyingSource(this._readFileInto.bind(this))); + } +} diff --git a/reference-implementation/lib/experimental/fake-unstoppable-byte-source.js b/reference-implementation/lib/experimental/fake-unstoppable-byte-source.js new file mode 100644 index 000000000..946aa6665 --- /dev/null +++ b/reference-implementation/lib/experimental/fake-unstoppable-byte-source.js @@ -0,0 +1,127 @@ +class FakeUnstoppablePushSource { + constructor(count) { + this._count = count; + + setTimeout(this._push.bind(this), 0); + } + + _push() { + if (this._count == 0) { + this.onend(); + return; + } + + this.ondata('foo'); + --this._count; + + setTimeout(this._push.bind(this), 0); + } +} + +test('Adapting unstoppable push source', t => { + class Source { + constructor() { + this._queue = []; + } + + init(delegate) { + this._pushSource = new FakeUnstoppablePushSource(10); + + t.equal(typeof delegate.markWaiting, 'function', 'markWaiting is a function'); + t.equal(typeof delegate.markReadable, 'function', 'markReadable is a function'); + t.equal(typeof delegate.markDrained, 'function', 'markDrained is a function'); + t.equal(typeof delegate.markErrored, 'function', 'markErrored is a function'); + + this._readableStreamDelegate = delegate; + + this._pushSource.ondata = chunk => { + this._queue.push({type: 'data', data: chunk}); + delegate.markReadable(); + }; + + this._pushSource.onend = () => { + this._queue.push({type: 'close'}); + delegate.markReadable(); + }; + + this._pushSource.onerror = () => { + this._queue = []; + delegate.markErrored(); + }; + } + + onWindowUpdate(v) { + } + + read() { + if (this._queue.length === 0) { + throw new TypeError('not readable'); + } + + const entry = this._queue.shift(); + + if (this._queue.length === 0) { + if (entry.type === 'close') { + this._readableStreamDelegate.markDrained(); + return ReadableStream.EOS; + } else { + this._readableStreamDelegate.markWaiting(); + return entry.data; + } + } + + return entry; + } + + cancel() { + this._queue = []; + + this._pushSource.close(); + } + } + + const source = new Source(); + const readableStream = new ReadableStream(source); + + let count = 0; + function pump() { + for (;;) { + if (readableStream.state === 'waiting') { + Promise.race([readableStream.readable, readableStream.errored]) + .then(pump) + .catch(e => { + t.fail(e); + t.end(); + }); + return; + } else if (readableStream.state === 'readable') { + const data = readableStream.read(); + if (count === 10) { + t.equal(data, ReadableStream.EOS); + t.end(); + return; + } else { + t.equal(data, 'foo'); + ++count; + } + } else if (readableStream.state === 'drained') { + t.fail(); + t.end(); + return; + } else if (readableStream.state === 'cancelled') { + t.fail(); + t.end(); + return; + } else if (readableStream.state === 'errored') { + t.fail(); + t.end(); + return; + } else { + t.fail(readableStream.state); + t.end(); + return; + } + } + } + pump(); +}); diff --git a/reference-implementation/lib/experimental/mock-byte-sink.js b/reference-implementation/lib/experimental/mock-byte-sink.js new file mode 100644 index 000000000..ef1c6fdd8 --- /dev/null +++ b/reference-implementation/lib/experimental/mock-byte-sink.js @@ -0,0 +1,109 @@ +import { NewWritableStream } from './new-writable-stream'; +import { NewWritableByteStream } from './new-writable-byte-stream'; + +class MockFileUnderlyingSink { + constructor(file) { + this._file = file; + } + + start(delegate) { + this._delegate = delegate; + } + + write(value) { + this._file._count(value.byteLength); + } + + close() { + this._file._close(); + } + + abort(reason) { + this._file._abort(reason); + } + + space() { + return 16; + } +} + +class MockFileUnderlyingSinkWithDisposal { + constructor(file) { + this._file = file; + + this._bytesRead = 0; + + this._resultPromise = new Promise((resolve, reject) => { + this._resolveResultPromise = resolve; + this._rejectResultPromise = reject; + }); + + this._disposedViews = []; + } + + start(delegate) { + this._delegate = delegate; + + this._delegate.markWritable(); + } + + write(view) { + this._file._count(view.byteLength); + this._disposedViews.push(view); + this._delegate.markHasDisposedView(); + } + + close() { + this._file._close(); + } + + abort(reason) { + this._file._abort(reason); + } + + space() { + return 16; + } + + readDisposedView() { + const view = this._disposedViews.shift(); + if (this._disposedViews.length === 0) { + this._delegate.markNoDisposedView(); + } + } +} + +export class MockFile { + constructor() { + this._bytesRead = 0; + + this._resultPromise = new Promise((resolve, reject) => { + this._resolveResultPromise = resolve; + this._rejectResultPromise = reject; + }); + } + + get result() { + return this._resultPromise; + } + + _count(size) { + this._bytesRead += size; + } + + _close() { + this._resolveResultPromise(this._bytesRead); + } + + _abort(reason) { + this._rejectResultPromise(reason); + } + + createStream() { + return new NewWritableStream(new MockFileUnderlyingSink(this)); + } + + createStreamWithDisposal() { + return new NewWritableByteStream(new MockFileUnderlyingSinkWithDisposal(this)); + } +} diff --git a/reference-implementation/lib/experimental/operation-queue.js b/reference-implementation/lib/experimental/operation-queue.js index 84e5a0517..639d46674 100644 --- a/reference-implementation/lib/experimental/operation-queue.js +++ b/reference-implementation/lib/experimental/operation-queue.js @@ -2,26 +2,24 @@ import { Operation, OperationStatus } from './operation-stream'; import { WritableStream } from './writable-stream'; import { ReadableStream } from './readable-stream'; -// Creates a pair of WritableStream implementation and ReadableStream implementation that are -// connected with a queue. This can be used for creating queue-backed operation streams. -export function createOperationQueue(strategy) { - const queue = new OperationQueue(strategy); - const source = new OperationQueueUnderlyingSource(queue); - const sink = new OperationQueueUnderlyingSink(queue); - sink.setSource(source); - source.setSink(sink); - return { writable: new WritableStream(sink), readable: new ReadableStream(source) }; +class OperationQueueShared { + constructor(strategy) { + this._shared = []; + this._sharedSize = 0; + + this._strategy = strategy; + } } class OperationQueueUnderlyingSink { _updateWritableStream() { - if (this._queue._strategy === undefined) { + if (this._shared._strategy === undefined) { return; } let shouldApplyBackpressure = false; - if (this._queue._strategy.shouldApplyBackpressure !== undefined) { - shouldApplyBackpressure = this._queue._strategy.shouldApplyBackpressure(this._queue._queueSize); + if (this._shared._strategy.shouldApplyBackpressure !== undefined) { + shouldApplyBackpressure = this._shared._strategy.shouldApplyBackpressure(this._shared._queueSize); } if (shouldApplyBackpressure) { this._delegate.markWaiting(); @@ -33,7 +31,7 @@ class OperationQueueUnderlyingSink { } constructor(queue) { - this._queue = queue; + this._shared = queue; } setSource(source) { @@ -47,8 +45,8 @@ class OperationQueueUnderlyingSink { } get space() { - if (this._queue._strategy.space !== undefined) { - return this._queue._strategy.space(this._queue._queueSize); + if (this._shared._strategy.space !== undefined) { + return this._shared._strategy.space(this._shared._queueSize); } return undefined; @@ -59,12 +57,12 @@ class OperationQueueUnderlyingSink { const operation = new Operation('data', value, operationStatus); var size = 1; - if (this._queue._strategy.size !== undefined) { - size = this._queue._strategy.size(operation.argument); + if (this._shared._strategy.size !== undefined) { + size = this._shared._strategy.size(operation.argument); } - this._queue._queue.push({value: operation, size}); - this._queue._queueSize += size; + this._shared._queue.push({value: operation, size}); + this._shared._queueSize += size; this._updateWritableStream(); @@ -77,10 +75,10 @@ class OperationQueueUnderlyingSink { const operationStatus = new OperationStatus(); const operation = new Operation('close', ReadableStream.EOS, operationStatus); - this._queue._queue.push({value: operation, size: 0}); + this._shared._queue.push({value: operation, size: 0}); // No longer necessary. - this._queue._strategy = undefined; + this._shared._strategy = undefined; this._source.onQueueFill(); @@ -91,13 +89,13 @@ class OperationQueueUnderlyingSink { const operationStatus = new OperationStatus(); const operation = new Operation('abort', reason, operationStatus); - for (var i = this._queue._queue.length - 1; i >= 0; --i) { - const op = this._queue._queue[i].value; + for (var i = this._shared._queue.length - 1; i >= 0; --i) { + const op = this._shared._queue[i].value; op.error(new TypeError('aborted')); } - this._queue._queue = []; + this._shared._queue = []; - this._queue._strategy = undefined; + this._shared._strategy = undefined; this._source.abort(operation); @@ -118,8 +116,8 @@ class OperationQueueUnderlyingSink { } class OperationQueueUnderlyingSource { - constructor(queue) { - this._queue = queue; + constructor(shared) { + this._shared = shared; } setSink(sink) { @@ -139,26 +137,26 @@ class OperationQueueUnderlyingSource { } onWindowUpdate(v) { - if (this._queue._strategy === undefined) { + if (this._shared._strategy === undefined) { return; } - if (this._queue._strategy.onWindowUpdate !== undefined) { - this._queue._strategy.onWindowUpdate(v); + if (this._shared._strategy.onWindowUpdate !== undefined) { + this._shared._strategy.onWindowUpdate(v); } this._sink.onWindowUpdate(); } read() { - if (this._queue._queue.length === 0) { + if (this._shared._queue.length === 0) { throw new TypeError('not readable'); } - const entry = this._queue._queue.shift(); - this._queue._queueSize -= entry.size; + const entry = this._shared._queue.shift(); + this._shared._queueSize -= entry.size; - if (this._queue._queue.length === 0) { + if (this._shared._queue.length === 0) { if (entry.value.type === 'close') { this._delegate.markDrained(); } else { @@ -175,13 +173,13 @@ class OperationQueueUnderlyingSource { const operationStatus = new OperationStatus(); const operation = new Operation('cancel', reason, operationStatus); - for (var i = 0; i < this._queue._queue.length; ++i) { - const op = this._queue._queue[i].value; + for (var i = 0; i < this._shared._queue.length; ++i) { + const op = this._shared._queue[i].value; op.error(operation.argument); } - this._queue._queue = []; + this._shared._queue = []; - this._queue._strategy = undefined; + this._shared._strategy = undefined; this._sink.onCancel(operation); @@ -189,11 +187,14 @@ class OperationQueueUnderlyingSource { } } -class OperationQueue { - constructor(strategy) { - this._queue = []; - this._queueSize = 0; - this._strategy = strategy; - } +// Creates a pair of WritableStream implementation and ReadableStream implementation that are +// connected with a queue. This can be used for creating queue-backed operation streams. +export function createOperationQueue(strategy) { + const queue = new OperationQueueShared(strategy); + const source = new OperationQueueUnderlyingSource(queue); + const sink = new OperationQueueUnderlyingSink(queue); + sink.setSource(source); + source.setSink(sink); + return { writable: new WritableStream(sink), readable: new ReadableStream(source) }; } diff --git a/reference-implementation/lib/experimental/readable-stream.js b/reference-implementation/lib/experimental/readable-stream.js deleted file mode 100644 index 19f15e760..000000000 --- a/reference-implementation/lib/experimental/readable-stream.js +++ /dev/null @@ -1,182 +0,0 @@ -import { readableAcceptsCancel } from './stream-base'; - -export class ReadableStream { - _initReadablePromise() { - this._readablePromise = new Promise((resolve, reject) => { - this._resolveReadablePromise = resolve; - }); - } - - constructor(source) { - this._source = source; - - this._state = 'waiting'; - - this._initReadablePromise(); - - this._erroredPromise = new Promise((resolve, reject) => { - this._resolveErroredPromise = resolve; - }); - - this._error = undefined; - - this._window = 0; - - this._reader = undefined; - - const delegate = { - markWaiting: this._markWaiting.bind(this), - markReadable: this._markReadable.bind(this), - markDrained: this._markDrained.bind(this), - markErrored: this._markErrored.bind(this) - }; - - this._source.init(delegate); - } - - get state() { - this._throwIfLocked(); - return this._state; - } - - // Main interfaces. - - get readable() { - this._throwIfLocked(); - return this._readablePromise; - } - - _readIgnoringLock() { - if (this._state !== 'readable') { - throw new TypeError('not readable'); - } - - return this._source.read(); - } - read() { - this._throwIfLocked(); - return this._readIgnoringLock(); - } - - _cancelIgnoringLock(reason) { - if (!readableAcceptsCancel(this._state)) { - throw new TypeError('already ' + this._state); - } - - const result = this._source.cancel(reason); - - if (this._reader !== undefined) { - this._reader._resolveErroredPromise(); - } else { - this._resolveErroredPromise(); - } - - this._state = 'cancelled'; - - return result; - } - cancel(reason) { - this._throwIfLocked(); - return this._cancelIgnoringLock(reason); - } - - // Error receiving interfaces. - - get errored() { - this._throwIfLocked(); - return this._erroredPromise; - } - - get _errorIgnoringLock() { - if (this._state !== 'errored') { - throw new TypeError('not errored'); - } - return this._error; - } - get error() { - this._throwIfLocked(); - return this._errorIgnoringLock; - } - - // Flow control interfaces. - - get _windowIgnoringLock() { - return this._window; - } - get window() { - this._throwIfLocked(); - return this._windowIgnoringLock; - } - - set _windowIgnoringLock(v) { - if (!readableAcceptsCancel(this._state)) { - throw new TypeError('already ' + this._state); - } - - this._window = v; - - this._source.onWindowUpdate(v); - } - set window(v) { - this._throwIfLocked(); - this._windowIgnoringLock = v; - } - - // Locking interfaces. - - _throwIfLocked() { - if (this._reader !== undefined) { - throw new TypeError('locked'); - } - } - - getReader() { - this._throwIfLocked(); - this._reader = new ExclusiveStreamWriter(this); - return this._reader; - } - - // Methods exposed only to the underlying source. - - _markWaiting() { - if (this._reader === undefined) { - this._initReadablePromise(); - } else { - this._reader._initReadablePromise(); - } - - this._state = 'waiting'; - } - - _markReadable() { - if (this._state !== 'waiting') { - return; - } - - if (this._reader === undefined) { - this._resolveReadablePromise(); - } else { - this._reader._resolveReadablePromise(); - } - - this._state = 'readable'; - } - - _markDrained() { - this._state = 'drained'; - } - - _markErrored(error) { - if (this._reader === undefined) { - this._resolveErroredPromise(); - } else { - this._writer._resolveErroredPromise(); - } - - this._state = 'errored'; - - this._error = error; - } -} - -ReadableStream.EOS = {}; diff --git a/reference-implementation/lib/experimental/stream-base.js b/reference-implementation/lib/experimental/stream-base.js index b63766957..e69de29bb 100644 --- a/reference-implementation/lib/experimental/stream-base.js +++ b/reference-implementation/lib/experimental/stream-base.js @@ -1,30 +0,0 @@ -export function writableAcceptsWriteAndClose(state) { - return state === 'waiting' || state === 'writable'; -} - -export function writableAcceptsAbort(state) { - return state === 'waiting' || state === 'writable' || state === 'closed'; -} - -export function readableAcceptsCancel(state) { - return state === 'waiting' || state === 'readable'; -} - -// Exported as a helper for building transformation. -export function selectStreams(readable, writable) { - const promises = []; - - promises.push(readable.errored); - if (readable.state === 'waiting') { - promises.push(readable.readable); - } - - promises.push(writable.errored); - if (writable.state === 'writable') { - promises.push(writable.waitSpaceChange()); - } else if (writable.state === 'waiting') { - promises.push(writable.writable); - } - - return Promise.race(promises); -} diff --git a/reference-implementation/lib/experimental/stream-queue.js b/reference-implementation/lib/experimental/stream-queue.js new file mode 100644 index 000000000..9d9cd2714 --- /dev/null +++ b/reference-implementation/lib/experimental/stream-queue.js @@ -0,0 +1,179 @@ +import { NewWritableStream } from './new-writable-stream'; +import { NewReadableStream } from './new-readable-stream'; + +class StreamQueueShared { + constructor(strategy) { + this._queue = []; + // Cached total size for optimization. + this._queueSize = 0; + + this._draining = false; + + this._strategy = strategy; + } +} + +class StreamQueueUnderlyingSink { + _updateWritableStream() { + if (this._shared._strategy === undefined) { + return; + } + + let shouldApplyBackpressure = false; + if (this._shared._strategy.shouldApplyBackpressure !== undefined) { + shouldApplyBackpressure = this._shared._strategy.shouldApplyBackpressure(this._shared._queueSize); + } + if (shouldApplyBackpressure) { + this._delegate.markWaiting(); + } else { + this._delegate.markWritable(); + } + + this._delegate.onSpaceChange(); + } + + constructor(shared) { + this._shared = shared; + } + + setSource(source) { + this._source = source; + } + + start(delegate) { + this._delegate = delegate; + + this._updateWritableStream(); + } + + get space() { + if (this._shared._strategy.space !== undefined) { + return this._shared._strategy.space(this._shared._queueSize); + } + + return undefined; + } + + write(value) { + console.log('fjdiojfdosi'); + var size = 1; + if (this._shared._strategy.size !== undefined) { + size = this._shared._strategy.size(value); + } + + this._shared._queue.push({value, size}); + this._shared._queueSize += size; + + this._updateWritableStream(); + + this._source.onQueueFill(); + } + + close() { + this._shared._draining = true; + + // No longer necessary. + this._shared._strategy = undefined; + + this._source.onStartDraining(); + } + + abort(reason) { + this._shared._queue = []; + this._shared._strategy = undefined; + + this._source.abort(reason); + } + + onWindowUpdate() { + // Reflect up-to-date strategy to the WritableStream. + this._updateWritableStream(); + } + + onQueueConsume() { + // Reflect up-to-date status of the queue to the WritableStream. + this._updateWritableStream(); + } + + onCancel(reason) { + this._delegate.markErrored(reason); + } +} + +class StreamQueueUnderlyingSource { + constructor(shared) { + this._shared = shared; + } + + setSink(sink) { + this._sink = sink; + } + + start(delegate) { + this._delegate = delegate; + } + + onQueueFill() { + console.log('fjfljlkfj'); + + this._delegate.markReadable(); + } + + onStartDraining() { + if (this._shared._queue.length === 0) { + this._delegate.markClosed(); + } + } + + abort(reason) { + this._delegate.markErrored(reason); + } + + onWindowUpdate(v) { + if (this._shared._strategy === undefined) { + return; + } + + if (this._shared._strategy.onWindowUpdate !== undefined) { + this._shared._strategy.onWindowUpdate(v); + } + + this._sink.onWindowUpdate(); + } + + read() { + const entry = this._shared._queue.shift(); + + if (this._shared._queue.length === 0) { + if (this._shared._draining) { + this._delegate.markClosed(); + } else { + this._delegate.markWaiting(); + } + } + + this._shared._queueSize -= entry.size; + + this._sink.onQueueConsume(); + + return entry.value; + } + + cancel(reason) { + this._shared._queue = []; + this._shared._strategy = undefined; + + this._sink.onCancel(reason); + } +} + +// Creates a pair of WritableStream implementation and ReadableStream implementation that are +// connected with a queue. This can be used for creating queue-backed operation streams. +export function createStreamQueue(strategy) { + const shared = new StreamQueueShared(strategy); + const source = new StreamQueueUnderlyingSource(shared); + const sink = new StreamQueueUnderlyingSink(shared); + sink.setSource(source); + source.setSink(sink); + return { writable: new NewWritableStream(sink), readable: new NewReadableStream(source) }; +} diff --git a/reference-implementation/lib/experimental/thin-readable-byte-stream.js b/reference-implementation/lib/experimental/thin-readable-byte-stream.js new file mode 100644 index 000000000..eb65e7463 --- /dev/null +++ b/reference-implementation/lib/experimental/thin-readable-byte-stream.js @@ -0,0 +1,149 @@ +import { readableAcceptsCancel } from './stream-base'; + +export class NewReadableByteStream { + _initReadyPromise() { + this._readyPromise = new Promise((resolve, reject) => { + this._resolveReadyPromise = resolve; + }); + } + + _initPullReadyPromise() { + this._pullReadyPromise = new Promise((resolve, reject) => { + this._resolvePullReadyPromise = resolve; + }); + } + + constructor(source) { + this._source = source; + + this._state = 'waiting'; + + this._initReadyPromise(); + + this._erroredPromise = new Promise((resolve, reject) => { + this._resolveErroredPromise = resolve; + }); + this._error = undefined; + + this._pullable = false; + this._initPullReadyPromise(); + + const delegate = { + markPullable: this._markPullable.bind(this), + markNotPullable: this._markNotPullable.bind(this), + + markWaiting: this._markWaiting.bind(this), + markReadable: this._markReadable.bind(this), + markClosed: this._markClosed.bind(this), + + markErrored: this._markErrored.bind(this) + }; + + this._source.start(delegate); + } + + get state() { + return this._state; + } + + // Manual pull interfaces. + + get pullable() { + return this._pullable; + } + + get pullReady() { + return this._pullReadyPromise; + } + + pull(view) { + if (!this._pullable) { + throw new TypeError('not pullable'); + } + + return this._source.pull(view); + } + + // Reading interfaces. + + get ready() { + return this._readyPromise; + } + + read() { + if (this._state !== 'readable') { + throw new TypeError('not readable'); + } + + return this._source.read(); + } + + cancel(reason) { + if (!readableAcceptsCancel(this._state)) { + throw new TypeError('already ' + this._state); + } + + this._source.cancel(reason); + + this._state = 'cancelled'; + } + + // Error receiving interfaces. + + get errored() { + return this._erroredPromise; + } + + get error() { + if (this._state !== 'errored') { + throw new TypeError('not errored'); + } + + return this._error; + } + + // Methods exposed only to the underlying source. + + _markNotPullable() { + if (!this._pullable) { + return; + } + + this._initPullReadyPromise(); + this._pullable = false; + } + + _markPullable() { + if (this._pullable) { + return; + } + + this._resolvePullReadyPromise(); + this._resolvePullReadyPromise = undefined; + this._pullable = true; + } + + _markWaiting() { + this._initReadyPromise(); + this._state = 'waiting'; + } + + _markReadable() { + if (this._state === 'readable') { + return; + } + + this._resolveReadyPromise(); + this._state = 'readable'; + } + + _markClosed() { + this._state = 'closed'; + } + + _markErrored(error) { + this._resolveErroredPromise(); + this._state = 'errored'; + this._error = error; + } +} diff --git a/reference-implementation/lib/experimental/thin-readable-stream.js b/reference-implementation/lib/experimental/thin-readable-stream.js new file mode 100644 index 000000000..0dc9008d5 --- /dev/null +++ b/reference-implementation/lib/experimental/thin-readable-stream.js @@ -0,0 +1,124 @@ +import { readableAcceptsCancel } from './new-stream-base'; + +export class NewReadableStream { + _initReadyPromise() { + this._readyPromise = new Promise((resolve, reject) => { + this._resolveReadyPromise = resolve; + }); + } + + constructor(source) { + this._source = source; + + this._state = 'waiting'; + + this._initReadyPromise(); + + this._erroredPromise = new Promise((resolve, reject) => { + this._resolveErroredPromise = resolve; + }); + this._error = undefined; + + this._window = 0; + + const delegate = { + markWaiting: this._markWaiting.bind(this), + markReadable: this._markReadable.bind(this), + markClosed: this._markClosed.bind(this), + + markErrored: this._markErrored.bind(this) + }; + + this._source.start(delegate); + } + + get state() { + return this._state; + } + + // Auto pull interfaces. + + get window() { + return this._window; + } + + set window(v) { + if (!readableAcceptsCancel(this._state)) { + throw new TypeError('already ' + this._state); + } + + this._window = v; + + this._source.onWindowUpdate(v); + } + + // Reading interfaces. + + get ready() { + return this._readyPromise; + } + + read() { + if (this._state !== 'readable') { + throw new TypeError('not readable'); + } + + return this._source.read(); + } + + cancel(reason) { + if (!readableAcceptsCancel(this._state)) { + throw new TypeError('already ' + this._state); + } + + this._source.cancel(reason); + + this._state = 'cancelled'; + } + + // Error receiving interfaces. + + get errored() { + return this._erroredPromise; + } + + get error() { + if (this._state !== 'errored') { + throw new TypeError('not errored'); + } + + return this._error; + } + + // Methods exposed only to the underlying source. + + _markWaiting() { + this._initReadyPromise(); + this._state = 'waiting'; + } + + _markReadable() { + if (this._state === 'readable') { + return; + } + + this._resolveReadyPromise(); + this._resolveReadyPromise = undefined; + this._state = 'readable'; + } + + _markClosed() { + if (this._state !== 'readable') { + this._resolveReadyPromise(); + this._resolveReadyPromise = undefined; + } + this._state = 'closed'; + } + + _markErrored(error) { + this._resolveErroredPromise(); + this._resolveErroredPromise = undefined; + this._state = 'errored'; + this._error = error; + } +} diff --git a/reference-implementation/lib/experimental/thin-stream-base.js b/reference-implementation/lib/experimental/thin-stream-base.js new file mode 100644 index 000000000..5a3eccb4c --- /dev/null +++ b/reference-implementation/lib/experimental/thin-stream-base.js @@ -0,0 +1,92 @@ +export function writableAcceptsWriteAndClose(state) { + return state === 'waiting' || state === 'writable'; +} + +export function writableAcceptsAbort(state) { + return state === 'waiting' || state === 'writable' || state === 'closed'; +} + +export function readableAcceptsCancel(state) { + return state === 'waiting' || state === 'readable'; +} + +// Exported as a helper for building transformation. +export function selectStreams(readable, writable) { + const promises = []; + + promises.push(readable.errored); + if (readable.state === 'waiting') { + promises.push(readable.ready); + } + + promises.push(writable.errored); + if (writable.state === 'writable') { + promises.push(writable.waitSpaceChange()); + } else if (writable.state === 'waiting') { + promises.push(writable.ready); + } + + return Promise.race(promises); +} + +// Pipes data from source to dest with no transformation. Abort signal, cancel signal and space are also propagated +// between source and dest. +export function pipeStreams(source, dest) { + return new Promise((resolve, reject) => { + const oldWindow = source.window; + + function disposeStreams(error) { + if (writableAcceptsAbort(dest.state)) { + dest.abort(error); + } + if (readableAcceptsCancel(source.state)) { + source.abort(error); + } + reject(error); + } + + function loop() { + for (;;) { + if (source.state === 'errored') { + if (writableAcceptsAbort(dest.state)) { + dest.abort(source.error); + } + reject(new TypeError('source is errored')); + return; + } + + if (dest.state === 'errored') { + if (readableAcceptsCancel(source.state)) { + source.cancel(dest.error); + } + reject(new TypeError('dest is errored')); + return; + } + + if (source.state === 'closed') { + dest.close(); + resolve(); + return; + } + + console.log('sjsjsjs' + source.state + dest.state); + + if (source.state === 'readable') { + if (dest.state === 'writable') { + console.log('sjsjsjs'); + dest.write(source.read()); + continue; + } + } else { + source.window = dest.space; + } + + selectStreams(source, dest) + .then(loop) + .catch(disposeStreams); + return; + } + } + loop(); + }); +} diff --git a/reference-implementation/lib/experimental/thin-stream-utils.js b/reference-implementation/lib/experimental/thin-stream-utils.js new file mode 100644 index 000000000..4775095d0 --- /dev/null +++ b/reference-implementation/lib/experimental/thin-stream-utils.js @@ -0,0 +1,8 @@ +export function fillArrayBufferView(view, c, size) { + if (size === undefined) { + size = view.byteLength; + } + for (var i = 0; i < size; ++i) { + view[i] = c; + } +} diff --git a/reference-implementation/lib/experimental/thin-writable-byte-stream.js b/reference-implementation/lib/experimental/thin-writable-byte-stream.js new file mode 100644 index 000000000..29726123d --- /dev/null +++ b/reference-implementation/lib/experimental/thin-writable-byte-stream.js @@ -0,0 +1,201 @@ +import { writableAcceptsWriteAndClose, writableAcceptsAbort } from './new-stream-base'; + +export class NewWritableByteStream { + _initReadyPromise() { + this._readyPromise = new Promise((resolve, reject) => { + this._resolveReadyPromise = resolve; + }); + } + + _initDisposedViewReadyPromise() { + this._disposedViewReadyPromise = new Promise((resolve, reject) => { + this._resolveDisposedViewReadyPromise = resolve; + }); + } + + constructor(sink) { + this._sink = sink; + + this._state = 'waiting'; + + this._erroredPromise = new Promise((resolve, reject) => { + this._resolveErroredPromise = resolve; + }); + this._error = undefined; + + this._initReadyPromise(); + this._lastSpace = undefined; + this._spaceChangePromise = undefined; + + this._initDisposedViewReadyPromise(); + this._hasDisposedView = false; + + const delegate = { + markWaiting: this._markWaiting.bind(this), + markWritable: this._markWritable.bind(this), + markErrored: this._markErrored.bind(this), + onSpaceChange: this._onSpaceChange.bind(this), + + markHasDisposedView: this._markHasDisposedView.bind(this), + markNoDisposedView: this._markNoDisposedView.bind(this), + }; + + this._sink.start(delegate); + } + + get state() { + return this._state; + } + + // Main interfaces. + + waitSpaceChange() { + if (!writableAcceptsWriteAndClose(this._state)) { + throw new TypeError('already ' + this._state); + } + + if (this._spaceChangePromise !== undefined) { + return this._spaceChangePromise; + } + + this._spaceChangePromise = new Promise((resolve, reject) => { + this._resolveSpaceChangePromise = resolve; + }); + this._lastSpace = this.space; + + return this._spaceChangePromise; + } + + write(view) { + if (!writableAcceptsWriteAndClose(this._state)) { + throw new TypeError('already ' + this._state); + } + + this._sink.write(view); + } + + close() { + if (!writableAcceptsWriteAndClose(this._state)) { + throw new TypeError('already ' + this._state); + } + + this._sink.close(); + + this._state = 'closed'; + } + + abort(reason) { + if (!writableAcceptsAbort(this._state)) { + throw new TypeError('already ' + this._state); + } + + this._sink.abort(reason); + + this._state = 'aborted'; + } + + // Error receiving interfaces. + + get errored() { + return this._erroredPromise; + } + + get error() { + if (this._state !== 'errored') { + throw new TypeError('not errored'); + } + + return this._error; + } + + // Flow control interfaces. + + get ready() { + return this._readyPromise; + } + + get space() { + if (!writableAcceptsWriteAndClose(this._state)) { + throw new TypeError('already ' + this._state); + } + + return this._sink.space; + } + + // Disposed buffer reading interfaces. + + get hasDisposedView() { + return this._hasDisposedView; + } + + get disposedViewReady() { + return this._disposedViewReady; + } + + get readDisposedView() { + if (this._disposedViews.length === 0) { + throw new TypeError('no disposed view'); + } + + return this._sink.readDisposedView(); + } + + // Methods exposed only to the underlying sink. + + _markWaiting() { + if (this._state === 'waiting') { + return; + } + + this._initReadyPromise(); + this._state = 'waiting'; + } + + _markWritable() { + if (this._state === 'writable') { + return; + } + + this._resolveReadyPromise(); + this._resolveReadyPromise = undefined; + this._state = 'writable'; + } + + _markErrored(error) { + this._resolveErroredPromise(); + this._resolveErroredPromise = undefined; + this._state = 'errored'; + this._error = error; + } + + _onSpaceChange() { + if (this._spaceChangePromise === undefined || this._lastSpace == this.space) { + return; + } + + this._resolveSpaceChangePromise(); + this._resolveSpaceChangePromise = undefined; + + this._lastSpace = undefined; + this._spaceChangePromise = undefined; + } + + _markNoDisposedView() { + if (!this._hasDisposedView) { + return; + } + + this._initDisposedViewReadyPromise(); + this._hasDisposedView = false; + } + + _markHasDisposedView() { + if (this._hasDisposedView) { + return; + } + + this._resolveDisposedViewReadyPromise(); + this._resolveDisposedViewReadyPromise = undefined; + this._hasDisposedView = true; + } +} diff --git a/reference-implementation/lib/experimental/thin-writable-stream.js b/reference-implementation/lib/experimental/thin-writable-stream.js new file mode 100644 index 000000000..7ac3b3c6f --- /dev/null +++ b/reference-implementation/lib/experimental/thin-writable-stream.js @@ -0,0 +1,152 @@ +import { writableAcceptsWriteAndClose, writableAcceptsAbort } from './new-stream-base'; + +export class NewWritableStream { + _initReadyPromise() { + this._readyPromise = new Promise((resolve, reject) => { + this._resolveReadyPromise = resolve; + }); + } + + constructor(sink) { + this._sink = sink; + + this._state = 'waiting'; + + this._erroredPromise = new Promise((resolve, reject) => { + this._resolveErroredPromise = resolve; + }); + this._error = undefined; + + this._initReadyPromise(); + this._lastSpace = undefined; + this._spaceChangePromise = undefined; + + const delegate = { + markWaiting: this._markWaiting.bind(this), + markWritable: this._markWritable.bind(this), + markErrored: this._markErrored.bind(this), + onSpaceChange: this._onSpaceChange.bind(this) + }; + + this._sink.start(delegate); + } + + get state() { + return this._state; + } + + // Main interfaces. + + waitSpaceChange() { + if (!writableAcceptsWriteAndClose(this._state)) { + throw new TypeError('already ' + this._state); + } + + if (this._spaceChangePromise !== undefined) { + return this._spaceChangePromise; + } + + this._spaceChangePromise = new Promise((resolve, reject) => { + this._resolveSpaceChangePromise = resolve; + }); + this._lastSpace = this.space; + + return this._spaceChangePromise; + } + + write(value) { + if (!writableAcceptsWriteAndClose(this._state)) { + throw new TypeError('already ' + this._state); + } + + this._sink.write(value); + } + + close() { + if (!writableAcceptsWriteAndClose(this._state)) { + throw new TypeError('already ' + this._state); + } + + this._sink.close(); + + this._state = 'closed'; + } + + abort(reason) { + if (!writableAcceptsAbort(this._state)) { + throw new TypeError('already ' + this._state); + } + + this._sink.abort(reason); + + this._state = 'aborted'; + } + + // Error receiving interfaces. + + get errored() { + return this._erroredPromise; + } + + get error() { + if (this._state !== 'errored') { + throw new TypeError('not errored'); + } + + return this._error; + } + + // Flow control interfaces. + + get ready() { + return this._readyPromise; + } + + get space() { + if (!writableAcceptsWriteAndClose(this._state)) { + throw new TypeError('already ' + this._state); + } + + return this._sink.space; + } + + // Methods exposed only to the underlying sink. + + _markWaiting() { + if (this._state === 'waiting') { + return; + } + + this._initReadyPromise(); + this._state = 'waiting'; + } + + _markWritable() { + if (this._state === 'writable') { + return; + } + + this._resolveReadyPromise(); + this._resolveReadyPromise = undefined; + this._state = 'writable'; + } + + _markErrored(error) { + this._resolveErroredPromise(); + this._resolveErroredPromise = undefined; + this._state = 'errored'; + this._error = error; + } + + _onSpaceChange() { + if (this._spaceChangePromise === undefined || this._lastSpace == this.space) { + return; + } + + this._resolveSpaceChangePromise(); + this._resolveSpaceChangePromise = undefined; + + this._lastSpace = undefined; + this._spaceChangePromise = undefined; + } +} diff --git a/reference-implementation/lib/experimental/writable-stream.js b/reference-implementation/lib/experimental/writable-stream.js deleted file mode 100644 index be2494c90..000000000 --- a/reference-implementation/lib/experimental/writable-stream.js +++ /dev/null @@ -1,244 +0,0 @@ -import { writableAcceptsWriteAndClose, writableAcceptsAbort } from './stream-base'; -import { ExclusiveStreamWriter } from './exclusive-stream-writer'; - -export class WritableStream { - _initWritablePromise() { - this._writablePromise = new Promise((resolve, reject) => { - this._resolveWritablePromise = resolve; - }); - } - - _syncStateAndWritablePromise() { - if (this._state === 'writable') { - if (this._resolveWritablePromise !== undefined) { - this._resolveWritablePromise(); - this._resolveWritablePromise = undefined; - } - } else { - if (this._resolveWritablePromise === undefined) { - this._initWritablePromise(); - } - } - } - - _syncStateAndErroredPromise() { - if (this._state === 'errored') { - // erroredPromise may be already fulfilled if this method is called on release of a writer. - if (this._resolveErroredPromise !== undefined) { - this._resolveErroredPromise(); - this._resolveErroredPromise = undefined; - } - } - } - - _syncSpaceAndSpaceChangePromise() { - if (this._spaceChangePromise !== undefined && this._lastSpace !== this.space) { - this._resolveSpaceChangePromise(); - this._resolveSpaceChangePromise = undefined; - - this._lastSpace = undefined; - this._spaceChangePromise = undefined; - } - } - - constructor(sink) { - this._sink = sink; - - this._state = 'waiting'; - - this._erroredPromise = new Promise((resolve, reject) => { - this._resolveErroredPromise = resolve; - }); - this._error = undefined; - - this._initWritablePromise(); - this._lastSpace = undefined; - this._spaceChangePromise = undefined; - - this._writer = undefined; - - const delegate = { - markWaiting: this._markWaiting.bind(this), - markWritable: this._markWritable.bind(this), - markErrored: this._markErrored.bind(this), - onSpaceChange: this._onSpaceChange.bind(this) - }; - - this._sink.init(delegate); - } - - get state() { - this._throwIfLocked(); - return this._state; - } - - // Main interfaces. - - _waitSpaceChangeIgnoringLock() { - if (!writableAcceptsWriteAndClose(this._state)) { - throw new TypeError('already ' + this._state); - } - - if (this._spaceChangePromise !== undefined) { - return this._spaceChangePromise; - } - - this._spaceChangePromise = new Promise((resolve, reject) => { - this._resolveSpaceChangePromise = resolve; - }); - this._lastSpace = this.space; - - return this._spaceChangePromise; - } - waitSpaceChange() { - this._throwIfLocked(); - return this._waitSpaceChangeIgnoringLock(); - } - - _writeIgnoringLock(value) { - if (!writableAcceptsWriteAndClose(this._state)) { - throw new TypeError('already ' + this._state); - } - - return this._sink.write(value); - } - write(value) { - this._throwIfLocked(); - return this._writeIgnoringLock(value); - } - - _closeIgnoringLock() { - if (!writableAcceptsWriteAndClose(this._state)) { - throw new TypeError('already ' + this._state); - } - - const result = this._sink.close(); - - this._state = 'closed'; - - return result; - } - close() { - this._throwIfLocked(); - return this._closeIgnoringLock(); - } - - _abortIgnoringLock(reason) { - if (!writableAcceptsAbort(this._state)) { - throw new TypeError('already ' + this._state); - } - - const result = this._sink.abort(reason); - - this._state = 'aborted'; - - return result; - } - abort(reason) { - this._throwIfLocked(); - return this._abortIgnoringLock(reason); - } - - // Error receiving interfaces. - - get errored() { - this._throwIfLocked(); - return this._erroredPromise; - } - - get _errorIgnoringLock() { - if (this._state !== 'errored') { - throw new TypeError('not errored'); - } - return this._error; - } - get error() { - this._throwIfLocked(); - return this._errorIgnoringLock; - } - - // Flow control interfaces. - - get writable() { - this._throwIfLocked(); - return this._writablePromise; - } - - get _spaceIgnoringLock() { - if (!writableAcceptsWriteAndClose(this._state)) { - throw new TypeError('already ' + this._state); - } - - return this._sink.space; - } - get space() { - this._throwIfLocked(); - return this._spaceIgnoringLock; - } - - // Locking interfaces. - - _throwIfLocked() { - if (this._writer !== undefined) { - throw new TypeError('locked'); - } - } - - _releaseWriter() { - this._writer = undefined; - - this._syncStateAndWritablePromise(); - this._syncStateAndErroredPromise(); - this._syncSpaceAndSpaceChangePromise(); - } - - getWriter() { - this._throwIfLocked(); - this._writer = new ExclusiveStreamWriter(this); - return this._writer; - } - - // Methods exposed only to the underlying sink. - - _markWaiting() { - this._state = 'waiting'; - - if (this._writer === undefined) { - this._syncStateAndWritablePromise(); - } else { - this._writer._syncStateAndWritablePromise(); - } - } - - _markWritable() { - this._state = 'writable'; - - if (this._writer === undefined) { - this._syncStateAndWritablePromise(); - } else { - this._writer._syncStateAndWritablePromise(); - } - } - - _markErrored(error) { - this._state = 'errored'; - - if (this._writer === undefined) { - this._syncStateAndWritablePromise(); - this._syncStateAndErroredPromise(); - } else { - this._writer._syncStateAndWritablePromise(); - this._writer._syncStateAndErroredPromise(); - } - - this._error = error; - } - - _onSpaceChange() { - if (this._writer === undefined) { - this._syncSpaceAndSpaceChangePromise(); - } else { - this._writer._syncSpaceAndSpaceChangePromise(); - } - } -} diff --git a/reference-implementation/run-tests.js b/reference-implementation/run-tests.js index b810bf900..ff59bc59f 100644 --- a/reference-implementation/run-tests.js +++ b/reference-implementation/run-tests.js @@ -19,4 +19,4 @@ global.TransformStream = TransformStream; //const tests = glob.sync(path.resolve(__dirname, 'test/*.js')); //const experimentalTests = glob.sync(path.resolve(__dirname, 'test/experimental/*.js')); //tests.concat(experimentalTests).forEach(require); -glob.sync(path.resolve(__dirname, 'test/experimental/operation-stream.js')).forEach(require); +glob.sync(path.resolve(__dirname, 'test/experimental/new-stream.js')).forEach(require); diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 1f45f5ed0..0f5d787d7 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -1,426 +1,3 @@ -const test = require('tape-catch'); - -import { ReadableStream } from '../../lib/experimental/readable-stream'; -import { selectStreams } from '../../lib/experimental/stream-base'; -import { jointOps, pipeOperationStreams } from '../../lib/experimental/operation-stream'; -import { createOperationQueue } from '../../lib/experimental/operation-queue'; - -test('Operation stream pair is constructed', t => { - const pair = createOperationQueue({ - size() { - return 1; - }, - shouldApplyBackpressure() { - return false; - } - }); - - t.end(); -}); - -class NoBackpressureStrategy { -} - -class ApplyBackpressureWhenNonEmptyStrategy { - shouldApplyBackpressure(queueSize) { - return queueSize > 0; - } -} - -class AdjustableArrayBufferStrategy { - constructor() { - this._window = 0; - } - - size(ab) { - return ab.byteLength; - } - shouldApplyBackpressure(queueSize) { - return queueSize >= this._window; - } - space(queueSize) { - return Math.max(0, this._window - queueSize); - } - onWindowUpdate(window) { - this._window = window; - } -} - -class AdjustableStringStrategy { - constructor() { - this._window = 0; - } - - size(s) { - return s.length; - } - shouldApplyBackpressure(queueSize) { - return queueSize >= this._window; - } - space(queueSize) { - return Math.max(0, this._window - queueSize); - } - onWindowUpdate(window) { - this._window = window; - } -} - -test('Synchronous write, read and completion of the operation', t => { - const pair = createOperationQueue(new ApplyBackpressureWhenNonEmptyStrategy()); - const wos = pair.writable; - const ros = pair.readable; - - t.equals(wos.state, 'writable'); - t.equals(ros.state, 'waiting'); - - const status = wos.write('hello'); - - t.equals(wos.state, 'waiting'); - t.equals(ros.state, 'readable'); - - t.equals(status.state, 'waiting'); - - const op = ros.read(); - t.equals(op.argument, 'hello'); - - t.equals(ros.state, 'waiting'); - t.equals(wos.state, 'writable'); - - t.equals(status.state, 'waiting'); - - op.complete('world'); - - t.equals(status.state, 'completed'); - t.equals(status.result, 'world'); - - t.end(); -}); - -test('Asynchronous write, read and completion of the operation', t => { - const pair = createOperationQueue(new ApplyBackpressureWhenNonEmptyStrategy()); - const wos = pair.writable; - const ros = pair.readable; - - t.equals(ros.state, 'waiting'); - - var calledComplete = false; - - ros.readable.then(() => { - t.equals(ros.state, 'readable'); - - const op = ros.read(); - t.equals(op.argument, 'hello'); - - t.equals(ros.state, 'waiting'); - - op.complete('world'); - calledComplete = true; - }, e => { - t.fail(e); - t.end(); - }); - - t.equals(wos.state, 'writable'); - - const status = wos.write('hello'); - - t.equals(wos.state, 'waiting'); - - t.equals(status.state, 'waiting'); - status.ready.then(() => { - t.equals(calledComplete, true); - - t.equals(status.state, 'completed'); - t.equals(status.result, 'world'); - - t.end(); - }, e => { - t.fail(e); - t.end(); - }); -}); - -test('Asynchronous write, read and completion of the operation', t => { - const pair = createOperationQueue(new AdjustableArrayBufferStrategy()); - const wos = pair.writable; - const ros = pair.readable; - - t.equals(wos.state, 'waiting'); - - ros.window = 5; - t.equals(wos.state, 'writable'); - t.equals(wos.space, 5); - - ros.window = 0; - const status = wos.write(new ArrayBuffer(10)); - - t.equals(wos.state, 'waiting'); - t.equals(wos.space, 0); - - ros.window = 10; - t.equals(wos.state, 'waiting'); - t.equals(wos.space, 0); - - ros.window = 15; - t.equals(wos.state, 'writable'); - t.equals(wos.space, 5); - - ros.window = 20; - t.equals(wos.state, 'writable'); - t.equals(wos.space, 10); - - ros.read(); - - t.equals(wos.state, 'writable'); - t.equals(wos.space, 20); - - t.end(); -}); - -test('close()', t => { - const pair = createOperationQueue(new AdjustableArrayBufferStrategy()); - const wos = pair.writable; - const ros = pair.readable; - - wos.write(new ArrayBuffer(10)); - wos.close(); - t.equals(wos.state, 'closed'); - - t.equals(ros.state, 'readable'); - const v0 = ros.read().argument; - t.equals(ros.state, 'readable'); - const v1 = ros.read().argument; - t.equals(v1, ros.constructor.EOS); - t.equals(ros.state, 'drained'); - - t.end(); -}); - -test('abort()', t => { - t.plan(9); - - const pair = createOperationQueue(new AdjustableStringStrategy()); - const wos = pair.writable; - const ros = pair.readable; - - const helloStatus = wos.write('hello'); - const worldStatus = wos.write('world'); - - const testError = new TypeError('foo'); - const testCompletion = 'good'; - - const status = wos.abort(testError); - t.equals(wos.state, 'aborted', 'wos.state'); - status.ready.then(() => { - t.equals(status.state, 'completed', 'status.state'); - t.equals(status.result, testCompletion, 'status.result'); - }); - - helloStatus.ready.then(() => { - t.equals(helloStatus.state, 'errored', 'helloStatus.state'); - t.equals(helloStatus.result.message, 'aborted', 'helloStatus.result'); - }); - worldStatus.ready.then(() => { - t.equals(worldStatus.state, 'errored', 'worldStatus.state'); - t.equals(worldStatus.result.message, 'aborted', 'worldStatus.result'); - }); - ros.errored.then(() => { - t.equals(ros.state, 'errored', 'ros.state'); - t.equals(ros.error.argument, testError, 'ros.error.argument'); - - ros.error.complete(testCompletion); - }); -}); - -test('cancel()', t => { - t.plan(9); - - const pair = createOperationQueue(new AdjustableStringStrategy()); - const wos = pair.writable; - const ros = pair.readable; - - const helloStatus = wos.write('hello'); - const worldStatus = wos.write('world'); - - const testError = new TypeError('foo'); - const testCompletion = 'good'; - - const status = ros.cancel(testError); - t.equals(ros.state, 'cancelled', 'ros.state'); - status.ready.then(() => { - t.equals(status.state, 'completed', 'status.state'); - t.equals(status.result, testCompletion, 'status.result'); - }); - - helloStatus.ready.then(() => { - t.equals(helloStatus.state, 'errored', 'helloStatus.state'); - t.equals(helloStatus.result, testError, 'helloStatus.result'); - }); - worldStatus.ready.then(() => { - t.equals(worldStatus.state, 'errored', 'worldStatus.state'); - t.equals(worldStatus.result, testError, 'worldStatus.result'); - }); - wos.errored.then(() => { - t.equals(wos.state, 'errored', 'wos.state'); - t.equals(wos.error.argument, testError, 'wos.error.argument'); - - wos.error.complete(testCompletion); - }); -}); - -test('pipeOperationStreams()', t => { - const pair0 = createOperationQueue(new AdjustableStringStrategy()); - const wos0 = pair0.writable; - const ros0 = pair0.readable; - - const pair1 = createOperationQueue(new AdjustableStringStrategy()); - const wos1 = pair1.writable; - const ros1 = pair1.readable; - - var helloStatus; - t.equals(wos0.state, 'waiting'); - // Check that wos0 becomes writable. - wos0.writable.then(() => { - t.equals(wos0.state, 'writable'); - - helloStatus = wos0.write('hello'); - - // Just write without state check. - wos0.write('world'); - wos0.close(); - }); - - pipeOperationStreams(ros0, wos1) - .catch(e => { - t.fail(e); - t.end(); - }); - - t.equals(ros1.state, 'waiting'); - - ros1.window = 20; - - t.equals(ros1.state, 'waiting'); - - ros1.readable.then(() => { - t.equals(ros1.state, 'readable'); - const op0 = ros1.read(); - t.equals(op0.type, 'data'); - t.equals(op0.argument, 'hello'); - - op0.complete('hi'); - - t.equals(ros1.state, 'readable'); - const op1 = ros1.read(); - t.equals(op1.type, 'data'); - t.equals(op1.argument, 'world'); - - t.equals(ros1.state, 'readable'); - const op2 = ros1.read(); - t.equals(op2.type, 'close'); - - t.equals(helloStatus.state, 'waiting'); - helloStatus.ready.then(() => { - t.equals(helloStatus.state, 'completed'); - t.equals(helloStatus.result, 'hi'); - t.end(); - }); - }).catch(e => { - t.fail(e); - t.end(); - }); -}); - -test('pipeOperationStreams(): abort() propagation', t => { - t.plan(9); - - const pair0 = createOperationQueue(new AdjustableStringStrategy()); - const wos0 = pair0.writable; - const ros0 = pair0.readable; - - const pair1 = createOperationQueue(new AdjustableStringStrategy()); - const wos1 = pair1.writable; - const ros1 = pair1.readable; - - const helloStatus = wos0.write('hello'); - const worldStatus = wos0.write('world'); - - const testError = new TypeError('foo'); - const testCompletion = 'good'; - - const pipePromise = pipeOperationStreams(ros0, wos1); - pipePromise - .then( - v => t.fail('pipePromise is fulfilled with ' + v), - e => t.equals(e.message, 'source is errored', 'rejection reason of pipePromise')); - - const status = wos0.abort(testError); - status.ready.then(() => { - t.equals(status.state, 'completed', 'status.state'); - t.equals(status.result, testCompletion, 'status.result'); - }); - - helloStatus.ready.then(() => { - t.equals(helloStatus.state, 'errored', 'helloStatus.state'); - t.equals(helloStatus.result.message, 'aborted', 'helloStatus.result'); - }); - worldStatus.ready.then(() => { - t.equals(worldStatus.state, 'errored', 'worldStatus.state'); - t.equals(worldStatus.result.message, 'aborted', 'worldStatus.result'); - }); - ros1.errored.then(() => { - t.equals(ros1.state, 'errored', 'ros.state'); - t.equals(ros1.error.argument, testError, 'ros1.error.argument'); - - ros1.error.complete(testCompletion); - }); -}); - -test('pipeOperationStreams(): cancel() propagation', t => { - t.plan(9); - - const pair0 = createOperationQueue(new AdjustableStringStrategy()); - const wos0 = pair0.writable; - const ros0 = pair0.readable; - - const pair1 = createOperationQueue(new AdjustableStringStrategy()); - const wos1 = pair1.writable; - const ros1 = pair1.readable; - - const helloStatus = wos0.write('hello'); - const worldStatus = wos0.write('world'); - - const testError = new TypeError('foo'); - const testCompletion = 'good'; - - const pipePromise = pipeOperationStreams(ros0, wos1); - pipePromise - .then( - v => t.fail('pipePromise is fulfilled with ' + v), - e => t.equals(e.message, 'dest is errored', 'rejection reason of pipePromise')); - - const status = ros1.cancel(testError); - status.ready.then(() => { - t.equals(status.state, 'completed', 'status.state'); - t.equals(status.result, testCompletion, 'status.result'); - }); - - helloStatus.ready.then(() => { - t.equals(helloStatus.state, 'errored', 'helloStatus.state'); - t.equals(helloStatus.result, testError, 'helloStatus.result'); - }); - worldStatus.ready.then(() => { - t.equals(worldStatus.state, 'errored', 'worldStatus.state'); - t.equals(worldStatus.result, testError, 'worldStatus.result'); - }); - wos0.errored.then(() => { - t.equals(wos0.state, 'errored', 'wos0.state'); - t.equals(wos0.error.argument, testError, 'wos0.error.argument'); - - wos0.error.complete(testCompletion); - }); -}); - test('Transformation example: Byte counting', t => { function byteCountingTransform(source, dest) { return new Promise((resolve, reject) => { @@ -486,827 +63,32 @@ test('Transformation example: Byte counting', t => { } const pair0 = createOperationQueue(new AdjustableStringStrategy()); - const wos0 = pair0.writable; - const ros0 = pair0.readable; + const ws0 = pair0.writable; + const rs0 = pair0.readable; const pair1 = createOperationQueue(new AdjustableStringStrategy()); - const wos1 = pair1.writable; - const ros1 = pair1.readable; + const ws1 = pair1.writable; + const rs1 = pair1.readable; - byteCountingTransform(ros0, wos1) + byteCountingTransform(rs0, ws1) .catch(e => { t.fail(e); t.end(); }); - wos0.write('hello'); - wos0.write('world'); - wos0.write('goodbye'); - wos0.close(); - - ros1.window = 1; - - ros1.readable.then(() => { - const v = ros1.read().argument; - t.equals(v, 17); - t.end(); - }).catch(e => { - t.fail(e); - t.end(); - }); -}); - -function fillArrayBufferView(view, c, size) { - if (size === undefined) { - size = view.byteLength; - } - for (var i = 0; i < size; ++i) { - view[i] = c; - } -} - -class FakeFileBackedByteSource { - constructor() { - this._bytesToWrite = 1024; - } - - createBufferProducingStreamWithPool(buffers) { - class Puller { - constructor(file, buffers, writableStream) { - this._writableStream = writableStream; - - this._file = file; - this._fileReadPromise = undefined; - - this._buffersAvailable = buffers; - - this._buffersPassedToUser = []; - - this._loop(); - } - - _handleFileReadResult(buffer, result) { - this._fileReadPromise = undefined; - const status = this._writableStream.write(result.view); - this._buffersPassedToUser.push({buffer, status}); - if (result.closed) { - this._writableStream.close(); - } - } - - _select() { - const promises = []; - - const ws = this._writableStream; - - promises.push(ws.errored); - if (ws.state === 'waiting') { - promises.push(ws.writable); - } - - if (this._fileReadPromise !== undefined) { - promises.push(this._fileReadPromise); - } - - if (this._buffersPassedToUser.length > 0) { - promises.push(this._buffersPassedToUser[0].status.ready); - } - - Promise.race(promises) - .then(this._loop.bind(this)) - .catch(ws.abort.bind(ws)); - } - - _loop() { - const ws = this._writableStream; - - for (;;) { - if (ws.state === 'errored') { - return; - } - - var hasProgress = false; - - if (this._buffersPassedToUser.length > 0) { - const entry = this._buffersPassedToUser[0]; - const status = entry.status; - if (status.state === 'completed') { - this._buffersPassedToUser.shift(); - - this._buffersAvailable.push(entry.buffer); - - hasProgress = true; - // Keep going. - } else if (status.state === 'errored') { - ws.abort(status.result); - return; - } - } - - if (this._fileReadPromise === undefined && - this._buffersAvailable.length > 0 && - ws.state === 'writable') { - const buffer = this._buffersAvailable.shift(); - - // Clear the acquired buffer for testing. - const view = new Uint8Array(buffer); - fillArrayBufferView(view, 0); - - this._fileReadPromise = this._file._readInto(view) - .then(this._handleFileReadResult.bind(this, buffer)) - .catch(ws.abort); - - hasProgress = true; - } - - if (hasProgress) { - continue; - } - - this._select(); - return; - } - } - } - - const pair = createOperationQueue(new AdjustableArrayBufferStrategy()); - new Puller(this, buffers, pair.writable); - return pair.readable; - } - - // Returns a WritableOperationStream. - // - // Example semantics: - // - // POSIX socket: - // - The stream becomes writable when epoll(7) returns, and stays to be writable until read(2) returns EAGAIN. - // - The status returned on write() call on the stream will set to the number of bytes written into the buffer passed - // on write() call. - // - // Blocking or async I/O interfaces which takes a buffer on reading function call: - // - The stream is always writable. - // - Same as POSIX socket. - // - // I/O that tells us how much data can be read: - // - Hide window setter/getter to the user but adjust based on the info (how much data can be read), so that the user - // can know the info via space getter. - createBufferFillingStream() { - class Filler { - constructor(file, readableStream) { - this._readableStream = readableStream; - this._readableStream.window = 1; - - this._currentRequest = undefined; - - this._file = file; - - this._fileReadStatus = undefined; - - this._loop(); - } - - _handleFileReadCompletion() { - this._currentRequest.complete(this._fileReadStatus.result); - this._currentRequest = undefined; - - this._fileReadStatus = undefined; - } - - _handleFileReadError() { - const error = this._fileReadStatus.result; - - this._currentRequest.error(error); - this._currentRequest = undefined; - - this._readableStream.cancel(error); - - this._fileReadStatus = undefined; - } - - _transformReadFromFileFulfillment(result) { - this._fileReadStatus.state = 'completed'; - this._fileReadStatus.result = {closed: result.closed, bytesWritten: result.view.byteLength}; - } - - _transformReadFromFileRejection(e) { - this._fileReadStatus.state = 'errored'; - this._fileReadStatus.result = e; - } - - _readFromFile(op) { - this._currentRequest = op; - - const status = {}; - status.state = 'waiting'; - status.ready = this._file._readInto(op.argument) - .then(this._transformReadFromFileFulfillment.bind(this)) - .catch(this._transformReadFromFileRejection.bind(this)); - - this._fileReadStatus = status; - } - - _select() { - const promises = []; - - const rs = this._readableStream; - - promises.push(rs.errored); - if (rs.state === 'waiting') { - promises.push(rs.readable); - } - - if (this._fileReadStatus !== undefined && this._fileReadStatus.state === 'waiting') { - promises.push(this._fileReadStatus.ready); - } - - Promise.race(promises) - .then(this._loop.bind(this)) - .catch(rs.cancel.bind(rs)); - } - - _loop() { - const rs = this._readableStream; - - for (;;) { - if (rs.state === 'errored') { - if (this._currentRequest !== undefined) { - this._currentRequest.error(rs.error.argument); - } - - return; - } - - if (this._currentRequest !== undefined) { - if (this._fileReadStatus.state === 'errored') { - this._handleFileReadError(); - return; - } else if (this._fileReadStatus.state === 'completed') { - this._handleFileReadCompletion(); - } - } - - if (rs.state === 'readable' && this._currentRequest === undefined) { - const op = this._readableStream.read(); - - if (op.type === 'close') { - return; - } else if (op.type === 'data') { - this._readFromFile(op); - continue; - } else { - // error - } - } - - this._select(); - return; - } - } - } - - const pair = createOperationQueue(new AdjustableArrayBufferStrategy()); - new Filler(this, pair.readable); - return pair.writable; - } - - _readInto(view) { - return new Promise((resolve, reject) => { - setTimeout(() => { - try { - const bytesToWriteThisTime = Math.min(this._bytesToWrite, view.byteLength); - - fillArrayBufferView(view, 1, bytesToWriteThisTime); - - this._bytesToWrite -= bytesToWriteThisTime; - - const writtenRegion = new Uint8Array(view.buffer, view.byteOffset, bytesToWriteThisTime); - if (this._bytesToWrite === 0) { - resolve({closed: true, view: writtenRegion}); - } else { - resolve({closed: false, view: writtenRegion}); - } - } catch (e) { - reject(e); - } - }, 0); - }); - } -} - -class BytesSetToOneExpectingByteSinkInternalWriter { - constructor(sink, readableStream) { - this._readableStream = readableStream; - this._readableStream.window = 64; - - this._sink = sink; - - this._loop(); - } - - _loop() { - const rs = this._readableStream; - - for (;;) { - if (rs.state === 'readable') { - const op = rs.read(); - if (op.type === 'data') { - const view = op.argument; - - // Verify contents of the buffer. - for (var i = 0; i < view.byteLength; ++i) { - if (view[i] === 1) { - this._sink._count(1); - } - } - - // Release the buffer. - op.complete(); - - continue; - } else if (op.type === 'close') { - // Acknowledge the closure. - op.complete(); - - this._sink._complete(); - } else { - this._sink._error(op.type); - } - } else if (rs.state === 'waiting') { - rs.readable - .then(this._loop.bind(this)) - .catch(this._sink._error.bind(this._sink)); - } else if (rs.state === 'errored') { - this._sink._error(rs.error.argument); - } else { - this._sink._error(rs.state); - } - return; - } - } -} - -class BytesSetToOneExpectingByteSink { - constructor() { - this._bytesRead = 0; - - this._resultPromise = new Promise((resolve, reject) => { - this._resolveResultPromise = resolve; - this._rejectResultPromise = reject; - }); - } - - get result() { - return this._resultPromise; - } - - _count(size) { - this._bytesRead += size; - } - - _complete() { - this._resolveResultPromise(this._bytesRead); - } - - _error(e) { - this._rejectResultPromise(e); - } - - createBufferProducingStream() { - class BufferProvidingWriter { - constructor(sink, writableStream) { - this._writableStream = writableStream; - - this._currentReadStatus = undefined; - - this._buffer = new ArrayBuffer(16); - - this._sink = sink; - - this._loop(); - } - - _handleReadCompletion() { - const bytesWritten = this._currentReadStatus.result.bytesWritten; - const closed = this._currentReadStatus.result.closed; - - // Verify contents of the buffer. - const view = new Uint8Array(this._buffer, 0, bytesWritten); - for (var i = 0; i < bytesWritten; ++i) { - if (view[i] === 1) { - this._sink._count(1); - } - } - - this._currentReadStatus = undefined; - - return closed; - } - - _select() { - const promises = []; - - const ws = this._writableStream; - - promises.push(ws.errored); - if (ws.state === 'waiting') { - promises.push(ws.writable); - } - - if (this._currentReadStatus !== undefined && this._currentReadStatus.state === 'waiting') { - promises.push(this._currentReadStatus.ready); - } - - Promise.race(promises) - .then(this._loop.bind(this)) - .catch(this._sink._error.bind(this._sink)); - } - - _loop() { - const ws = this._writableStream; - - for (;;) { - if (this._currentReadStatus !== undefined && this._currentReadStatus.state === 'errored') { - const error = this._currentReadStatus.result; - this._currentReadStatus = undefined; - - ws.abort(error); - this._sink._error(error); - - return; - } - - if (ws.state === 'errored') { - this._sink._error(ws.error.argument); - return; - } - - let hasProgress = false; - - if (this._currentReadStatus !== undefined && this._currentReadStatus.state === 'completed') { - const closed = this._handleReadCompletion(); - if (closed) { - ws.close(); - this._sink._complete(); - return; - } - - hasProgress = true; - } - - if (ws.state === 'writable' && this._currentReadStatus === undefined) { - this._currentReadStatus = ws.write(new Uint8Array(this._buffer)); - - hasProgress = true; - } - - if (hasProgress) { - continue; - } - - this._select(); - return; - } - } - } - - const pair = createOperationQueue(new AdjustableArrayBufferStrategy()); - new BufferProvidingWriter(this, pair.writable); - return pair.readable; - } - - createBufferConsumingStream() { - const pair = createOperationQueue(new AdjustableArrayBufferStrategy()); - new BytesSetToOneExpectingByteSinkInternalWriter(this, pair.readable); - return pair.writable; - } - - writeFrom(readableStream) { - new BytesSetToOneExpectingByteSinkInternalWriter(this, readableStream); - } - -} - -test('Piping from a source with a buffer pool to a buffer taking sink', t => { - const pool = []; - for (var i = 0; i < 10; ++i) { - pool.push(new ArrayBuffer(10)); - } - - const file = new FakeFileBackedByteSource(); - - const sink = new BytesSetToOneExpectingByteSink(); - const bufferConsumingStream = sink.createBufferConsumingStream(); - - // pipeOperationStreams automatically adjusts window of the readable side. - const pipePromise = pipeOperationStreams(file.createBufferProducingStreamWithPool(pool), bufferConsumingStream); - pipePromise.catch(e => { - t.fail(e); - t.end(); - }); - - sink.result.then(bytesRead => { - t.equals(bytesRead, 1024); - - pipePromise.then(() => { - Promise.resolve().then(() => { - // Check that the buffers have been returned to the pool. - t.equals(pool.length, 10); - t.end(); - }); - }); - }).catch(e => { - t.fail(e); - t.end(); - }); -}); - -test('Consuming bytes from a source with a buffer pool via the ReadableOperationStream interface', t => { - const pool = []; - for (var i = 0; i < 10; ++i) { - pool.push(new ArrayBuffer(10)); - } - - const file = new FakeFileBackedByteSource(); - const bufferProducingStream = file.createBufferProducingStreamWithPool(pool); - bufferProducingStream.window = 64; + ws0.write('hello'); + ws0.write('world'); + ws0.write('goodbye'); + ws0.close(); - const sink = new BytesSetToOneExpectingByteSink(); + rs1.window = 1; - sink.writeFrom(bufferProducingStream); - - sink.result.then(bytesRead => { - t.equals(bytesRead, 1024); - - Promise.resolve().then(() => { - Promise.resolve().then(() => { - // Check that the buffers have been returned to the pool. - t.equals(pool.length, 10); - t.end(); - }); - }); - }).catch(e => { - t.fail(e); - t.end(); - }); -}); - -test('Piping from a buffer taking source to a sink with buffer', t => { - const file = new FakeFileBackedByteSource(); - const bufferFillingStream = file.createBufferFillingStream(); - bufferFillingStream.window = 16; - // This also works. - // writableBufferProvidingStream.window = 16; - - const sink = new BytesSetToOneExpectingByteSink(); - const writableBufferProducingStream = sink.createBufferProducingStream(); - - const pipePromise = pipeOperationStreams(writableBufferProducingStream, bufferFillingStream); - pipePromise.catch(e => { - t.fail(e); + rs1.readable.then(() => { + const v = rs1.read().argument; + t.equal(v, 17); t.end(); - }); - - sink.result.then(bytesRead => { - t.equals(bytesRead, 1024); - pipePromise.then(() => t.end()); }).catch(e => { t.fail(e); t.end(); }); }); - -test('getWriter()', t => { - const pair = createOperationQueue(new AdjustableArrayBufferStrategy()); - const wos = pair.writable; - const ros = pair.readable; - - const writer = wos.getWriter(); - - t.throws(() => wos.state, /TypeError/); - t.throws(() => wos.writable, /TypeError/); - t.throws(() => wos.errored, /TypeError/); - t.throws(() => wos.error, /TypeError/); - t.throws(() => wos.space, /TypeError/); - t.throws(() => wos.waitSpaceChange(), /TypeError/); - t.throws(() => wos.write(new ArrayBuffer(10)), /TypeError/); - t.throws(() => wos.close(), /TypeError/); - t.throws(() => wos.abort(), /TypeError/); - t.throws(() => wos.getWriter(), /TypeError/); - - t.equals(writer.state, 'waiting'); - - ros.window = 1; - - t.equals(writer.state, 'writable'); - t.ok(writer.writable); - t.ok(writer.errored); - t.ok(writer.space); - t.ok(writer.waitSpaceChange()); - const writeStatus = writer.write(new ArrayBuffer(10)); - t.equals(writer.state, 'waiting'); - const closeStatus = writer.close(); - t.equals(writer.state, 'closed'); - const abortStatus = writer.abort(); - t.equals(writer.state, 'aborted'); - - writer.release(); - - t.equals(wos.state, 'aborted'); - t.ok(wos.writable); - t.ok(wos.errored); - - t.end(); -}); - -test('writable and waitSpaceChange() on the stream obtained before getWriter() call', t => { - t.plan(2); - - const pair = createOperationQueue(new AdjustableArrayBufferStrategy()); - const wos = pair.writable; - const ros = pair.readable; - - let released = false; - - wos.writable.then(r => { - if (released) { - t.pass(); - } else { - t.fail(r); - } - }); - - wos.waitSpaceChange().then(r => { - if (released) { - t.pass(); - } else { - t.fail(r); - } - }); - - const writer = wos.getWriter(); - - ros.window = 1; - - setTimeout(() => { - writer.release(); - - released = true; - }, 10); -}); - -test('errored on the stream obtained before getWriter() call', t => { - t.plan(1); - - const pair = createOperationQueue(new AdjustableArrayBufferStrategy()); - const wos = pair.writable; - const ros = pair.readable; - - let released = false; - - wos.errored.then(r => { - if (released) { - t.pass(); - } else { - t.fail(r); - } - }); - - const writer = wos.getWriter(); - - ros.cancel(); - - setTimeout(() => { - writer.release(); - - released = true; - }, 10); -}); - -class FakeUnstoppablePushSource { - constructor(count) { - this._count = count; - - setTimeout(this._push.bind(this), 0); - } - - _push() { - if (this._count == 0) { - this.onend(); - return; - } - - this.ondata('foo'); - --this._count; - - setTimeout(this._push.bind(this), 0); - } -} - -test('Adapting unstoppable push source', t => { - class Source { - constructor() { - this._queue = []; - } - - init(delegate) { - this._pushSource = new FakeUnstoppablePushSource(10); - - t.equal(typeof delegate.markWaiting, 'function', 'markWaiting is a function'); - t.equal(typeof delegate.markReadable, 'function', 'markReadable is a function'); - t.equal(typeof delegate.markDrained, 'function', 'markDrained is a function'); - t.equal(typeof delegate.markErrored, 'function', 'markErrored is a function'); - - this._readableStreamDelegate = delegate; - - this._pushSource.ondata = chunk => { - this._queue.push({type: 'data', data: chunk}); - delegate.markReadable(); - }; - - this._pushSource.onend = () => { - this._queue.push({type: 'close'}); - delegate.markReadable(); - }; - - this._pushSource.onerror = () => { - this._queue = []; - delegate.markErrored(); - }; - } - - onWindowUpdate(v) { - } - - read() { - if (this._queue.length === 0) { - throw new TypeError('not readable'); - } - - const entry = this._queue.shift(); - - if (this._queue.length === 0) { - if (entry.type === 'close') { - this._readableStreamDelegate.markDrained(); - return ReadableStream.EOS; - } else { - this._readableStreamDelegate.markWaiting(); - return entry.data; - } - } - - return entry; - } - - cancel() { - this._queue = []; - - this._pushSource.close(); - } - } - - const source = new Source(); - const readableStream = new ReadableStream(source); - - let count = 0; - function pump() { - for (;;) { - if (readableStream.state === 'waiting') { - Promise.race([readableStream.readable, readableStream.errored]) - .then(pump) - .catch(e => { - t.fail(e); - t.end(); - }); - return; - } else if (readableStream.state === 'readable') { - const data = readableStream.read(); - if (count === 10) { - t.equals(data, ReadableStream.EOS); - t.end(); - return; - } else { - t.equals(data, 'foo'); - ++count; - } - } else if (readableStream.state === 'drained') { - t.fail(); - t.end(); - return; - } else if (readableStream.state === 'cancelled') { - t.fail(); - t.end(); - return; - } else if (readableStream.state === 'errored') { - t.fail(); - t.end(); - return; - } else { - t.fail(readableStream.state); - t.end(); - return; - } - } - } - pump(); -}); - diff --git a/reference-implementation/test/experimental/thin-stream.js b/reference-implementation/test/experimental/thin-stream.js new file mode 100644 index 000000000..89acccd68 --- /dev/null +++ b/reference-implementation/test/experimental/thin-stream.js @@ -0,0 +1,409 @@ +const test = require('tape-catch'); + +import { NewReadableStream } from '../../lib/experimental/new-readable-stream'; +import { selectStreams, pipeStreams } from '../../lib/experimental/new-stream-base'; +import { createStreamQueue } from '../../lib/experimental/stream-queue'; +import { FakeFile } from '../../lib/experimental/fake-file-backed-byte-source'; +import { MockFile } from '../../lib/experimental/mock-byte-sink'; + +test('Stream queue is constructed', t => { + const pair = createStreamQueue({ + size() { + return 1; + }, + shouldApplyBackpressure() { + return false; + } + }); + + t.end(); +}); + +class NoBackpressureStrategy { +} + +class ApplyBackpressureWhenNonEmptyStrategy { + shouldApplyBackpressure(queueSize) { + return queueSize > 0; + } +} + +class AdjustableArrayBufferStrategy { + constructor() { + this._window = 0; + } + + size(ab) { + return ab.byteLength; + } + shouldApplyBackpressure(queueSize) { + return queueSize >= this._window; + } + space(queueSize) { + return Math.max(0, this._window - queueSize); + } + onWindowUpdate(window) { + this._window = window; + } +} + +class AdjustableStringStrategy { + constructor() { + this._window = 0; + } + + size(s) { + return s.length; + } + shouldApplyBackpressure(queueSize) { + return queueSize >= this._window; + } + space(queueSize) { + return Math.max(0, this._window - queueSize); + } + onWindowUpdate(window) { + this._window = window; + } +} + +test('Synchronous write and read', t => { + const pair = createStreamQueue(new ApplyBackpressureWhenNonEmptyStrategy()); + const ws = pair.writable; + const rs = pair.readable; + + t.equals(ws.state, 'writable'); + t.equals(rs.state, 'waiting'); + + ws.write('hello'); + + t.equal(ws.state, 'waiting'); + t.equal(rs.state, 'readable'); + + const v = rs.read(); + t.equal(v, 'hello'); + + t.equal(ws.state, 'writable'); + t.equal(rs.state, 'waiting'); + + t.end(); +}); + +test('Asynchronous read', t => { + const pair = createStreamQueue(new ApplyBackpressureWhenNonEmptyStrategy()); + const ws = pair.writable; + const rs = pair.readable; + + t.equal(rs.state, 'waiting'); + + rs.ready.then(() => { + t.equal(rs.state, 'readable'); + + const v = rs.read(); + t.equal(v, 'hello'); + + t.equal(rs.state, 'waiting'); + + t.end(); + }, e => { + t.fail(e); + t.end(); + }); + + t.equal(ws.state, 'writable'); + + ws.write('hello'); + + t.equal(ws.state, 'waiting'); +}); + +test('Controlling strategy via window setter', t => { + const pair = createStreamQueue(new AdjustableArrayBufferStrategy()); + const ws = pair.writable; + const rs = pair.readable; + + t.equals(ws.state, 'waiting'); + + rs.window = 5; + t.equal(ws.state, 'writable'); + t.equal(ws.space, 5); + + rs.window = 0; + ws.write(new ArrayBuffer(10)); + + t.equal(ws.state, 'waiting'); + t.equal(ws.space, 0); + + rs.window = 10; + t.equal(ws.state, 'waiting'); + t.equal(ws.space, 0); + + rs.window = 15; + t.equal(ws.state, 'writable'); + t.equal(ws.space, 5); + + rs.window = 20; + t.equal(ws.state, 'writable'); + t.equal(ws.space, 10); + + rs.read(); + + t.equal(ws.state, 'writable'); + t.equal(ws.space, 20); + + t.end(); +}); + +test('close()', t => { + const pair = createStreamQueue(new AdjustableArrayBufferStrategy()); + const ws = pair.writable; + const rs = pair.readable; + + ws.write(new ArrayBuffer(10)); + ws.close(); + t.equal(ws.state, 'closed'); + + t.equal(rs.state, 'readable'); + const v0 = rs.read(); + t.equal(rs.state, 'closed'); + + t.end(); +}); + +test('abort()', t => { + const pair = createStreamQueue(new AdjustableStringStrategy()); + const ws = pair.writable; + const rs = pair.readable; + + ws.write('hello'); + ws.write('world'); + + const testError = new TypeError('foo'); + + rs.errored.then(() => { + t.equal(rs.state, 'errored', 'rs.state'); + t.equal(rs.error, testError, 'rs.error'); + + t.end(); + }); + + ws.abort(testError); + t.equal(ws.state, 'aborted', 'ws.state'); +}); + +test('cancel()', t => { + const pair = createStreamQueue(new AdjustableStringStrategy()); + const ws = pair.writable; + const rs = pair.readable; + + ws.write('hello'); + ws.write('world'); + + const testError = new TypeError('foo'); + + rs.cancel(testError); + t.equal(rs.state, 'cancelled', 'rs.state'); + + ws.errored.then(() => { + t.equal(ws.state, 'errored', 'ws.state'); + t.equal(ws.error, testError, 'ws.error'); + + t.end(); + }); +}); + +test('pipeStreams()', t => { + const pair0 = createStreamQueue(new AdjustableStringStrategy()); + const ws0 = pair0.writable; + const rs0 = pair0.readable; + + const pair1 = createStreamQueue(new AdjustableStringStrategy()); + const ws1 = pair1.writable; + const rs1 = pair1.readable; + + t.equal(ws0.state, 'waiting'); + // Check that ws0 becomes writable. + ws0.ready.then(() => { + + console.log('ZZ'); + t.equals(ws0.state, 'writable'); + + ws0.write('hello'); + + // Just write without state check. + ws0.write('world'); + ws0.close(); + }); + + pipeStreams(rs0, ws1) + .catch(e => { + t.fail(e); + t.end(); + }); + + t.equal(rs1.state, 'waiting'); + + rs1.window = 20; + + t.equal(rs1.state, 'waiting'); + + rs1.ready.then(() => { + t.equals(rs1.state, 'readable'); + const v0 = rs1.read(); + t.equal(v0, 'hello'); + + t.equal(rs1.state, 'readable'); + const v1 = rs1.read(); + t.equal(v1, 'world'); + + t.equal(rs1.state, 'closed'); + + t.end(); + }).catch(e => { + t.fail(e); + t.end(); + }); +}); + +test('pipeStreams(): abort() propagation', t => { + t.plan(3); + + const pair0 = createStreamQueue(new AdjustableStringStrategy()); + const ws0 = pair0.writable; + const rs0 = pair0.readable; + + const pair1 = createStreamQueue(new AdjustableStringStrategy()); + const ws1 = pair1.writable; + const rs1 = pair1.readable; + + ws0.write('hello'); + ws0.write('world'); + + const testError = new TypeError('foo'); + + const pipePromise = pipeStreams(rs0, ws1); + pipePromise + .then( + v => t.fail('pipePromise is fulfilled with ' + v), + e => t.equal(e.message, 'source is errored', 'rejection reason of pipePromise')); + + rs1.errored.then(() => { + t.equal(rs1.state, 'errored', 'rs.state'); + t.equal(rs1.error, testError, 'rs1.error'); + }); + + ws0.abort(testError); +}); + +test('pipeStreams(): cancel() propagation', t => { + t.plan(3); + + const pair0 = createStreamQueue(new AdjustableStringStrategy()); + const ws0 = pair0.writable; + const rs0 = pair0.readable; + + const pair1 = createStreamQueue(new AdjustableStringStrategy()); + const ws1 = pair1.writable; + const rs1 = pair1.readable; + + ws0.write('hello'); + ws0.write('world'); + + const testError = new TypeError('foo'); + + const pipePromise = pipeStreams(rs0, ws1); + pipePromise + .then( + v => t.fail('pipePromise is fulfilled with ' + v), + e => t.equal(e.message, 'dest is errored', 'rejection reason of pipePromise')); + + ws0.errored.then(() => { + t.equal(ws0.state, 'errored', 'ws0.state'); + t.equal(ws0.error, testError, 'ws0.error'); + }); + + rs1.cancel(testError); +}); + +test('Piping from a source with a buffer pool to a buffer taking sink', t => { + const file = new FakeFile(1024); + const rs = file.createManualPullStream(); + + let view = new Uint8Array(10); + + let count = 0; + function pump() { + for (;;) { + if (rs.state === 'errored') { + + } else if (rs.state === 'closed') { + t.equal(count, 1024); + t.end(); + return; + } else if (rs.state === 'readable') { + const readView = rs.read(); + count += readView.byteLength; + view = new Uint8Array(readView.buffer); + } else if (rs.state === 'waiting') { + if (view !== undefined && rs.pullable) { + rs.pull(view); + view = undefined; + continue; + } + + const promises = [rs.ready, rs.errored]; + if (!rs.pullable) { + promises.push(rs.pullReady); + } + Promise.race(promises).then(pump); + return; + } + } + } + pump(); +}); + +test('Consuming bytes from a source with a buffer pool via the ReadableOperationStream interface', t => { + const file = new FakeFile(1024); + const rs = file.createStream(new AdjustableArrayBufferStrategy()); + rs.window = 64; + + let count = 0; + function pump() { + for (;;) { + if (rs.state === 'errored') { + console.log('ss' + rs.error); + + } else if (rs.state === 'closed') { + console.log('cl'); + t.equal(count, 1024); + t.end(); + return; + } else if (rs.state === 'readable') { + const readView = rs.read(); + console.log('rd' + readView.byteLength); + count += readView.byteLength; + } else if (rs.state === 'waiting') { + console.log('wt'); + Promise.race([rs.ready, rs.errored]).then(pump); + return; + } else { + console.log('sss'); + } + } + } + pump(); +}); + +test('Piping from a buffer taking source to a sink with buffer', t => { + const file = new MockFile(); + + const ws = file.createStream(); + + ws.write(new Uint8Array(10)); + ws.close(); + + file.result.then(bytesRead => { + t.equal(bytesRead, 10); + t.end(); + }); +}); From b79a7aff04e306f62a74511833812c59487004a0 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Wed, 4 Mar 2015 23:47:16 +0900 Subject: [PATCH 86/93] Rename --- .../fake-file-backed-byte-source.js | 23 +- .../lib/experimental/mock-byte-sink.js | 48 ++--- .../lib/experimental/operation-queue.js | 200 ------------------ .../lib/experimental/operation-stream.js | 166 --------------- .../lib/experimental/stream-queue.js | 9 +- .../experimental/thin-readable-byte-stream.js | 16 +- .../lib/experimental/thin-readable-stream.js | 8 +- .../lib/experimental/thin-stream-base.js | 3 - .../experimental/thin-writable-byte-stream.js | 55 ++--- .../lib/experimental/thin-writable-stream.js | 9 +- reference-implementation/run-tests.js | 2 +- .../test/experimental/thin-stream.js | 24 +-- 12 files changed, 98 insertions(+), 465 deletions(-) delete mode 100644 reference-implementation/lib/experimental/operation-queue.js delete mode 100644 reference-implementation/lib/experimental/operation-stream.js diff --git a/reference-implementation/lib/experimental/fake-file-backed-byte-source.js b/reference-implementation/lib/experimental/fake-file-backed-byte-source.js index 57d5e4dc6..8098478dc 100644 --- a/reference-implementation/lib/experimental/fake-file-backed-byte-source.js +++ b/reference-implementation/lib/experimental/fake-file-backed-byte-source.js @@ -1,7 +1,7 @@ -import { NewReadableStream } from './new-readable-stream'; -import { NewReadableByteStream } from './new-readable-byte-stream'; +import { ThinReadableStream } from './thin-readable-stream'; +import { ThinReadableByteStream } from './thin-readable-byte-stream'; -import { fillArrayBufferView } from './new-stream-utils'; +import { fillArrayBufferView } from './thin-stream-utils'; class FileBackedUnderlyingSource { constructor(readFileInto, strategy) { @@ -29,8 +29,6 @@ class FileBackedUnderlyingSource { this._queue.push(result.writtenRegion); this._queueSize += result.writtenRegion.byteLength; - console.log('queue' + this._queueSize); - this._delegate.markReadable(); if (result.closed) { @@ -43,21 +41,15 @@ class FileBackedUnderlyingSource { } _pull() { - console.log('222'); - if (this._cancelled) { return; } - console.log('223'); - if (this._strategy.shouldApplyBackpressure === undefined || this._strategy.shouldApplyBackpressure(this._queueSize)) { return; } - console.log('224'); - if (this._fileReadPromise !== undefined) { return; } @@ -67,10 +59,8 @@ class FileBackedUnderlyingSource { size = this._strategy.space(this._queueSize); } if (size === 0) { - console.log('full'); return; } - console.log('ssss' + size); const view = new Uint8Array(size); this._fileReadPromise = this._readFileInto(view) @@ -198,7 +188,6 @@ export class FakeFile { this._bytesRemaining -= bytesToWrite; const writtenRegion = view.subarray(0, bytesToWrite); - console.log('zzzz ' + writtenRegion.byteLength); if (this._bytesRemaining === 0) { resolve({closed: true, writtenRegion}); } else { @@ -212,7 +201,8 @@ export class FakeFile { } createStream(strategy) { - return new NewReadableStream(new FileBackedUnderlyingSource(this._readFileInto.bind(this), strategy)); + const source = new FileBackedUnderlyingSource(this._readFileInto.bind(this), strategy); + return new ThinReadableStream(source); } // Returns a manual pull readable stream. @@ -225,6 +215,7 @@ export class FakeFile { // Blocking or async I/O interfaces which takes a buffer on reading function call: // - The stream is always pullable. createManualPullStream() { - return new NewReadableByteStream(new FileBackedManualPullUnderlyingSource(this._readFileInto.bind(this))); + const source = new FileBackedManualPullUnderlyingSource(this._readFileInto.bind(this)); + return new ThinReadableByteStream(source); } } diff --git a/reference-implementation/lib/experimental/mock-byte-sink.js b/reference-implementation/lib/experimental/mock-byte-sink.js index ef1c6fdd8..a7fbd9771 100644 --- a/reference-implementation/lib/experimental/mock-byte-sink.js +++ b/reference-implementation/lib/experimental/mock-byte-sink.js @@ -1,5 +1,5 @@ -import { NewWritableStream } from './new-writable-stream'; -import { NewWritableByteStream } from './new-writable-byte-stream'; +import { ThinWritableStream } from './thin-writable-stream'; +import { ThinWritableByteStream } from './thin-writable-byte-stream'; class MockFileUnderlyingSink { constructor(file) { @@ -8,10 +8,12 @@ class MockFileUnderlyingSink { start(delegate) { this._delegate = delegate; + + this._delegate.markWritable(); } write(value) { - this._file._count(value.byteLength); + this._file._write(value); } close() { @@ -27,18 +29,11 @@ class MockFileUnderlyingSink { } } -class MockFileUnderlyingSinkWithDisposal { +class MockFileUnderlyingSinkWithGarbage { constructor(file) { this._file = file; - this._bytesRead = 0; - - this._resultPromise = new Promise((resolve, reject) => { - this._resolveResultPromise = resolve; - this._rejectResultPromise = reject; - }); - - this._disposedViews = []; + this._garbages = []; } start(delegate) { @@ -48,9 +43,10 @@ class MockFileUnderlyingSinkWithDisposal { } write(view) { - this._file._count(view.byteLength); - this._disposedViews.push(view); - this._delegate.markHasDisposedView(); + this._file._write(view); + + this._garbages.push(view); + this._delegate.markHasGarbage(); } close() { @@ -65,10 +61,10 @@ class MockFileUnderlyingSinkWithDisposal { return 16; } - readDisposedView() { - const view = this._disposedViews.shift(); - if (this._disposedViews.length === 0) { - this._delegate.markNoDisposedView(); + readGarbage() { + const view = this._garbages.shift(); + if (this._garbages.length === 0) { + this._delegate.markNoGarbage(); } } } @@ -87,23 +83,27 @@ export class MockFile { return this._resultPromise; } - _count(size) { - this._bytesRead += size; + _write(view) { + this._bytesRead += view.byteLength; } _close() { this._resolveResultPromise(this._bytesRead); + this._resolveResultPromise = undefined; + this._rejectResultPromise = undefined; } _abort(reason) { this._rejectResultPromise(reason); + this._resolveResultPromise = undefined; + this._rejectResultPromise = undefined; } createStream() { - return new NewWritableStream(new MockFileUnderlyingSink(this)); + return new ThinWritableStream(new MockFileUnderlyingSink(this)); } - createStreamWithDisposal() { - return new NewWritableByteStream(new MockFileUnderlyingSinkWithDisposal(this)); + createStreamWithGarbage() { + return new ThinWritableByteStream(new MockFileUnderlyingSinkWithGarbage(this)); } } diff --git a/reference-implementation/lib/experimental/operation-queue.js b/reference-implementation/lib/experimental/operation-queue.js deleted file mode 100644 index 639d46674..000000000 --- a/reference-implementation/lib/experimental/operation-queue.js +++ /dev/null @@ -1,200 +0,0 @@ -import { Operation, OperationStatus } from './operation-stream'; -import { WritableStream } from './writable-stream'; -import { ReadableStream } from './readable-stream'; - -class OperationQueueShared { - constructor(strategy) { - this._shared = []; - this._sharedSize = 0; - - this._strategy = strategy; - } -} - -class OperationQueueUnderlyingSink { - _updateWritableStream() { - if (this._shared._strategy === undefined) { - return; - } - - let shouldApplyBackpressure = false; - if (this._shared._strategy.shouldApplyBackpressure !== undefined) { - shouldApplyBackpressure = this._shared._strategy.shouldApplyBackpressure(this._shared._queueSize); - } - if (shouldApplyBackpressure) { - this._delegate.markWaiting(); - } else { - this._delegate.markWritable(); - } - - this._delegate.onSpaceChange(); - } - - constructor(queue) { - this._shared = queue; - } - - setSource(source) { - this._source = source; - } - - init(delegate) { - this._delegate = delegate; - - this._updateWritableStream(); - } - - get space() { - if (this._shared._strategy.space !== undefined) { - return this._shared._strategy.space(this._shared._queueSize); - } - - return undefined; - } - - write(value) { - const operationStatus = new OperationStatus(); - const operation = new Operation('data', value, operationStatus); - - var size = 1; - if (this._shared._strategy.size !== undefined) { - size = this._shared._strategy.size(operation.argument); - } - - this._shared._queue.push({value: operation, size}); - this._shared._queueSize += size; - - this._updateWritableStream(); - - this._source.onQueueFill(); - - return operationStatus; - } - - close() { - const operationStatus = new OperationStatus(); - const operation = new Operation('close', ReadableStream.EOS, operationStatus); - - this._shared._queue.push({value: operation, size: 0}); - - // No longer necessary. - this._shared._strategy = undefined; - - this._source.onQueueFill(); - - return operationStatus; - } - - abort(reason) { - const operationStatus = new OperationStatus(); - const operation = new Operation('abort', reason, operationStatus); - - for (var i = this._shared._queue.length - 1; i >= 0; --i) { - const op = this._shared._queue[i].value; - op.error(new TypeError('aborted')); - } - this._shared._queue = []; - - this._shared._strategy = undefined; - - this._source.abort(operation); - - return operationStatus; - } - - onWindowUpdate() { - this._updateWritableStream(); - } - - onQueueConsume() { - this._updateWritableStream(); - } - - onCancel(reason) { - this._delegate.markErrored(reason); - } -} - -class OperationQueueUnderlyingSource { - constructor(shared) { - this._shared = shared; - } - - setSink(sink) { - this._sink = sink; - } - - init(delegate) { - this._delegate = delegate; - } - - onQueueFill() { - this._delegate.markReadable(); - } - - abort(reason) { - this._delegate.markErrored(reason); - } - - onWindowUpdate(v) { - if (this._shared._strategy === undefined) { - return; - } - - if (this._shared._strategy.onWindowUpdate !== undefined) { - this._shared._strategy.onWindowUpdate(v); - } - - this._sink.onWindowUpdate(); - } - - read() { - if (this._shared._queue.length === 0) { - throw new TypeError('not readable'); - } - - const entry = this._shared._queue.shift(); - this._shared._queueSize -= entry.size; - - if (this._shared._queue.length === 0) { - if (entry.value.type === 'close') { - this._delegate.markDrained(); - } else { - this._delegate.markWaiting(); - } - } - - this._sink.onQueueConsume(); - - return entry.value; - } - - cancel(reason) { - const operationStatus = new OperationStatus(); - const operation = new Operation('cancel', reason, operationStatus); - - for (var i = 0; i < this._shared._queue.length; ++i) { - const op = this._shared._queue[i].value; - op.error(operation.argument); - } - this._shared._queue = []; - - this._shared._strategy = undefined; - - this._sink.onCancel(operation); - - return operationStatus; - } -} - - -// Creates a pair of WritableStream implementation and ReadableStream implementation that are -// connected with a queue. This can be used for creating queue-backed operation streams. -export function createOperationQueue(strategy) { - const queue = new OperationQueueShared(strategy); - const source = new OperationQueueUnderlyingSource(queue); - const sink = new OperationQueueUnderlyingSink(queue); - sink.setSource(source); - source.setSink(sink); - return { writable: new WritableStream(sink), readable: new ReadableStream(source) }; -} diff --git a/reference-implementation/lib/experimental/operation-stream.js b/reference-implementation/lib/experimental/operation-stream.js deleted file mode 100644 index bad0e00de..000000000 --- a/reference-implementation/lib/experimental/operation-stream.js +++ /dev/null @@ -1,166 +0,0 @@ -import { selectStreams, writableAcceptsWriteAndClose, writableAcceptsAbort, readableAcceptsCancel - } from './stream-base'; - -export function jointOps(op, status) { - function forward() { - if (status.state === 'waiting') { - status.ready.then(forward); - } else if (status.state === 'errored') { - op.error(status.result); - } else if (status.state === 'completed') { - op.complete(status.result); - } - } - forward(); -} - -// Pipes data from source to dest with no transformation. Abort signal, cancel signal and space are also propagated -// between source and dest. -export function pipeOperationStreams(source, dest) { - return new Promise((resolve, reject) => { - const oldWindow = source.window; - - function disposeStreams(error) { - if (writableAcceptsAbort(dest.state)) { - dest.abort(error); - } - if (readableAcceptsCancel(source.state)) { - source.abort(error); - } - reject(error); - } - - function loop() { - for (;;) { - // Handle interrupt. - if (source.state === 'drained') { - reject(new TypeError('source is drained')); - return; - } - if (source.state === 'cancelled') { - reject(new TypeError('source is cancelled')); - return; - } - - if (dest.state === 'closed') { - reject(new TypeError('dest is closed')); - return; - } - if (dest.state === 'aborted') { - reject(new TypeError('dest is aborted')); - return; - } - - // Propagate errors. - - if (source.state === 'errored') { - if (writableAcceptsAbort(dest.state)) { - if (source.error.constructor === Operation) { - jointOps(source.error, dest.abort(source.error.argument)); - } else { - dest.abort(source.error); - } - } - reject(new TypeError('source is errored')); - return; - } - - if (dest.state === 'errored') { - if (readableAcceptsCancel(source.state)) { - if (dest.error.constructor === Operation) { - jointOps(dest.error, source.cancel(dest.error.argument)); - } else { - source.cancel(dest.error); - } - } - reject(new TypeError('dest is errored')); - return; - } - - if (dest.state === 'writable') { - if (source.state === 'readable') { - const op = source.read(); - if (op.type === 'data') { - jointOps(op, dest.write(op.argument)); - } else if (op.type === 'close') { - jointOps(op, dest.close()); - - resolve(); - - return; - } else { - const error = new TypeError('unexpected operation type: ' + op.type); - disposeStreams(error); - return; - } - - continue; - } else { - source.window = dest.space; - } - } - - selectStreams(source, dest) - .then(loop) - .catch(disposeStreams); - return; - } - } - loop(); - }); -} - -export class OperationStatus { - constructor() { - this._state = 'waiting'; - this._result = undefined; - this._readyPromise = new Promise((resolve, reject) => { - this._resolveReadyPromise = resolve; - }); - } - - _onCompletion(v) { - this._state = 'completed'; - this._result = v; - this._resolveReadyPromise(); - this._resolveReadyPromise = undefined; - } - _onError(e) { - this._state = 'errored'; - this._result = e; - this._resolveReadyPromise(); - this._resolveReadyPromise = undefined; - } - - get state() { - return this._state; - } - get result() { - return this._result; - } - get ready() { - return this._readyPromise; - } -} - -export class Operation { - constructor(type, argument, status) { - this._type = type; - this._argument = argument; - this._status = status; - } - - get type() { - return this._type; - } - get argument() { - return this._argument; - } - - complete(result) { - this._status._onCompletion(result); - } - error(reason) { - this._status._onError(reason); - } -} diff --git a/reference-implementation/lib/experimental/stream-queue.js b/reference-implementation/lib/experimental/stream-queue.js index 9d9cd2714..daa3e8302 100644 --- a/reference-implementation/lib/experimental/stream-queue.js +++ b/reference-implementation/lib/experimental/stream-queue.js @@ -1,5 +1,5 @@ -import { NewWritableStream } from './new-writable-stream'; -import { NewReadableStream } from './new-readable-stream'; +import { ThinWritableStream } from './thin-writable-stream'; +import { ThinReadableStream } from './thin-readable-stream'; class StreamQueueShared { constructor(strategy) { @@ -55,7 +55,6 @@ class StreamQueueUnderlyingSink { } write(value) { - console.log('fjdiojfdosi'); var size = 1; if (this._shared._strategy.size !== undefined) { size = this._shared._strategy.size(value); @@ -114,8 +113,6 @@ class StreamQueueUnderlyingSource { } onQueueFill() { - console.log('fjfljlkfj'); - this._delegate.markReadable(); } @@ -175,5 +172,5 @@ export function createStreamQueue(strategy) { const sink = new StreamQueueUnderlyingSink(shared); sink.setSource(source); source.setSink(sink); - return { writable: new NewWritableStream(sink), readable: new NewReadableStream(source) }; + return { writable: new ThinWritableStream(sink), readable: new ThinReadableStream(source) }; } diff --git a/reference-implementation/lib/experimental/thin-readable-byte-stream.js b/reference-implementation/lib/experimental/thin-readable-byte-stream.js index eb65e7463..01d956e75 100644 --- a/reference-implementation/lib/experimental/thin-readable-byte-stream.js +++ b/reference-implementation/lib/experimental/thin-readable-byte-stream.js @@ -1,6 +1,6 @@ -import { readableAcceptsCancel } from './stream-base'; +import { readableAcceptsCancel } from './thin-stream-base'; -export class NewReadableByteStream { +export class ThinReadableByteStream { _initReadyPromise() { this._readyPromise = new Promise((resolve, reject) => { this._resolveReadyPromise = resolve; @@ -61,7 +61,7 @@ export class NewReadableByteStream { throw new TypeError('not pullable'); } - return this._source.pull(view); + this._source.pull(view); } // Reading interfaces. @@ -124,6 +124,10 @@ export class NewReadableByteStream { } _markWaiting() { + if (this._state === 'waiting') { + return; + } + this._initReadyPromise(); this._state = 'waiting'; } @@ -134,15 +138,21 @@ export class NewReadableByteStream { } this._resolveReadyPromise(); + this._resolveReadyPromise = undefined; this._state = 'readable'; } _markClosed() { + if (this._state !== 'readable') { + this._resolveReadyPromise(); + this._resolveReadyPromise = undefined; + } this._state = 'closed'; } _markErrored(error) { this._resolveErroredPromise(); + this._resolveErroredPromise = undefined; this._state = 'errored'; this._error = error; } diff --git a/reference-implementation/lib/experimental/thin-readable-stream.js b/reference-implementation/lib/experimental/thin-readable-stream.js index 0dc9008d5..2f6a6b188 100644 --- a/reference-implementation/lib/experimental/thin-readable-stream.js +++ b/reference-implementation/lib/experimental/thin-readable-stream.js @@ -1,6 +1,6 @@ -import { readableAcceptsCancel } from './new-stream-base'; +import { readableAcceptsCancel } from './thin-stream-base'; -export class NewReadableStream { +export class ThinReadableStream { _initReadyPromise() { this._readyPromise = new Promise((resolve, reject) => { this._resolveReadyPromise = resolve; @@ -93,6 +93,10 @@ export class NewReadableStream { // Methods exposed only to the underlying source. _markWaiting() { + if (this._state === 'waiting') { + return; + } + this._initReadyPromise(); this._state = 'waiting'; } diff --git a/reference-implementation/lib/experimental/thin-stream-base.js b/reference-implementation/lib/experimental/thin-stream-base.js index 5a3eccb4c..1d857c057 100644 --- a/reference-implementation/lib/experimental/thin-stream-base.js +++ b/reference-implementation/lib/experimental/thin-stream-base.js @@ -69,11 +69,8 @@ export function pipeStreams(source, dest) { return; } - console.log('sjsjsjs' + source.state + dest.state); - if (source.state === 'readable') { if (dest.state === 'writable') { - console.log('sjsjsjs'); dest.write(source.read()); continue; } diff --git a/reference-implementation/lib/experimental/thin-writable-byte-stream.js b/reference-implementation/lib/experimental/thin-writable-byte-stream.js index 29726123d..cd79b76ff 100644 --- a/reference-implementation/lib/experimental/thin-writable-byte-stream.js +++ b/reference-implementation/lib/experimental/thin-writable-byte-stream.js @@ -1,15 +1,15 @@ -import { writableAcceptsWriteAndClose, writableAcceptsAbort } from './new-stream-base'; +import { writableAcceptsWriteAndClose, writableAcceptsAbort } from './thin-stream-base'; -export class NewWritableByteStream { +export class ThinWritableByteStream { _initReadyPromise() { this._readyPromise = new Promise((resolve, reject) => { this._resolveReadyPromise = resolve; }); } - _initDisposedViewReadyPromise() { - this._disposedViewReadyPromise = new Promise((resolve, reject) => { - this._resolveDisposedViewReadyPromise = resolve; + _initGarbageReadyPromise() { + this._garbageReadyPromise = new Promise((resolve, reject) => { + this._resolveGarbageReadyPromise = resolve; }); } @@ -27,17 +27,18 @@ export class NewWritableByteStream { this._lastSpace = undefined; this._spaceChangePromise = undefined; - this._initDisposedViewReadyPromise(); - this._hasDisposedView = false; + this._hasGarbage = false; + this._initGarbageReadyPromise(); const delegate = { markWaiting: this._markWaiting.bind(this), markWritable: this._markWritable.bind(this), - markErrored: this._markErrored.bind(this), onSpaceChange: this._onSpaceChange.bind(this), - markHasDisposedView: this._markHasDisposedView.bind(this), - markNoDisposedView: this._markNoDisposedView.bind(this), + markErrored: this._markErrored.bind(this), + + markHasGarbage: this._markHasGarbage.bind(this), + markNoGarbage: this._markNoGarbage.bind(this) }; this._sink.start(delegate); @@ -124,20 +125,20 @@ export class NewWritableByteStream { // Disposed buffer reading interfaces. - get hasDisposedView() { - return this._hasDisposedView; + get hasGarbage() { + return this._hasGarbage; } - get disposedViewReady() { - return this._disposedViewReady; + get garbageReady() { + return this._garbageReadyPromise; } - get readDisposedView() { - if (this._disposedViews.length === 0) { - throw new TypeError('no disposed view'); + get readGarbage() { + if (!this._hasGarbage) { + throw new TypeError('no garbage'); } - return this._sink.readDisposedView(); + return this._sink.readGarbage(); } // Methods exposed only to the underlying sink. @@ -180,22 +181,22 @@ export class NewWritableByteStream { this._spaceChangePromise = undefined; } - _markNoDisposedView() { - if (!this._hasDisposedView) { + _markNoGarbage() { + if (!this._hasGarbage) { return; } - this._initDisposedViewReadyPromise(); - this._hasDisposedView = false; + this._initGarbageReadyPromise(); + this._hasGarbage = false; } - _markHasDisposedView() { - if (this._hasDisposedView) { + _markHasGarbage() { + if (this._hasGarbage) { return; } - this._resolveDisposedViewReadyPromise(); - this._resolveDisposedViewReadyPromise = undefined; - this._hasDisposedView = true; + this._resolveGarbageReadyPromise(); + this._resolveGarbageReadyPromise = undefined; + this._hasGarbage = true; } } diff --git a/reference-implementation/lib/experimental/thin-writable-stream.js b/reference-implementation/lib/experimental/thin-writable-stream.js index 7ac3b3c6f..99badb86c 100644 --- a/reference-implementation/lib/experimental/thin-writable-stream.js +++ b/reference-implementation/lib/experimental/thin-writable-stream.js @@ -1,6 +1,6 @@ -import { writableAcceptsWriteAndClose, writableAcceptsAbort } from './new-stream-base'; +import { writableAcceptsWriteAndClose, writableAcceptsAbort } from './thin-stream-base'; -export class NewWritableStream { +export class ThinWritableStream { _initReadyPromise() { this._readyPromise = new Promise((resolve, reject) => { this._resolveReadyPromise = resolve; @@ -24,8 +24,9 @@ export class NewWritableStream { const delegate = { markWaiting: this._markWaiting.bind(this), markWritable: this._markWritable.bind(this), - markErrored: this._markErrored.bind(this), - onSpaceChange: this._onSpaceChange.bind(this) + onSpaceChange: this._onSpaceChange.bind(this), + + markErrored: this._markErrored.bind(this) }; this._sink.start(delegate); diff --git a/reference-implementation/run-tests.js b/reference-implementation/run-tests.js index ff59bc59f..f61d2cc78 100644 --- a/reference-implementation/run-tests.js +++ b/reference-implementation/run-tests.js @@ -19,4 +19,4 @@ global.TransformStream = TransformStream; //const tests = glob.sync(path.resolve(__dirname, 'test/*.js')); //const experimentalTests = glob.sync(path.resolve(__dirname, 'test/experimental/*.js')); //tests.concat(experimentalTests).forEach(require); -glob.sync(path.resolve(__dirname, 'test/experimental/new-stream.js')).forEach(require); +glob.sync(path.resolve(__dirname, 'test/experimental/thin-stream.js')).forEach(require); diff --git a/reference-implementation/test/experimental/thin-stream.js b/reference-implementation/test/experimental/thin-stream.js index 89acccd68..652a69766 100644 --- a/reference-implementation/test/experimental/thin-stream.js +++ b/reference-implementation/test/experimental/thin-stream.js @@ -1,7 +1,7 @@ const test = require('tape-catch'); -import { NewReadableStream } from '../../lib/experimental/new-readable-stream'; -import { selectStreams, pipeStreams } from '../../lib/experimental/new-stream-base'; +import { ThinReadableStream } from '../../lib/experimental/thin-readable-stream'; +import { selectStreams, pipeStreams } from '../../lib/experimental/thin-stream-base'; import { createStreamQueue } from '../../lib/experimental/stream-queue'; import { FakeFile } from '../../lib/experimental/fake-file-backed-byte-source'; import { MockFile } from '../../lib/experimental/mock-byte-sink'; @@ -223,8 +223,6 @@ test('pipeStreams()', t => { t.equal(ws0.state, 'waiting'); // Check that ws0 becomes writable. ws0.ready.then(() => { - - console.log('ZZ'); t.equals(ws0.state, 'writable'); ws0.write('hello'); @@ -324,7 +322,7 @@ test('pipeStreams(): cancel() propagation', t => { rs1.cancel(testError); }); -test('Piping from a source with a buffer pool to a buffer taking sink', t => { +test('Reading from a file backed byte source using manual pull readable stream', t => { const file = new FakeFile(1024); const rs = file.createManualPullStream(); @@ -362,7 +360,7 @@ test('Piping from a source with a buffer pool to a buffer taking sink', t => { pump(); }); -test('Consuming bytes from a source with a buffer pool via the ReadableOperationStream interface', t => { +test('Reading from a file backed byte source using auto pull readable stream', t => { const file = new FakeFile(1024); const rs = file.createStream(new AdjustableArrayBufferStrategy()); rs.window = 64; @@ -371,30 +369,30 @@ test('Consuming bytes from a source with a buffer pool via the ReadableOperation function pump() { for (;;) { if (rs.state === 'errored') { - console.log('ss' + rs.error); - + t.fail('rs is errored'); + t.end(); + return; } else if (rs.state === 'closed') { - console.log('cl'); t.equal(count, 1024); t.end(); return; } else if (rs.state === 'readable') { const readView = rs.read(); - console.log('rd' + readView.byteLength); count += readView.byteLength; } else if (rs.state === 'waiting') { - console.log('wt'); Promise.race([rs.ready, rs.errored]).then(pump); return; } else { - console.log('sss'); + t.fail('rs.state is invalid: ' + rs.state); + t.end(); + return; } } } pump(); }); -test('Piping from a buffer taking source to a sink with buffer', t => { +test('Writing to a mock byte sink using auto disposing writable stream', t => { const file = new MockFile(); const ws = file.createStream(); From 6144e4774f44cfc6b8f49d2e1b57765a67deb4b1 Mon Sep 17 00:00:00 2001 From: tyoshino Date: Thu, 5 Mar 2015 01:40:32 +0900 Subject: [PATCH 87/93] Mark work in progress files with a comment WIP WIP ... --- reference-implementation/lib/experimental/byte-stream-queue.js | 3 +++ .../lib/experimental/fake-unstoppable-byte-source.js | 2 ++ reference-implementation/lib/experimental/stream-base.js | 0 reference-implementation/test/experimental/operation-stream.js | 2 ++ 4 files changed, 7 insertions(+) delete mode 100644 reference-implementation/lib/experimental/stream-base.js diff --git a/reference-implementation/lib/experimental/byte-stream-queue.js b/reference-implementation/lib/experimental/byte-stream-queue.js index 6a48ffa84..a7d59696e 100644 --- a/reference-implementation/lib/experimental/byte-stream-queue.js +++ b/reference-implementation/lib/experimental/byte-stream-queue.js @@ -1,3 +1,6 @@ +/// WIP WIP WIP WIP + + pull(container) { if (container === undefined) { throw new TypeError('container is undefined'); diff --git a/reference-implementation/lib/experimental/fake-unstoppable-byte-source.js b/reference-implementation/lib/experimental/fake-unstoppable-byte-source.js index 946aa6665..8f70b4ff3 100644 --- a/reference-implementation/lib/experimental/fake-unstoppable-byte-source.js +++ b/reference-implementation/lib/experimental/fake-unstoppable-byte-source.js @@ -1,3 +1,5 @@ +// WIP WIP WIP WIP + class FakeUnstoppablePushSource { constructor(count) { this._count = count; diff --git a/reference-implementation/lib/experimental/stream-base.js b/reference-implementation/lib/experimental/stream-base.js deleted file mode 100644 index e69de29bb..000000000 diff --git a/reference-implementation/test/experimental/operation-stream.js b/reference-implementation/test/experimental/operation-stream.js index 0f5d787d7..8badef1ae 100644 --- a/reference-implementation/test/experimental/operation-stream.js +++ b/reference-implementation/test/experimental/operation-stream.js @@ -1,3 +1,5 @@ +// WIP WIP WIP WIP + test('Transformation example: Byte counting', t => { function byteCountingTransform(source, dest) { return new Promise((resolve, reject) => { From 2ea1755a1147d185cd2c10dc5fa242073fd6077a Mon Sep 17 00:00:00 2001 From: tyoshino Date: Thu, 5 Mar 2015 01:54:54 +0900 Subject: [PATCH 88/93] Update ThinStream.md --- ThinStream.md | 126 ++++++++++++++++++-------------------------------- 1 file changed, 45 insertions(+), 81 deletions(-) diff --git a/ThinStream.md b/ThinStream.md index 83e06e932..8262371c1 100644 --- a/ThinStream.md +++ b/ThinStream.md @@ -1,27 +1,29 @@ ```es6 class UnderlyingSink { // delegate contains: - // - markWaiting(): Tells the stream that backpressure is applied. - // - markWritable(): Tells the stream that backpressure is not applied. - // - markErrored(error): Tells the stream that the sink is errored. - // - onSpaceChange(): Tells the stream that get space() may return something new. - init(delegate) - - // May return something. If backpressure state is changed, should call appropriate - // delegate function to update the stream. + // - Back pressure signaling + // - markWaiting(): Tells the stream that backpressure is applied. + // - markWritable(): Tells the stream that backpressure is not applied. + // - onSpaceChange(): Tells the stream that get space() may return something new. + // - Error signaling + // - markErrored(error): Tells the stream that the sink is errored. + start(delegate) + + // Takes value and do some processing. May return something. If backpressure state is changed, + // should call appropriate delegate function to update the stream. write(value) - // May return something. + // Do something to finalize the sink. No more write() or close() comes. May return something. close() - // May return something. - abort() + // Takes reason and do something to abort the sink. May return something. + abort(reason) + // Should returns a number indicating how much data can be written without causing backpressure. + // Maye return undefined if it's unknown. get space() } -// Once a writer is created, promises obtained from this stream are not fulfilled until the -// writer is released. -class WritableStream { - // At the end of construction, calls init() on underlyingSink with delegate functions for this +class ThinWritableStream { + // At the end of construction, calls start() on underlyingSink with delegate functions for this // stream. constructor(underlyingSink) @@ -30,14 +32,12 @@ class WritableStream { // X is state name or a name of a group of states. // A, B, C, ... are the states X may transit to. // - // - "locked" -> available - // - available -> "locked" - // - (write()/close() are allowed) -> "closed", "aborted", "errored" - // - "waiting" (backpressure not applied) -> "writable" - // - "writable" (backpressure applied) -> "waiting" - // - "closed" -> "aborted", "errored" - // - "aborted" - // - "errored" + // - (write()/close() are allowed) -> "closed", "aborted", "errored" + // - "waiting" (backpressure not applied) -> "writable" + // - "writable" (backpressure applied) -> "waiting" + // - "closed" -> "aborted", "errored" + // - "aborted" + // - "errored" // // Distinction between "waiting" and "writable" is a part of the flow control interfaces. get state() @@ -69,7 +69,7 @@ class WritableStream { // Flow control interfaces. // Returns a promise which gets fulfilled when this instance enters "writable" state. - get writable() + get ready() // Returns the space available for write. // // Available in "waiting" and "writable" state. @@ -79,74 +79,42 @@ class WritableStream { // // Available in "waiting" and "writable" state. waitSpaceChange() - - // Locking interfaces. - - // Creates and returns an ExclusiveStreamWriter instance. Once a writer is created, all methods - // and accessors on this stream throws until the writer is released. - // - // Available in all states if there is no existing active writer. - getWriter() -} - -// Once release() is called, all methods and accessors on this writer throw. -class ExclusiveStreamWriter { - // - "locked" -> available - // - available -> "locked" - // - (write()/close() are allowed) -> "closed", "aborted", "errored" - // - "waiting" (backpressure not applied) -> "writable" - // - "writable" (backpressure applied) -> "waiting" - // - "closed -> "aborted", "errored" - // - "aborted" - // - "errored" - get state() - - get writable() - get space() - waitSpaceChange() - - get errored() - get error() - - write(argument) - close() - abort(reason) - - release() } class UnderlyingSource { // delegate contains: - // - markWaiting(): Tells the stream that nothing is ready for synchronous reading. - // - markReadable(): Tells the stream that something is ready for synchronous reading. - // - markDrained(): Tells the stream that no more data will become available for read. - // - markErrored(error): Tells the stream that the source is errored. - init(delegate) - - // Returns something. If data availability is changed, should call appropriate + // - Readability signaling + // - markWaiting(): Tells the stream that nothing is ready for synchronous reading. + // - markReadable(): Tells the stream that something is ready for synchronous reading. + // - markClosed(): Tells the stream that no more data will become available for read. + // - Error signaling + // - markErrored(error): Tells the stream that the source is errored. + start(delegate) + + // Returns something generated. If data availability is changed, should call appropriate // delegate function to update the stream. read() - // May return something. + // Do something to cancel the source. May return something. cancel(reason) + // Should interpret the given number v to update how much data to generate / pull and buffer + // in the source. get onWindowUpdate(v) } -class ReadableStream { +class ThinReadableStream { constructor(underlyingSource) - // - "locked" -> available - // - available -> "locked" - // - normal -> "cancelled", "errored" - // - "waiting" -> "readable" - // - "readable" -> "waiting", "drained" - // - "drained" - // - "cancelled" - // - "errored" + // - normal -> "cancelled", "errored", "closed" + // - "waiting" -> "readable" + // - "readable" -> "waiting" + // - "closed" + // - "cancelled" + // - "errored" get state() - // Returns a promise which gets fulfilled when this instance enters "readable" state. - get readable() + // Returns a promise which gets fulfilled when this instance enters "readable" or "closed" state. + get ready() // Returns something returned by the underlying source. // // Available in "readable" state. @@ -167,9 +135,5 @@ class ReadableStream { // // Available in "waiting" and "readable" state. set window(v) - - // Locking interfaces. - - getReader() } ``` From 52ec38f8a8abf0711f499d2d1faf426afd99f404 Mon Sep 17 00:00:00 2001 From: tyoshino Date: Sun, 8 Mar 2015 23:07:12 +0900 Subject: [PATCH 89/93] Add ThinAsyncReadableByteStream --- .../thin-async-readable-byte-stream.js | 154 +++++++++++++ reference-implementation/run-tests.js | 2 +- .../thin-async-readable-byte-stream.js | 210 ++++++++++++++++++ 3 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 reference-implementation/lib/experimental/thin-async-readable-byte-stream.js create mode 100644 reference-implementation/test/experimental/thin-async-readable-byte-stream.js diff --git a/reference-implementation/lib/experimental/thin-async-readable-byte-stream.js b/reference-implementation/lib/experimental/thin-async-readable-byte-stream.js new file mode 100644 index 000000000..ecff15676 --- /dev/null +++ b/reference-implementation/lib/experimental/thin-async-readable-byte-stream.js @@ -0,0 +1,154 @@ +export class ThinAsyncReadableByteStream { + constructor(source) { + this._source = source; + + this._pendingReadRequests = []; + this._cancelled = false; + + this._errored = false; + this._errorReason = undefined; + this._closed = false; + + this._closedPromise = new Promise((resolve, reject) => { + this._resolveClosedPromise = resolve; + this._rejectClosedPromise = reject; + }); + + const delegate = { + fulfill: this._fulfill.bind(this), + close: this._close.bind(this), + error: this._error.bind(this) + }; + + this._source.start(delegate); + } + + get closed() { + return this._closedPromise; + } + + read(container) { + // TODO: Detach container + + if (this._cancelled) { + return Promise.reject({value: new TypeError('already cancelled'), container}); + } + + if (this._errored) { + return Promise.reject({value: this._errorReason, container}); + } + if (this._closed) { + return Promise.resolve({done: true, container}); + } + + return new Promise((resolve, reject) => { + this._pendingReadRequests.push({resolve, reject, container}); + this._source.read(container); + }); + } + + cancel(reason) { + if (this._cancelled) { + return Promise.reject(new TypeError('already cancelled')); + } + + if (this._errored) { + return Promise.reject(this._errorReason); + } + if (this._closed) { + return Promise.reject(new TypeError('already closed')); + } + + this._cancelled = true; + + while (this._pendingReadRequests.length !== 0) { + const request = this._pendingReadRequests.shift(); + request.reject({value: new TypeError('cancelled'), container: request.container}); + } + + this._resolveClosedPromise(); + this._resolveClosedPromise = undefined; + this._rejectClosedPromise = undefined; + + return Promise.resolve(this._source.cancel(reason)); + } + + release() { + if (this._pendingReadRequests.length !== 0) { + throw new TypeError('there are pending reads'); + } + + this._source = undefined; + } + + _fulfill(value) { + // TODO: Detach value + + if (this._source === undefined) { + throw new TypeError('already released'); + } + + if (this._errored) { + throw new TypeError('already errored'); + } + if (this._closed) { + throw new TypeError('already closed'); + } + + if (this._pendingReadRequests.length === 0) { + throw new TypeError('no pending read request'); + } + + const request = this._pendingReadRequests.shift(); + request.resolve({done: false, value}); + } + + _close() { + if (this._source === undefined) { + throw new TypeError('already released'); + } + + if (this._errored) { + throw new TypeError('already errored'); + } + if (this._closed) { + throw new TypeError('already closed'); + } + + this._closed = true; + + while (this._pendingReadRequests.length !== 0) { + const request = this._pendingReadRequests.shift(); + request.resolve({done: true, container: request.container}); + } + + this._resolveClosedPromise(); + this._resolveClosedPromise = undefined; + this._rejectClosedPromise = undefined; + } + + _error(reason) { + if (this._source === undefined) { + throw new TypeError('already released'); + } + + if (this._errored) { + throw new TypeError('already errored'); + } + if (this._closed) { + throw new TypeError('already closed'); + } + + this._errored = true; + this._errorReason = reason; + + while (this._pendingReadRequests.length !== 0) { + const request = this._pendingReadRequests.shift(); + request.reject({value: reason, container: request.container}); + } + + this._rejectClosedPromise(reason); + this._resolveClosedPromise = undefined; + this._rejectClosedPromise = undefined; + } +} diff --git a/reference-implementation/run-tests.js b/reference-implementation/run-tests.js index f61d2cc78..2489ed50e 100644 --- a/reference-implementation/run-tests.js +++ b/reference-implementation/run-tests.js @@ -19,4 +19,4 @@ global.TransformStream = TransformStream; //const tests = glob.sync(path.resolve(__dirname, 'test/*.js')); //const experimentalTests = glob.sync(path.resolve(__dirname, 'test/experimental/*.js')); //tests.concat(experimentalTests).forEach(require); -glob.sync(path.resolve(__dirname, 'test/experimental/thin-stream.js')).forEach(require); +glob.sync(path.resolve(__dirname, 'test/experimental/thin-*.js')).forEach(require); diff --git a/reference-implementation/test/experimental/thin-async-readable-byte-stream.js b/reference-implementation/test/experimental/thin-async-readable-byte-stream.js new file mode 100644 index 000000000..539f51225 --- /dev/null +++ b/reference-implementation/test/experimental/thin-async-readable-byte-stream.js @@ -0,0 +1,210 @@ +const test = require('tape-catch'); + +import { fillArrayBufferView } from '../../lib/experimental/thin-stream-utils'; +import { ThinAsyncReadableByteStream } from '../../lib/experimental/thin-async-readable-byte-stream'; + +test('Construct a ThinAsyncReadableByteStream', t => { + class Source { + constructor(size) { + this._bytesRemaining = size; + } + + start(delegate) { + this._delegate = delegate; + } + + read(container) { + if (this._bytesRemaining === 0) { + this._delegate.close(); + return; + } + + const bytesToFill = Math.min(this._bytesRemaining, container.byteLength); + fillArrayBufferView(container, 1, bytesToFill); + this._bytesRemaining -= bytesToFill; + this._delegate.fulfill(container.subarray(0, bytesToFill)); + } + + cancel(reason) { + t.fail('cancel() is called'); + t.end(); + } + } + + let bytesRead = 0; + + const buffer = new Uint8Array(16); + + const rs = new ThinAsyncReadableByteStream(new Source(1024)); + function pump() { + fillArrayBufferView(buffer, 0); + rs.read(buffer).then(result => { + if (result.done) { + t.equal(bytesRead, 1024); + t.end(); + } else { + const view = result.value; + for (let i = 0; i < view.byteLength; ++i) { + if (view[i] !== 1) { + t.fail('view[' + i + '] is ' + view[i]); + t.end(); + return; + } + } + bytesRead += view.byteLength; + pump(); + } + }, reason => { + t.fail(reason); + t.end(); + }).catch(e => { + t.fail(e); + t.end(); + }); + } + pump(); +}); + +test('read() on a closed stream', t => { + class Source { + start(delegate) { + delegate.close(); + } + + read() { + t.fail('read() is called'); + t.end(); + } + + cancel(reason) { + t.fail('cancel() is called'); + t.end(); + } + } + + const rs = new ThinAsyncReadableByteStream(new Source()); + + rs.read(new Uint8Array(16)).then(result => { + t.equal(result.done, true); + t.equal(result.container.byteLength, 16); + + rs.read(new Uint8Array(16)).then(result => { + t.equal(result.done, true); + t.equal(result.container.byteLength, 16); + t.end(); + }); + }); +}); + +test('Close a stream with pending read()s', t => { + let readCount = 0; + + let delegate = undefined; + class Source { + start(delegate_) { + delegate = delegate_; + } + + read(container) { + t.equal(container.byteLength, 16); + ++readCount; + } + + cancel(reason) { + t.fail('cancel() is called'); + t.end(); + } + } + + const rs = new ThinAsyncReadableByteStream(new Source()); + + let firstReadFulfilled = false; + rs.read(new Uint8Array(16)).then(result => { + t.equal(result.done, true); + t.equal(result.container.byteLength, 16); + firstReadFulfilled = true; + }); + rs.read(new Uint8Array(16)).then(result => { + t.equal(result.done, true); + t.equal(result.container.byteLength, 16); + t.equal(firstReadFulfilled, true); + t.equal(readCount, 2); + t.end(); + }); + + delegate.close(); +}); + +test('read() on a errored stream', t => { + const testError = new TypeError('test'); + + class Source { + start(delegate) { + delegate.error(testError); + } + + read() { + t.fail('read() is called'); + t.end(); + } + + cancel(reason) { + t.fail('cancel() is called'); + t.end(); + } + } + + const rs = new ThinAsyncReadableByteStream(new Source()); + + rs.read(new Uint8Array(16)).catch(e => { + t.equal(e.value, testError); + t.equal(e.container.byteLength, 16); + + rs.read(new Uint8Array(16)).catch(e => { + t.equal(e.value, testError); + t.equal(e.container.byteLength, 16); + t.end(); + }); + }); +}); + +test('Error a stream with pending read()s', t => { + let readCount = 0; + + let delegate = undefined; + class Source { + start(delegate_) { + delegate = delegate_; + } + + read(container) { + t.equal(container.byteLength, 16); + ++readCount; + } + + cancel(reason) { + t.fail('cancel() is called'); + t.end(); + } + } + + const testError = new TypeError('test'); + + const rs = new ThinAsyncReadableByteStream(new Source()); + + let firstReadRejected = false; + rs.read(new Uint8Array(16)).catch(e => { + t.equal(e.value, testError); + t.equal(e.container.byteLength, 16); + firstReadRejected = true; + }); + rs.read(new Uint8Array(16)).catch(e => { + t.equal(e.value, testError); + t.equal(e.container.byteLength, 16); + t.equal(firstReadRejected, true); + t.equal(readCount, 2); + t.end(); + }); + + delegate.error(testError); +}); From 4b9f4382dbc053922979f1966bc6a1c666cb55b0 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Mon, 9 Mar 2015 14:51:21 +0900 Subject: [PATCH 90/93] Rename streams to Reader. Stream suffix would be given to the source from now --- .../experimental/fake-file-backed-byte-source.js | 8 ++++---- .../experimental/fake-unstoppable-byte-source.js | 2 +- .../lib/experimental/mock-byte-sink.js | 8 ++++---- .../lib/experimental/stream-queue.js | 6 +++--- ...e-stream.js => thin-byob-byte-stream-reader.js} | 2 +- ...e-byte-stream.js => thin-byte-stream-reader.js} | 2 +- ...e-byte-stream.js => thin-byte-stream-writer.js} | 2 +- ...in-readable-stream.js => thin-stream-reader.js} | 2 +- ...in-writable-stream.js => thin-stream-writer.js} | 2 +- .../thin-async-readable-byte-stream.js | 14 +++++++------- .../test/experimental/thin-stream.js | 1 - 11 files changed, 24 insertions(+), 25 deletions(-) rename reference-implementation/lib/experimental/{thin-async-readable-byte-stream.js => thin-byob-byte-stream-reader.js} (98%) rename reference-implementation/lib/experimental/{thin-readable-byte-stream.js => thin-byte-stream-reader.js} (98%) rename reference-implementation/lib/experimental/{thin-writable-byte-stream.js => thin-byte-stream-writer.js} (99%) rename reference-implementation/lib/experimental/{thin-readable-stream.js => thin-stream-reader.js} (98%) rename reference-implementation/lib/experimental/{thin-writable-stream.js => thin-stream-writer.js} (98%) diff --git a/reference-implementation/lib/experimental/fake-file-backed-byte-source.js b/reference-implementation/lib/experimental/fake-file-backed-byte-source.js index 8098478dc..f9c04f32d 100644 --- a/reference-implementation/lib/experimental/fake-file-backed-byte-source.js +++ b/reference-implementation/lib/experimental/fake-file-backed-byte-source.js @@ -1,5 +1,5 @@ -import { ThinReadableStream } from './thin-readable-stream'; -import { ThinReadableByteStream } from './thin-readable-byte-stream'; +import { ThinStreamReader } from './thin-stream-reader'; +import { ThinByteStreamReader } from './thin-byte-stream-reader'; import { fillArrayBufferView } from './thin-stream-utils'; @@ -202,7 +202,7 @@ export class FakeFile { createStream(strategy) { const source = new FileBackedUnderlyingSource(this._readFileInto.bind(this), strategy); - return new ThinReadableStream(source); + return new ThinStreamReader(source); } // Returns a manual pull readable stream. @@ -216,6 +216,6 @@ export class FakeFile { // - The stream is always pullable. createManualPullStream() { const source = new FileBackedManualPullUnderlyingSource(this._readFileInto.bind(this)); - return new ThinReadableByteStream(source); + return new ThinByteStreamReader(source); } } diff --git a/reference-implementation/lib/experimental/fake-unstoppable-byte-source.js b/reference-implementation/lib/experimental/fake-unstoppable-byte-source.js index 8f70b4ff3..c82e2f311 100644 --- a/reference-implementation/lib/experimental/fake-unstoppable-byte-source.js +++ b/reference-implementation/lib/experimental/fake-unstoppable-byte-source.js @@ -83,7 +83,7 @@ test('Adapting unstoppable push source', t => { } const source = new Source(); - const readableStream = new ReadableStream(source); + const readableStream = new ThinStreamReader(source); let count = 0; function pump() { diff --git a/reference-implementation/lib/experimental/mock-byte-sink.js b/reference-implementation/lib/experimental/mock-byte-sink.js index a7fbd9771..0254abf69 100644 --- a/reference-implementation/lib/experimental/mock-byte-sink.js +++ b/reference-implementation/lib/experimental/mock-byte-sink.js @@ -1,5 +1,5 @@ -import { ThinWritableStream } from './thin-writable-stream'; -import { ThinWritableByteStream } from './thin-writable-byte-stream'; +import { ThinStreamWriter } from './thin-stream-writer'; +import { ThinByteStreamWriter } from './thin-byte-stream-writer'; class MockFileUnderlyingSink { constructor(file) { @@ -100,10 +100,10 @@ export class MockFile { } createStream() { - return new ThinWritableStream(new MockFileUnderlyingSink(this)); + return new ThinStreamWriter(new MockFileUnderlyingSink(this)); } createStreamWithGarbage() { - return new ThinWritableByteStream(new MockFileUnderlyingSinkWithGarbage(this)); + return new ThinByteStreamWriter(new MockFileUnderlyingSinkWithGarbage(this)); } } diff --git a/reference-implementation/lib/experimental/stream-queue.js b/reference-implementation/lib/experimental/stream-queue.js index daa3e8302..dd4842361 100644 --- a/reference-implementation/lib/experimental/stream-queue.js +++ b/reference-implementation/lib/experimental/stream-queue.js @@ -1,5 +1,5 @@ -import { ThinWritableStream } from './thin-writable-stream'; -import { ThinReadableStream } from './thin-readable-stream'; +import { ThinStreamWriter } from './thin-stream-writer'; +import { ThinStreamReader } from './thin-stream-reader'; class StreamQueueShared { constructor(strategy) { @@ -172,5 +172,5 @@ export function createStreamQueue(strategy) { const sink = new StreamQueueUnderlyingSink(shared); sink.setSource(source); source.setSink(sink); - return { writable: new ThinWritableStream(sink), readable: new ThinReadableStream(source) }; + return { writable: new ThinStreamWriter(sink), readable: new ThinStreamReader(source) }; } diff --git a/reference-implementation/lib/experimental/thin-async-readable-byte-stream.js b/reference-implementation/lib/experimental/thin-byob-byte-stream-reader.js similarity index 98% rename from reference-implementation/lib/experimental/thin-async-readable-byte-stream.js rename to reference-implementation/lib/experimental/thin-byob-byte-stream-reader.js index ecff15676..e27cf75c7 100644 --- a/reference-implementation/lib/experimental/thin-async-readable-byte-stream.js +++ b/reference-implementation/lib/experimental/thin-byob-byte-stream-reader.js @@ -1,4 +1,4 @@ -export class ThinAsyncReadableByteStream { +export class ThinByobByteStreamReader { constructor(source) { this._source = source; diff --git a/reference-implementation/lib/experimental/thin-readable-byte-stream.js b/reference-implementation/lib/experimental/thin-byte-stream-reader.js similarity index 98% rename from reference-implementation/lib/experimental/thin-readable-byte-stream.js rename to reference-implementation/lib/experimental/thin-byte-stream-reader.js index 01d956e75..e20ced530 100644 --- a/reference-implementation/lib/experimental/thin-readable-byte-stream.js +++ b/reference-implementation/lib/experimental/thin-byte-stream-reader.js @@ -1,6 +1,6 @@ import { readableAcceptsCancel } from './thin-stream-base'; -export class ThinReadableByteStream { +export class ThinByteStreamReader { _initReadyPromise() { this._readyPromise = new Promise((resolve, reject) => { this._resolveReadyPromise = resolve; diff --git a/reference-implementation/lib/experimental/thin-writable-byte-stream.js b/reference-implementation/lib/experimental/thin-byte-stream-writer.js similarity index 99% rename from reference-implementation/lib/experimental/thin-writable-byte-stream.js rename to reference-implementation/lib/experimental/thin-byte-stream-writer.js index cd79b76ff..d12dbe04f 100644 --- a/reference-implementation/lib/experimental/thin-writable-byte-stream.js +++ b/reference-implementation/lib/experimental/thin-byte-stream-writer.js @@ -1,6 +1,6 @@ import { writableAcceptsWriteAndClose, writableAcceptsAbort } from './thin-stream-base'; -export class ThinWritableByteStream { +export class ThinByteStreamWriter { _initReadyPromise() { this._readyPromise = new Promise((resolve, reject) => { this._resolveReadyPromise = resolve; diff --git a/reference-implementation/lib/experimental/thin-readable-stream.js b/reference-implementation/lib/experimental/thin-stream-reader.js similarity index 98% rename from reference-implementation/lib/experimental/thin-readable-stream.js rename to reference-implementation/lib/experimental/thin-stream-reader.js index 2f6a6b188..ce2a39f03 100644 --- a/reference-implementation/lib/experimental/thin-readable-stream.js +++ b/reference-implementation/lib/experimental/thin-stream-reader.js @@ -1,6 +1,6 @@ import { readableAcceptsCancel } from './thin-stream-base'; -export class ThinReadableStream { +export class ThinStreamReader { _initReadyPromise() { this._readyPromise = new Promise((resolve, reject) => { this._resolveReadyPromise = resolve; diff --git a/reference-implementation/lib/experimental/thin-writable-stream.js b/reference-implementation/lib/experimental/thin-stream-writer.js similarity index 98% rename from reference-implementation/lib/experimental/thin-writable-stream.js rename to reference-implementation/lib/experimental/thin-stream-writer.js index 99badb86c..fbd0f3e5e 100644 --- a/reference-implementation/lib/experimental/thin-writable-stream.js +++ b/reference-implementation/lib/experimental/thin-stream-writer.js @@ -1,6 +1,6 @@ import { writableAcceptsWriteAndClose, writableAcceptsAbort } from './thin-stream-base'; -export class ThinWritableStream { +export class ThinStreamWriter { _initReadyPromise() { this._readyPromise = new Promise((resolve, reject) => { this._resolveReadyPromise = resolve; diff --git a/reference-implementation/test/experimental/thin-async-readable-byte-stream.js b/reference-implementation/test/experimental/thin-async-readable-byte-stream.js index 539f51225..5b96fac78 100644 --- a/reference-implementation/test/experimental/thin-async-readable-byte-stream.js +++ b/reference-implementation/test/experimental/thin-async-readable-byte-stream.js @@ -1,9 +1,9 @@ const test = require('tape-catch'); import { fillArrayBufferView } from '../../lib/experimental/thin-stream-utils'; -import { ThinAsyncReadableByteStream } from '../../lib/experimental/thin-async-readable-byte-stream'; +import { ThinByobByteStreamReader } from '../../lib/experimental/thin-byob-byte-stream-reader'; -test('Construct a ThinAsyncReadableByteStream', t => { +test('Construct a ThinByobByteStreamReader', t => { class Source { constructor(size) { this._bytesRemaining = size; @@ -35,7 +35,7 @@ test('Construct a ThinAsyncReadableByteStream', t => { const buffer = new Uint8Array(16); - const rs = new ThinAsyncReadableByteStream(new Source(1024)); + const rs = new ThinByobByteStreamReader(new Source(1024)); function pump() { fillArrayBufferView(buffer, 0); rs.read(buffer).then(result => { @@ -82,7 +82,7 @@ test('read() on a closed stream', t => { } } - const rs = new ThinAsyncReadableByteStream(new Source()); + const rs = new ThinByobByteStreamReader(new Source()); rs.read(new Uint8Array(16)).then(result => { t.equal(result.done, true); @@ -116,7 +116,7 @@ test('Close a stream with pending read()s', t => { } } - const rs = new ThinAsyncReadableByteStream(new Source()); + const rs = new ThinByobByteStreamReader(new Source()); let firstReadFulfilled = false; rs.read(new Uint8Array(16)).then(result => { @@ -154,7 +154,7 @@ test('read() on a errored stream', t => { } } - const rs = new ThinAsyncReadableByteStream(new Source()); + const rs = new ThinByobByteStreamReader(new Source()); rs.read(new Uint8Array(16)).catch(e => { t.equal(e.value, testError); @@ -190,7 +190,7 @@ test('Error a stream with pending read()s', t => { const testError = new TypeError('test'); - const rs = new ThinAsyncReadableByteStream(new Source()); + const rs = new ThinByobByteStreamReader(new Source()); let firstReadRejected = false; rs.read(new Uint8Array(16)).catch(e => { diff --git a/reference-implementation/test/experimental/thin-stream.js b/reference-implementation/test/experimental/thin-stream.js index 652a69766..363085942 100644 --- a/reference-implementation/test/experimental/thin-stream.js +++ b/reference-implementation/test/experimental/thin-stream.js @@ -1,6 +1,5 @@ const test = require('tape-catch'); -import { ThinReadableStream } from '../../lib/experimental/thin-readable-stream'; import { selectStreams, pipeStreams } from '../../lib/experimental/thin-stream-base'; import { createStreamQueue } from '../../lib/experimental/stream-queue'; import { FakeFile } from '../../lib/experimental/fake-file-backed-byte-source'; From 4be050fade1c4cefa2e7b410b620bb98ec5f9124 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Mon, 9 Mar 2015 17:00:47 +0900 Subject: [PATCH 91/93] Add ThinStreamReader delegate revocation --- .../fake-file-backed-byte-source.js | 27 +++- .../lib/experimental/stream-queue.js | 37 ++++-- .../lib/experimental/thin-stream-base.js | 8 +- .../lib/experimental/thin-stream-reader.js | 123 +++++++++++------- .../test/experimental/thin-stream.js | 12 +- 5 files changed, 133 insertions(+), 74 deletions(-) diff --git a/reference-implementation/lib/experimental/fake-file-backed-byte-source.js b/reference-implementation/lib/experimental/fake-file-backed-byte-source.js index f9c04f32d..decc3f469 100644 --- a/reference-implementation/lib/experimental/fake-file-backed-byte-source.js +++ b/reference-implementation/lib/experimental/fake-file-backed-byte-source.js @@ -29,7 +29,10 @@ class FileBackedUnderlyingSource { this._queue.push(result.writtenRegion); this._queueSize += result.writtenRegion.byteLength; - this._delegate.markReadable(); + if (this._markReadable !== undefined) { + this._markReadable(); + this._markReadable = undefined; + } if (result.closed) { this._strategy = undefined; @@ -65,11 +68,18 @@ class FileBackedUnderlyingSource { const view = new Uint8Array(size); this._fileReadPromise = this._readFileInto(view) .then(this._handleFileReadResult.bind(this)) - .catch(this._delegate.markErrored.bind(this._delegate)); + .catch(e => { + if (this._markErrored !== undefined) { + this._markErrored(e); + this._markErrored = undefined; + } + }); } - start(delegate) { - this._delegate = delegate; + start(markReadable, markClosed, markErrored) { + this._markReadable = markReadable; + this._markClosed = markClosed; + this._markErrored = markErrored; this._pull(); } @@ -86,16 +96,19 @@ class FileBackedUnderlyingSource { this._pull(); } - read() { + read(markReadable, markClosed) { const view = this._queue.shift(); this._queueSize -= view.byteLength; if (this._queue.length === 0) { if (this._draining) { - this._delegate.markClosed(); + markClosed(); } else { - this._delegate.markWaiting(); + this._markReadable = markReadable; + this._markClosed = markClosed; } + } else { + markReadable(); } if (!this._draining) { diff --git a/reference-implementation/lib/experimental/stream-queue.js b/reference-implementation/lib/experimental/stream-queue.js index dd4842361..b53081088 100644 --- a/reference-implementation/lib/experimental/stream-queue.js +++ b/reference-implementation/lib/experimental/stream-queue.js @@ -108,22 +108,39 @@ class StreamQueueUnderlyingSource { this._sink = sink; } - start(delegate) { - this._delegate = delegate; + start(markReadable, markClosed, markErrored) { + this._markReadable = markReadable; + this._markClosed = markClosed; + this._markErrored = markErrored; } onQueueFill() { - this._delegate.markReadable(); + if (this._markReadable === undefined) { + return; + } + + this._markReadable(); + this._markReadable = undefined; + this._markClosed = undefined; } onStartDraining() { - if (this._shared._queue.length === 0) { - this._delegate.markClosed(); + if (this._shared._queue.length !== 0 || this._markClosed === undefined) { + return; } + + this._markClosed(); + this._markReadable = undefined; + this._markClosed = undefined; } abort(reason) { - this._delegate.markErrored(reason); + if (this._markErrored === undefined) { + return; + } + + this._markErrored(reason); + this._markErrored = undefined; } onWindowUpdate(v) { @@ -138,15 +155,15 @@ class StreamQueueUnderlyingSource { this._sink.onWindowUpdate(); } - read() { + read(markReadable, markClosed) { const entry = this._shared._queue.shift(); if (this._shared._queue.length === 0) { if (this._shared._draining) { - this._delegate.markClosed(); - } else { - this._delegate.markWaiting(); + markClosed(); } + } else { + markReadable(); } this._shared._queueSize -= entry.size; diff --git a/reference-implementation/lib/experimental/thin-stream-base.js b/reference-implementation/lib/experimental/thin-stream-base.js index 1d857c057..cac5d1ecc 100644 --- a/reference-implementation/lib/experimental/thin-stream-base.js +++ b/reference-implementation/lib/experimental/thin-stream-base.js @@ -14,7 +14,7 @@ export function readableAcceptsCancel(state) { export function selectStreams(readable, writable) { const promises = []; - promises.push(readable.errored); + promises.push(readable.closed); if (readable.state === 'waiting') { promises.push(readable.ready); } @@ -40,7 +40,7 @@ export function pipeStreams(source, dest) { dest.abort(error); } if (readableAcceptsCancel(source.state)) { - source.abort(error); + source.cancel(error); } reject(error); } @@ -49,7 +49,7 @@ export function pipeStreams(source, dest) { for (;;) { if (source.state === 'errored') { if (writableAcceptsAbort(dest.state)) { - dest.abort(source.error); + source.closed.catch(dest.abort.bind(dest)); } reject(new TypeError('source is errored')); return; @@ -79,7 +79,7 @@ export function pipeStreams(source, dest) { } selectStreams(source, dest) - .then(loop) + .then(loop, loop) .catch(disposeStreams); return; } diff --git a/reference-implementation/lib/experimental/thin-stream-reader.js b/reference-implementation/lib/experimental/thin-stream-reader.js index ce2a39f03..7c8f97123 100644 --- a/reference-implementation/lib/experimental/thin-stream-reader.js +++ b/reference-implementation/lib/experimental/thin-stream-reader.js @@ -14,30 +14,25 @@ export class ThinStreamReader { this._initReadyPromise(); - this._erroredPromise = new Promise((resolve, reject) => { - this._resolveErroredPromise = resolve; + this._closedPromise = new Promise((resolve, reject) => { + this._resolveClosedPromise = resolve; + this._rejectClosedPromise = reject; }); - this._error = undefined; this._window = 0; - const delegate = { - markWaiting: this._markWaiting.bind(this), - markReadable: this._markReadable.bind(this), - markClosed: this._markClosed.bind(this), + this._waitingID = {}; - markErrored: this._markErrored.bind(this) - }; - - this._source.start(delegate); + this._source.start( + this._markReadable.bind(this, this._waitingID), + this._markClosed.bind(this, this._waitingID), + this._markErrored.bind(this)); } get state() { return this._state; } - // Auto pull interfaces. - get window() { return this._window; } @@ -52,8 +47,6 @@ export class ThinStreamReader { this._source.onWindowUpdate(v); } - // Reading interfaces. - get ready() { return this._readyPromise; } @@ -63,66 +56,102 @@ export class ThinStreamReader { throw new TypeError('not readable'); } - return this._source.read(); + this._waitingID = {}; + + try { + const result = this._source.read( + this._markReadable.bind(this, this._waitingID), + this._markClosed.bind(this, this._waitingID)); + + if (this._waitingID !== undefined) { + this._initReadyPromise(); + this._state = 'waiting'; + } + + return result; + } catch (e) { + const error = new TypeError('underlying source is broken: ' + e); + this._markErrored(error); + throw error; + } } cancel(reason) { if (!readableAcceptsCancel(this._state)) { - throw new TypeError('already ' + this._state); + return Promise.reject(new TypeError('already ' + this._state)); } - this._source.cancel(reason); + if (this._waitingID !== undefined) { + this._waitingID = undefined; - this._state = 'cancelled'; - } + this._resolveReadyPromise(); + this._resolveReadyPromise = undefined; + } - // Error receiving interfaces. + this._resolveClosedPromise(); + this._resolveClosedPromise = undefined; + this._rejectClosedPromise = undefined; - get errored() { - return this._erroredPromise; - } + this._state = 'closed'; - get error() { - if (this._state !== 'errored') { - throw new TypeError('not errored'); - } + return this._source.cancel(reason); + } - return this._error; + get closed() { + return this._closedPromise; } - // Methods exposed only to the underlying source. + _markReadable(waitingID) { + if (this._waitingID !== waitingID) { + throw new TypeError('this callback is already expired'); + } + this._waitingID = undefined; - _markWaiting() { if (this._state === 'waiting') { - return; + this._resolveReadyPromise(); + this._resolveReadyPromise = undefined; } - this._initReadyPromise(); - this._state = 'waiting'; + this._state = 'readable'; } - _markReadable() { - if (this._state === 'readable') { - return; + _markClosed(waitingID) { + if (this._waitingID !== waitingID) { + throw new TypeError('this callback is already expired'); } + this._waitingID = undefined; - this._resolveReadyPromise(); - this._resolveReadyPromise = undefined; - this._state = 'readable'; - } - - _markClosed() { - if (this._state !== 'readable') { + if (this._state === 'waiting') { this._resolveReadyPromise(); this._resolveReadyPromise = undefined; } + + this._resolveClosedPromise(); + this._resolveClosedPromise = undefined; + this._rejectClosedPromise = undefined; + this._state = 'closed'; } _markErrored(error) { - this._resolveErroredPromise(); - this._resolveErroredPromise = undefined; + if (this._state === 'closed') { + throw new TypeError('already closed'); + } + if (this._state === 'errored') { + throw new TypeError('already errored'); + } + + if (this._waitingID !== undefined) { + this._waitingID = undefined; + + this._resolveReadyPromise(); + this._resolveReadyPromise = undefined; + } + + this._rejectClosedPromise(error); + this._resolveClosedPromise = undefined; + this._rejectClosedPromise = undefined; + this._state = 'errored'; - this._error = error; } } diff --git a/reference-implementation/test/experimental/thin-stream.js b/reference-implementation/test/experimental/thin-stream.js index 363085942..17c4d0826 100644 --- a/reference-implementation/test/experimental/thin-stream.js +++ b/reference-implementation/test/experimental/thin-stream.js @@ -178,9 +178,9 @@ test('abort()', t => { const testError = new TypeError('foo'); - rs.errored.then(() => { + rs.closed.catch(e => { t.equal(rs.state, 'errored', 'rs.state'); - t.equal(rs.error, testError, 'rs.error'); + t.equal(e, testError, 'rs.closed'); t.end(); }); @@ -200,7 +200,7 @@ test('cancel()', t => { const testError = new TypeError('foo'); rs.cancel(testError); - t.equal(rs.state, 'cancelled', 'rs.state'); + t.equal(rs.state, 'closed', 'rs.state'); ws.errored.then(() => { t.equal(ws.state, 'errored', 'ws.state'); @@ -283,9 +283,9 @@ test('pipeStreams(): abort() propagation', t => { v => t.fail('pipePromise is fulfilled with ' + v), e => t.equal(e.message, 'source is errored', 'rejection reason of pipePromise')); - rs1.errored.then(() => { + rs1.closed.catch(e => { t.equal(rs1.state, 'errored', 'rs.state'); - t.equal(rs1.error, testError, 'rs1.error'); + t.equal(e, testError, 'rs1.closed'); }); ws0.abort(testError); @@ -379,7 +379,7 @@ test('Reading from a file backed byte source using auto pull readable stream', t const readView = rs.read(); count += readView.byteLength; } else if (rs.state === 'waiting') { - Promise.race([rs.ready, rs.errored]).then(pump); + Promise.race([rs.ready, rs.closed]).then(pump, pump); return; } else { t.fail('rs.state is invalid: ' + rs.state); From a0d8aea7642d8e5248d4672ec724bbacb14ef3bd Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Wed, 11 Mar 2015 23:46:20 +0900 Subject: [PATCH 92/93] Implement ThinByobByteStreamReader --- .../fake-file-backed-byte-source.js | 93 ++++---- .../thin-byob-byte-stream-reader.js | 219 +++++++++++------- .../experimental/thin-byte-stream-reader.js | 159 ------------- ...eam.js => thin-byob-byte-stream-reader.js} | 142 +++++++----- .../test/experimental/thin-stream.js | 39 ++-- 5 files changed, 289 insertions(+), 363 deletions(-) delete mode 100644 reference-implementation/lib/experimental/thin-byte-stream-reader.js rename reference-implementation/test/experimental/{thin-async-readable-byte-stream.js => thin-byob-byte-stream-reader.js} (50%) diff --git a/reference-implementation/lib/experimental/fake-file-backed-byte-source.js b/reference-implementation/lib/experimental/fake-file-backed-byte-source.js index decc3f469..858211c0d 100644 --- a/reference-implementation/lib/experimental/fake-file-backed-byte-source.js +++ b/reference-implementation/lib/experimental/fake-file-backed-byte-source.js @@ -1,5 +1,5 @@ import { ThinStreamReader } from './thin-stream-reader'; -import { ThinByteStreamReader } from './thin-byte-stream-reader'; +import { ThinByobByteStreamReader } from './thin-byob-byte-stream-reader'; import { fillArrayBufferView } from './thin-stream-utils'; @@ -125,62 +125,76 @@ class FileBackedUnderlyingSource { } } -class FileBackedManualPullUnderlyingSource { +class FileBackedByobUnderlyingSource { constructor(readFileInto) { this._readFileInto = readFileInto; - this._currentResult = undefined; - - this._draining = false; - this._cancelled = false; this._fileReadPromise = undefined; + + this._pendingRequests = []; } - _handleFileReadResult(result) { - if (this._cancelled) { - return; + _clearPendingRequests() { + while (this._pendingRequests.length > 0) { + const entry = this._pendingRequests.shift(); + entry.done(); } + } + + _callReadFileInto(view, done) { + this._fileReadPromise = this._readFileInto(view) + .then(this._handleFileReadFulfillment.bind(this, done)) + .catch(this._handleFileReadRejection.bind(this, done)); + } + _handleFileReadFulfillment(done, result) { this._fileReadPromise = undefined; - this._delegate.markPullable(); - this._currentResult = result.writtenRegion; - this._delegate.markReadable(); + if (this._cancelled) { + this._clearPendingRequests(); + return; + } + + done(result.writtenRegion.byteLength); if (result.closed) { - this._draining = true; + this._close(); + this._clearPendingRequests(); + } else { + if (this._pendingRequests.length > 0) { + const entry = this._pendingRequests.shift(); + this._callReadFileInto(entry.view, entry.done); + } } } - start(delegate) { - this._delegate = delegate; + _handleFileReadRejection(done, reason) { + if (this._cancelled) { + this._clearPendingRequests(); + return; + } - this._delegate.markPullable(); + this._error(reason); + this._clearPendingRequests(); } - pull(view) { - this._fileReadPromise = this._readFileInto(view) - .then(this._handleFileReadResult.bind(this)) - .catch(this._delegate.markErrored.bind(this._delegate)); - - this._delegate.markNotPullable(); + start(close, error) { + this._close = close; + this._error = error; } - read() { - if (this._draining) { - this._delegate.markClosed(); - } else { - this._delegate.markWaiting(); + read(view, done) { + if (this._fileReadPromise !== undefined) { + this._pendingRequests.push({view, done}); + return; } - const result = this._currentResult; - this._currentResult = undefined; - return result; + this._callReadFileInto(view, done); } - cancel() { + cancel(reason) { this._cancelled = true; } } @@ -213,22 +227,13 @@ export class FakeFile { }); } - createStream(strategy) { + createReader(strategy) { const source = new FileBackedUnderlyingSource(this._readFileInto.bind(this), strategy); return new ThinStreamReader(source); } - // Returns a manual pull readable stream. - // - // Example semantics: - // - // POSIX socket: - // - The stream becomes pullable when epoll(7) returns, and stays to be pullable until read(2) returns EAGAIN. - // - // Blocking or async I/O interfaces which takes a buffer on reading function call: - // - The stream is always pullable. - createManualPullStream() { - const source = new FileBackedManualPullUnderlyingSource(this._readFileInto.bind(this)); - return new ThinByteStreamReader(source); + createByobReader() { + const source = new FileBackedByobUnderlyingSource(this._readFileInto.bind(this)); + return new ThinByobByteStreamReader(source); } } diff --git a/reference-implementation/lib/experimental/thin-byob-byte-stream-reader.js b/reference-implementation/lib/experimental/thin-byob-byte-stream-reader.js index e27cf75c7..1ef561281 100644 --- a/reference-implementation/lib/experimental/thin-byob-byte-stream-reader.js +++ b/reference-implementation/lib/experimental/thin-byob-byte-stream-reader.js @@ -2,92 +2,129 @@ export class ThinByobByteStreamReader { constructor(source) { this._source = source; - this._pendingReadRequests = []; - this._cancelled = false; + this._pendingReads = []; + this._pendingSourceReads = []; + + this._closed = false; this._errored = false; this._errorReason = undefined; - this._closed = false; + + this._fatalErrored = false; this._closedPromise = new Promise((resolve, reject) => { this._resolveClosedPromise = resolve; this._rejectClosedPromise = reject; }); - const delegate = { - fulfill: this._fulfill.bind(this), - close: this._close.bind(this), - error: this._error.bind(this) - }; - - this._source.start(delegate); + if (this._source.start !== undefined) { + this._source.start(this._close.bind(this), this._error.bind(this), this._fatalError.bind(this)); + } } get closed() { return this._closedPromise; } - read(container) { - // TODO: Detach container + _processPendingReads() { + while (this._pendingReads.length > 0) { + const entry = this._pendingReads.shift(); + if (this._fatalErrored) { + entry.reject({reason: this._errorReason, view: undefined}); + } else if (this._errored) { + entry.reject({reason: this._errorReason, view: entry.view.subarray(0, 0)}); + } else if (this._closed) { + entry.resolve({view: entry.view, done: true}); + } else if (this._source === undefined) { + entry.reject({reason: new TypeError('already released'), view: entry.view.subarray(0, 0)}); + } else { + // Not reached + } + } + } - if (this._cancelled) { - return Promise.reject({value: new TypeError('already cancelled'), container}); + _handleFatalError(reason) { + this._fatalErrored = true; + this._errorReason = reason; + + while (this._pendingSourceReads.length > 0) { + const entry = this._pendingSourceReads.shift(); + entry.reject({reason, view: undefined}); } + this._processPendingReads(); + } + + // Tell the stream that the underlying source has done or aborted writing to the oldest pending view. + _done(bytesWritten) { + if (this._fatalErrored) { + throw new TypeError('already fatal-errored'); + } + + if (this._pendingSourceReads.length === 0) { + throw new TypeError('no pending read'); + } + const entry = this._pendingSourceReads.shift(); + + // TODO: Detach entry.view + if (this._errored) { - return Promise.reject({value: this._errorReason, container}); + entry.reject({reason: this._errorReason, view: entry.view.subarray(0, 0)}); + if (this._pendingSourceReads.length === 0) { + this._processPendingReads(); + } + return; } if (this._closed) { - return Promise.resolve({done: true, container}); + entry.resolve({value: entry.view, done: true}); + if (this._pendingSourceReads.length === 0) { + this._processPendingReads(); + } + return; } - return new Promise((resolve, reject) => { - this._pendingReadRequests.push({resolve, reject, container}); - this._source.read(container); - }); - } + if (bytesWritten === undefined) { + throw new TypeError('bytesWritten is undefined'); + } - cancel(reason) { - if (this._cancelled) { - return Promise.reject(new TypeError('already cancelled')); + if (entry.view.byteLength < bytesWritten) { + throw new RangeError('bytesWritten is bigger than the given view'); } + entry.resolve({value: entry.view.subarray(0, bytesWritten), done: false}); + } + + _close() { + if (this._fatalErrored) { + throw new TypeError('already fatal-errored'); + } if (this._errored) { - return Promise.reject(this._errorReason); + throw new TypeError('already errored'); } if (this._closed) { - return Promise.reject(new TypeError('already closed')); + throw new TypeError('already closed'); } - this._cancelled = true; - - while (this._pendingReadRequests.length !== 0) { - const request = this._pendingReadRequests.shift(); - request.reject({value: new TypeError('cancelled'), container: request.container}); - } + this._closed = true; this._resolveClosedPromise(); this._resolveClosedPromise = undefined; this._rejectClosedPromise = undefined; - - return Promise.resolve(this._source.cancel(reason)); } - release() { - if (this._pendingReadRequests.length !== 0) { - throw new TypeError('there are pending reads'); - } + _errorInternal(reason) { + this._errored = true; + this._errorReason = reason; - this._source = undefined; + this._rejectClosedPromise(reason); + this._resolveClosedPromise = undefined; + this._rejectClosedPromise = undefined; } - _fulfill(value) { - // TODO: Detach value - - if (this._source === undefined) { - throw new TypeError('already released'); + _error(reason) { + if (this._fatalErrored) { + throw new TypeError('already fatal-errored'); } - if (this._errored) { throw new TypeError('already errored'); } @@ -95,60 +132,86 @@ export class ThinByobByteStreamReader { throw new TypeError('already closed'); } - if (this._pendingReadRequests.length === 0) { - throw new TypeError('no pending read request'); + this._errorInternal(reason); + } + + _fatalError(reason) { + if (this._fatalErrored) { + throw new TypeError('already fatal-errored'); } - const request = this._pendingReadRequests.shift(); - request.resolve({done: false, value}); + this._handleFatalError(reason); } - _close() { + read(view) { + if (this._fatalErrored) { + return Promise.reject({reason: this._errorReason, view: undefined}); + } + + return new Promise((resolve, reject) => { + if (this._errored || this._closed || this._source === undefined) { + if (this._pendingSourceReads.length > 0) { + this._pendingReads.push({view, resolve, reject}); + } else { + if (this._errored) { + reject({reason: this._errorReason, view: view.subarray(0, 0)}); + } else if (this._closed) { + resolve({value: view, done: true}); + } else { + reject({reason: new TypeError('already released'), view: view.subarray(0, 0)}); + } + } + return; + } + + // TODO: Detach view + + this._pendingSourceReads.push({view, resolve, reject}); + + try { + this._source.read( + view, + this._done.bind(this), + this._close.bind(this), + this._error.bind(this), + this._fatalError.bind(this)); + } catch (e) { + if (!(this._fatalErrored || this._errored || this._closed)) { + this._errorInternal(e); + } + } + }); + } + + cancel(reason) { if (this._source === undefined) { - throw new TypeError('already released'); + return Promise.reject(new TypeError('already released')); } - if (this._errored) { - throw new TypeError('already errored'); + if (this._fatalErrored || this._errored) { + return Promise.reject(this._errorReason); } + if (this._closed) { - throw new TypeError('already closed'); + return Promise.reject(new TypeError('already closed')); } this._closed = true; - while (this._pendingReadRequests.length !== 0) { - const request = this._pendingReadRequests.shift(); - request.resolve({done: true, container: request.container}); - } - this._resolveClosedPromise(); this._resolveClosedPromise = undefined; this._rejectClosedPromise = undefined; + + return new Promise((resolve, reject) => { + resolve(this._source.cancel(reason)); + }).then(r => undefined); } - _error(reason) { + releaseLock() { if (this._source === undefined) { throw new TypeError('already released'); } - if (this._errored) { - throw new TypeError('already errored'); - } - if (this._closed) { - throw new TypeError('already closed'); - } - - this._errored = true; - this._errorReason = reason; - - while (this._pendingReadRequests.length !== 0) { - const request = this._pendingReadRequests.shift(); - request.reject({value: reason, container: request.container}); - } - - this._rejectClosedPromise(reason); - this._resolveClosedPromise = undefined; - this._rejectClosedPromise = undefined; + this._source = undefined; } } diff --git a/reference-implementation/lib/experimental/thin-byte-stream-reader.js b/reference-implementation/lib/experimental/thin-byte-stream-reader.js deleted file mode 100644 index e20ced530..000000000 --- a/reference-implementation/lib/experimental/thin-byte-stream-reader.js +++ /dev/null @@ -1,159 +0,0 @@ -import { readableAcceptsCancel } from './thin-stream-base'; - -export class ThinByteStreamReader { - _initReadyPromise() { - this._readyPromise = new Promise((resolve, reject) => { - this._resolveReadyPromise = resolve; - }); - } - - _initPullReadyPromise() { - this._pullReadyPromise = new Promise((resolve, reject) => { - this._resolvePullReadyPromise = resolve; - }); - } - - constructor(source) { - this._source = source; - - this._state = 'waiting'; - - this._initReadyPromise(); - - this._erroredPromise = new Promise((resolve, reject) => { - this._resolveErroredPromise = resolve; - }); - this._error = undefined; - - this._pullable = false; - this._initPullReadyPromise(); - - const delegate = { - markPullable: this._markPullable.bind(this), - markNotPullable: this._markNotPullable.bind(this), - - markWaiting: this._markWaiting.bind(this), - markReadable: this._markReadable.bind(this), - markClosed: this._markClosed.bind(this), - - markErrored: this._markErrored.bind(this) - }; - - this._source.start(delegate); - } - - get state() { - return this._state; - } - - // Manual pull interfaces. - - get pullable() { - return this._pullable; - } - - get pullReady() { - return this._pullReadyPromise; - } - - pull(view) { - if (!this._pullable) { - throw new TypeError('not pullable'); - } - - this._source.pull(view); - } - - // Reading interfaces. - - get ready() { - return this._readyPromise; - } - - read() { - if (this._state !== 'readable') { - throw new TypeError('not readable'); - } - - return this._source.read(); - } - - cancel(reason) { - if (!readableAcceptsCancel(this._state)) { - throw new TypeError('already ' + this._state); - } - - this._source.cancel(reason); - - this._state = 'cancelled'; - } - - // Error receiving interfaces. - - get errored() { - return this._erroredPromise; - } - - get error() { - if (this._state !== 'errored') { - throw new TypeError('not errored'); - } - - return this._error; - } - - // Methods exposed only to the underlying source. - - _markNotPullable() { - if (!this._pullable) { - return; - } - - this._initPullReadyPromise(); - this._pullable = false; - } - - _markPullable() { - if (this._pullable) { - return; - } - - this._resolvePullReadyPromise(); - this._resolvePullReadyPromise = undefined; - this._pullable = true; - } - - _markWaiting() { - if (this._state === 'waiting') { - return; - } - - this._initReadyPromise(); - this._state = 'waiting'; - } - - _markReadable() { - if (this._state === 'readable') { - return; - } - - this._resolveReadyPromise(); - this._resolveReadyPromise = undefined; - this._state = 'readable'; - } - - _markClosed() { - if (this._state !== 'readable') { - this._resolveReadyPromise(); - this._resolveReadyPromise = undefined; - } - this._state = 'closed'; - } - - _markErrored(error) { - this._resolveErroredPromise(); - this._resolveErroredPromise = undefined; - this._state = 'errored'; - this._error = error; - } -} diff --git a/reference-implementation/test/experimental/thin-async-readable-byte-stream.js b/reference-implementation/test/experimental/thin-byob-byte-stream-reader.js similarity index 50% rename from reference-implementation/test/experimental/thin-async-readable-byte-stream.js rename to reference-implementation/test/experimental/thin-byob-byte-stream-reader.js index 5b96fac78..8eb712bbb 100644 --- a/reference-implementation/test/experimental/thin-async-readable-byte-stream.js +++ b/reference-implementation/test/experimental/thin-byob-byte-stream-reader.js @@ -9,20 +9,18 @@ test('Construct a ThinByobByteStreamReader', t => { this._bytesRemaining = size; } - start(delegate) { - this._delegate = delegate; - } - - read(container) { + read(view, done, close) { if (this._bytesRemaining === 0) { - this._delegate.close(); + close(); + // Call done() only to return the buffer. + done(); return; } - const bytesToFill = Math.min(this._bytesRemaining, container.byteLength); - fillArrayBufferView(container, 1, bytesToFill); + const bytesToFill = Math.min(this._bytesRemaining, view.byteLength); + fillArrayBufferView(view, 1, bytesToFill); this._bytesRemaining -= bytesToFill; - this._delegate.fulfill(container.subarray(0, bytesToFill)); + done(bytesToFill); } cancel(reason) { @@ -38,24 +36,24 @@ test('Construct a ThinByobByteStreamReader', t => { const rs = new ThinByobByteStreamReader(new Source(1024)); function pump() { fillArrayBufferView(buffer, 0); - rs.read(buffer).then(result => { - if (result.done) { + rs.read(buffer).then(({value, done}) => { + if (done) { t.equal(bytesRead, 1024); t.end(); + return; } else { - const view = result.value; - for (let i = 0; i < view.byteLength; ++i) { - if (view[i] !== 1) { - t.fail('view[' + i + '] is ' + view[i]); + for (let i = 0; i < value.byteLength; ++i) { + if (value[i] !== 1) { + t.fail('value[' + i + '] is ' + value[i]); t.end(); return; } } - bytesRead += view.byteLength; + bytesRead += value.byteLength; pump(); } - }, reason => { - t.fail(reason); + }, e => { + t.fail(e); t.end(); }).catch(e => { t.fail(e); @@ -67,8 +65,8 @@ test('Construct a ThinByobByteStreamReader', t => { test('read() on a closed stream', t => { class Source { - start(delegate) { - delegate.close(); + start(close) { + close(); } read() { @@ -84,13 +82,13 @@ test('read() on a closed stream', t => { const rs = new ThinByobByteStreamReader(new Source()); - rs.read(new Uint8Array(16)).then(result => { - t.equal(result.done, true); - t.equal(result.container.byteLength, 16); + rs.read(new Uint8Array(16)).then(({value, done}) => { + t.equal(done, true); + t.equal(value.byteLength, 16); - rs.read(new Uint8Array(16)).then(result => { - t.equal(result.done, true); - t.equal(result.container.byteLength, 16); + rs.read(new Uint8Array(16)).then(({value, done}) => { + t.equal(done, true); + t.equal(value.byteLength, 16); t.end(); }); }); @@ -99,15 +97,18 @@ test('read() on a closed stream', t => { test('Close a stream with pending read()s', t => { let readCount = 0; - let delegate = undefined; + let close = undefined; + let done = undefined; class Source { - start(delegate_) { - delegate = delegate_; + start(close_) { + close = close_; } - read(container) { - t.equal(container.byteLength, 16); + read(view, done_) { + t.equal(view.byteLength, 16); ++readCount; + + done = done_; } cancel(reason) { @@ -119,28 +120,37 @@ test('Close a stream with pending read()s', t => { const rs = new ThinByobByteStreamReader(new Source()); let firstReadFulfilled = false; - rs.read(new Uint8Array(16)).then(result => { - t.equal(result.done, true); - t.equal(result.container.byteLength, 16); + rs.read(new Uint8Array(16)).then(({value, done}) => { + t.equal(done, true); + t.equal(value.byteLength, 16); firstReadFulfilled = true; + }).catch(e => { + t.fail(e); + t.end(); }); - rs.read(new Uint8Array(16)).then(result => { - t.equal(result.done, true); - t.equal(result.container.byteLength, 16); + rs.read(new Uint8Array(16)).then(({value, done}) => { + t.equal(done, true); + t.equal(value.byteLength, 16); t.equal(firstReadFulfilled, true); t.equal(readCount, 2); t.end(); + }).catch(e => { + t.fail(e); + t.end(); }); - delegate.close(); + close(); + // Return the buffers. + done(); + done(); }); test('read() on a errored stream', t => { const testError = new TypeError('test'); class Source { - start(delegate) { - delegate.error(testError); + start(close, error) { + error(testError); } read() { @@ -156,30 +166,41 @@ test('read() on a errored stream', t => { const rs = new ThinByobByteStreamReader(new Source()); - rs.read(new Uint8Array(16)).catch(e => { - t.equal(e.value, testError); - t.equal(e.container.byteLength, 16); + rs.read(new Uint8Array(16)).catch(({reason, view}) => { + t.equal(reason, testError); + t.equal(view.byteLength, 0); + t.equal(view.buffer.byteLength, 16); - rs.read(new Uint8Array(16)).catch(e => { - t.equal(e.value, testError); - t.equal(e.container.byteLength, 16); + rs.read(new Uint8Array(16)).catch(({reason, view}) => { + t.equal(reason, testError); + t.equal(view.byteLength, 0); + t.equal(view.buffer.byteLength, 16); + t.end(); + }).catch(e => { + t.fail(e); t.end(); }); + }).catch(e => { + t.fail(e); + t.end(); }); }); test('Error a stream with pending read()s', t => { let readCount = 0; - let delegate = undefined; + let error = undefined; + let done = undefined; class Source { - start(delegate_) { - delegate = delegate_; + start(close, error_) { + error = error_; } - read(container) { - t.equal(container.byteLength, 16); + read(view, done_) { + t.equal(view.byteLength, 16); ++readCount; + + done = done_; } cancel(reason) { @@ -193,18 +214,23 @@ test('Error a stream with pending read()s', t => { const rs = new ThinByobByteStreamReader(new Source()); let firstReadRejected = false; - rs.read(new Uint8Array(16)).catch(e => { - t.equal(e.value, testError); - t.equal(e.container.byteLength, 16); + rs.read(new Uint8Array(16)).catch(({reason, view}) => { + t.equal(reason, testError); + t.equal(view.byteLength, 0); + t.equal(view.buffer.byteLength, 16); firstReadRejected = true; }); - rs.read(new Uint8Array(16)).catch(e => { - t.equal(e.value, testError); - t.equal(e.container.byteLength, 16); + rs.read(new Uint8Array(16)).catch(({reason, view}) => { + t.equal(reason, testError); + t.equal(view.byteLength, 0); + t.equal(view.buffer.byteLength, 16); t.equal(firstReadRejected, true); t.equal(readCount, 2); t.end(); }); - delegate.error(testError); + error(testError); + // Return the buffers. + done(); + done(); }); diff --git a/reference-implementation/test/experimental/thin-stream.js b/reference-implementation/test/experimental/thin-stream.js index 17c4d0826..779451ad1 100644 --- a/reference-implementation/test/experimental/thin-stream.js +++ b/reference-implementation/test/experimental/thin-stream.js @@ -323,45 +323,36 @@ test('pipeStreams(): cancel() propagation', t => { test('Reading from a file backed byte source using manual pull readable stream', t => { const file = new FakeFile(1024); - const rs = file.createManualPullStream(); + const rs = file.createByobReader(); let view = new Uint8Array(10); let count = 0; function pump() { - for (;;) { - if (rs.state === 'errored') { - - } else if (rs.state === 'closed') { + rs.read(view).then(({value, done}) => { + if (done) { t.equal(count, 1024); t.end(); return; - } else if (rs.state === 'readable') { - const readView = rs.read(); - count += readView.byteLength; - view = new Uint8Array(readView.buffer); - } else if (rs.state === 'waiting') { - if (view !== undefined && rs.pullable) { - rs.pull(view); - view = undefined; - continue; - } - - const promises = [rs.ready, rs.errored]; - if (!rs.pullable) { - promises.push(rs.pullReady); - } - Promise.race(promises).then(pump); - return; } - } + + count += value.byteLength; + view = new Uint8Array(value.buffer); + pump(); + }, ({reason, view}) => { + t.fail(reason); + t.end(); + }).catch(e => { + t.fail(e); + t.end(); + }); } pump(); }); test('Reading from a file backed byte source using auto pull readable stream', t => { const file = new FakeFile(1024); - const rs = file.createStream(new AdjustableArrayBufferStrategy()); + const rs = file.createReader(new AdjustableArrayBufferStrategy()); rs.window = 64; let count = 0; From 406ac64e7f83375497eee273bf638c314fce10a4 Mon Sep 17 00:00:00 2001 From: Takeshi Yoshino Date: Thu, 12 Mar 2015 13:53:40 +0900 Subject: [PATCH 93/93] More tests and comments for ThinByobByteStreamReader --- .../thin-byob-byte-stream-reader.js | 43 ++-- .../thin-byob-byte-stream-reader.js | 235 ++++++++++++++---- 2 files changed, 213 insertions(+), 65 deletions(-) diff --git a/reference-implementation/lib/experimental/thin-byob-byte-stream-reader.js b/reference-implementation/lib/experimental/thin-byob-byte-stream-reader.js index 1ef561281..390c08ed9 100644 --- a/reference-implementation/lib/experimental/thin-byob-byte-stream-reader.js +++ b/reference-implementation/lib/experimental/thin-byob-byte-stream-reader.js @@ -2,14 +2,25 @@ export class ThinByobByteStreamReader { constructor(source) { this._source = source; + // Stores a tuple of the following data for reader.read() calls which were made after the reader is closed or + // errored until the underlying source finishes returning all the buffers given to the underlying source: + // - the Uint8Array provided by the user of the reader on the reader.read() call + // - resolution/rejection function of the promise returned to the user of the reader on the reader.read() call this._pendingReads = []; + // Stores a tuple of the following data for _source.read() calls which are not yet fulfilled: + // - the Uint8Array given to the source + // - resolution/rejection function of the promise returned to the user of the reader on the corresponding + // reader.read() call this._pendingSourceReads = []; + // True when closed but the source may still return the given buffers using done(). this._closed = false; + // True when errored but the source may still return the given buffers using done(). this._errored = false; this._errorReason = undefined; + // True when errored and the source cannot return the given buffers. this._fatalErrored = false; this._closedPromise = new Promise((resolve, reject) => { @@ -34,7 +45,7 @@ export class ThinByobByteStreamReader { } else if (this._errored) { entry.reject({reason: this._errorReason, view: entry.view.subarray(0, 0)}); } else if (this._closed) { - entry.resolve({view: entry.view, done: true}); + entry.resolve({view: entry.view.subarray(0, 0), done: true}); } else if (this._source === undefined) { entry.reject({reason: new TypeError('already released'), view: entry.view.subarray(0, 0)}); } else { @@ -76,7 +87,7 @@ export class ThinByobByteStreamReader { return; } if (this._closed) { - entry.resolve({value: entry.view, done: true}); + entry.resolve({value: entry.view.subarray(0, 0), done: true}); if (this._pendingSourceReads.length === 0) { this._processPendingReads(); } @@ -94,6 +105,14 @@ export class ThinByobByteStreamReader { entry.resolve({value: entry.view.subarray(0, bytesWritten), done: false}); } + _markClosed() { + this._closed = true; + + this._resolveClosedPromise(); + this._resolveClosedPromise = undefined; + this._rejectClosedPromise = undefined; + } + _close() { if (this._fatalErrored) { throw new TypeError('already fatal-errored'); @@ -105,14 +124,10 @@ export class ThinByobByteStreamReader { throw new TypeError('already closed'); } - this._closed = true; - - this._resolveClosedPromise(); - this._resolveClosedPromise = undefined; - this._rejectClosedPromise = undefined; + this._markClosed(); } - _errorInternal(reason) { + _markErrored(reason) { this._errored = true; this._errorReason = reason; @@ -132,7 +147,7 @@ export class ThinByobByteStreamReader { throw new TypeError('already closed'); } - this._errorInternal(reason); + this._markErrored(reason); } _fatalError(reason) { @@ -156,7 +171,7 @@ export class ThinByobByteStreamReader { if (this._errored) { reject({reason: this._errorReason, view: view.subarray(0, 0)}); } else if (this._closed) { - resolve({value: view, done: true}); + resolve({value: view.subarray(0, 0), done: true}); } else { reject({reason: new TypeError('already released'), view: view.subarray(0, 0)}); } @@ -177,7 +192,7 @@ export class ThinByobByteStreamReader { this._fatalError.bind(this)); } catch (e) { if (!(this._fatalErrored || this._errored || this._closed)) { - this._errorInternal(e); + this._markErrored(e); } } }); @@ -196,11 +211,7 @@ export class ThinByobByteStreamReader { return Promise.reject(new TypeError('already closed')); } - this._closed = true; - - this._resolveClosedPromise(); - this._resolveClosedPromise = undefined; - this._rejectClosedPromise = undefined; + this._markClosed(); return new Promise((resolve, reject) => { resolve(this._source.cancel(reason)); diff --git a/reference-implementation/test/experimental/thin-byob-byte-stream-reader.js b/reference-implementation/test/experimental/thin-byob-byte-stream-reader.js index 8eb712bbb..bb9ff255c 100644 --- a/reference-implementation/test/experimental/thin-byob-byte-stream-reader.js +++ b/reference-implementation/test/experimental/thin-byob-byte-stream-reader.js @@ -2,42 +2,29 @@ const test = require('tape-catch'); import { fillArrayBufferView } from '../../lib/experimental/thin-stream-utils'; import { ThinByobByteStreamReader } from '../../lib/experimental/thin-byob-byte-stream-reader'; +import { FakeByteSource } from '../../lib/experimental/fake-byte-source'; test('Construct a ThinByobByteStreamReader', t => { - class Source { - constructor(size) { - this._bytesRemaining = size; - } - - read(view, done, close) { - if (this._bytesRemaining === 0) { - close(); - // Call done() only to return the buffer. - done(); - return; - } - - const bytesToFill = Math.min(this._bytesRemaining, view.byteLength); - fillArrayBufferView(view, 1, bytesToFill); - this._bytesRemaining -= bytesToFill; - done(bytesToFill); - } - - cancel(reason) { - t.fail('cancel() is called'); - t.end(); - } + class UnderlyingSource { } + const underlyingSource = new UnderlyingSource(); + new ThinByobByteStreamReader(underlyingSource); + t.end(); +}); + +test('Read data from a reader', t => { + const source = new FakeByteSource(1024); + const reader = source.getByobReader(); let bytesRead = 0; const buffer = new Uint8Array(16); - const rs = new ThinByobByteStreamReader(new Source(1024)); function pump() { fillArrayBufferView(buffer, 0); - rs.read(buffer).then(({value, done}) => { + reader.read(buffer).then(({value, done}) => { if (done) { + t.equal(value.byteLength, 0); t.equal(bytesRead, 1024); t.end(); return; @@ -63,7 +50,46 @@ test('Construct a ThinByobByteStreamReader', t => { pump(); }); -test('read() on a closed stream', t => { +test('Read data from a reader into a single Uint8Array', t => { + const source = new FakeByteSource(1024, 10); + const reader = source.getByobReader(); + + let bytesRead = 0; + + const buffer = new Uint8Array(2048); + fillArrayBufferView(buffer, 0); + + function pump() { + reader.read(buffer.subarray(bytesRead)).then(({value, done}) => { + if (done) { + t.equal(value.byteLength, 0); + t.equal(bytesRead, 1024); + const buffer = new Uint8Array(value.buffer); + for (let i = 0; i < bytesRead; ++i) { + if (buffer[i] !== 1) { + t.fail('buffer[' + i + '] is ' + buffer[i]); + t.end(); + return; + } + } + t.end(); + return; + } else { + bytesRead += value.byteLength; + pump(); + } + }, e => { + t.fail(e); + t.end(); + }).catch(e => { + t.fail(e); + t.end(); + }); + } + pump(); +}); + +test('read() on a closed reader', t => { class Source { start(close) { close(); @@ -79,22 +105,28 @@ test('read() on a closed stream', t => { t.end(); } } + const reader = new ThinByobByteStreamReader(new Source()); - const rs = new ThinByobByteStreamReader(new Source()); - - rs.read(new Uint8Array(16)).then(({value, done}) => { + reader.read(new Uint8Array(16)).then(({value, done}) => { t.equal(done, true); - t.equal(value.byteLength, 16); - rs.read(new Uint8Array(16)).then(({value, done}) => { + t.equal(value.byteOffset, 0); + t.equal(value.byteLength, 0); + t.equal(value.buffer.byteLength, 16); + + reader.read(new Uint8Array(16)).then(({value, done}) => { t.equal(done, true); - t.equal(value.byteLength, 16); + + t.equal(value.byteOffset, 0); + t.equal(value.byteLength, 0); + t.equal(value.buffer.byteLength, 16); + t.end(); }); }); }); -test('Close a stream with pending read()s', t => { +test('Close a reader with pending read()s', t => { let readCount = 0; let close = undefined; @@ -116,21 +148,28 @@ test('Close a stream with pending read()s', t => { t.end(); } } - - const rs = new ThinByobByteStreamReader(new Source()); + const reader = new ThinByobByteStreamReader(new Source()); let firstReadFulfilled = false; - rs.read(new Uint8Array(16)).then(({value, done}) => { + reader.read(new Uint8Array(16)).then(({value, done}) => { t.equal(done, true); - t.equal(value.byteLength, 16); + + t.equal(value.byteOffset, 0); + t.equal(value.byteLength, 0); + t.equal(value.buffer.byteLength, 16); + firstReadFulfilled = true; }).catch(e => { t.fail(e); t.end(); }); - rs.read(new Uint8Array(16)).then(({value, done}) => { + reader.read(new Uint8Array(16)).then(({value, done}) => { t.equal(done, true); - t.equal(value.byteLength, 16); + + t.equal(value.byteOffset, 0); + t.equal(value.byteLength, 0); + t.equal(value.buffer.byteLength, 16); + t.equal(firstReadFulfilled, true); t.equal(readCount, 2); t.end(); @@ -145,7 +184,7 @@ test('Close a stream with pending read()s', t => { done(); }); -test('read() on a errored stream', t => { +test('read() on a errored reader', t => { const testError = new TypeError('test'); class Source { @@ -163,18 +202,22 @@ test('read() on a errored stream', t => { t.end(); } } + const reader = new ThinByobByteStreamReader(new Source()); - const rs = new ThinByobByteStreamReader(new Source()); - - rs.read(new Uint8Array(16)).catch(({reason, view}) => { + reader.read(new Uint8Array(16)).catch(({reason, view}) => { t.equal(reason, testError); + + t.equal(view.byteOffset, 0); t.equal(view.byteLength, 0); t.equal(view.buffer.byteLength, 16); - rs.read(new Uint8Array(16)).catch(({reason, view}) => { + reader.read(new Uint8Array(16)).catch(({reason, view}) => { t.equal(reason, testError); + + t.equal(view.byteOffset, 0); t.equal(view.byteLength, 0); t.equal(view.buffer.byteLength, 16); + t.end(); }).catch(e => { t.fail(e); @@ -186,7 +229,7 @@ test('read() on a errored stream', t => { }); }); -test('Error a stream with pending read()s', t => { +test('Error a reader with pending read()s', t => { let readCount = 0; let error = undefined; @@ -208,22 +251,27 @@ test('Error a stream with pending read()s', t => { t.end(); } } + const reader = new ThinByobByteStreamReader(new Source()); const testError = new TypeError('test'); - const rs = new ThinByobByteStreamReader(new Source()); - let firstReadRejected = false; - rs.read(new Uint8Array(16)).catch(({reason, view}) => { + reader.read(new Uint8Array(16)).catch(({reason, view}) => { t.equal(reason, testError); + + t.equal(view.byteOffset, 0); t.equal(view.byteLength, 0); t.equal(view.buffer.byteLength, 16); + firstReadRejected = true; }); - rs.read(new Uint8Array(16)).catch(({reason, view}) => { + reader.read(new Uint8Array(16)).catch(({reason, view}) => { t.equal(reason, testError); + + t.equal(view.byteOffset, 0); t.equal(view.byteLength, 0); t.equal(view.buffer.byteLength, 16); + t.equal(firstReadRejected, true); t.equal(readCount, 2); t.end(); @@ -234,3 +282,92 @@ test('Error a stream with pending read()s', t => { done(); done(); }); + +test('read() on a fatal-errored reader', t => { + const testError = new TypeError('test'); + + class Source { + start(close, error, fatalError) { + fatalError(testError); + } + + read() { + t.fail('read() is called'); + t.end(); + } + + cancel(reason) { + t.fail('cancel() is called'); + t.end(); + } + } + const reader = new ThinByobByteStreamReader(new Source()); + + reader.read(new Uint8Array(16)).catch(({reason, view}) => { + t.equal(reason, testError); + + t.equal(view, undefined); + + reader.read(new Uint8Array(16)).catch(({reason, view}) => { + t.equal(reason, testError); + + t.equal(view, undefined); + + t.end(); + }).catch(e => { + t.fail(e); + t.end(); + }); + }).catch(e => { + t.fail(e); + t.end(); + }); +}); + +test('Fatal-error a reader with pending read()s', t => { + let readCount = 0; + + let fatalError = undefined; + let done = undefined; + class Source { + start(close, error, fatalError_) { + fatalError = fatalError_; + } + + read(view, done_) { + t.equal(view.byteLength, 16); + ++readCount; + + done = done_; + } + + cancel(reason) { + t.fail('cancel() is called'); + t.end(); + } + } + const reader = new ThinByobByteStreamReader(new Source()); + + const testError = new TypeError('test'); + + let firstReadRejected = false; + reader.read(new Uint8Array(16)).catch(({reason, view}) => { + t.equal(reason, testError); + + t.equal(view, undefined); + + firstReadRejected = true; + }); + reader.read(new Uint8Array(16)).catch(({reason, view}) => { + t.equal(reason, testError); + + t.equal(view, undefined); + + t.equal(firstReadRejected, true); + t.equal(readCount, 2); + t.end(); + }); + + fatalError(testError); + t.throws(done, /TypeError/); +});