From 065d7cafd1f28396fe162903a4b05f67c3074e09 Mon Sep 17 00:00:00 2001 From: William Chargin Date: Tue, 18 Aug 2020 02:47:32 -0700 Subject: [PATCH 1/7] data-loader-behavior: add fine-grained batching support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: The data loader behavior maintains a fine-grained cache of key-value pairs. This is useful because when the set of requested keys expands, only the new keys need to be fetched. But, until now, the behavior has been hard-coded to fire a separate request for each key-value pair. Clients can have either fine-grained cache invalidation or efficient batched requests, but not both at the same time. This patch enriches the behavior to support just that. In a future change, the scalars dashboard will take advantage of this to batch requests for multiple runs and a single tag. In doing so, we need to shuffle around the API a bit. Instead of asking clients to provide `getDataLoadUrl: (Item) => string` plus a separate function `requestData: (url: string) => Promise` (where “`Item`” and “`Data`” are the keys and values, respectively), clients now provide a single function `requestData` that takes raw `Item`s (now plural), performs the request(s), and returns the data. The function provides a stream of key-value pairs, which is represented in callback style for convenience. (We don’t want to drag Observable into this.) The purpose of this approach, as opposed to a perhaps more natural approach that simply adapts `getDataLoadUrl` to return some kind of request struct with a callback to map a response into key-value pairs, is to accommodate the variety of existing clients. The structures get pretty wild: e.g., `tf-line-chart-data-loader` mixes in the behavior but doesn’t actually provide the needed properties; they’re provided instead by `tf-scalar-card`, but then `tf-hparams-session-group-details` further overrides some of those properties of `tf-scalar-card` with an entirely different backend. It’s a bit wordier for clients, but at least there are fewer moving pieces to keep track of. Test Plan: The scalars, custom scalars, distributions, histograms, and hparams dashboards all work. The fine-grained invalidation on the scalars dashboard works: e.g., set the tag filter to `mnist` and then to `mnist|hparams`, and watch only the hparams demo data load; then, set it to `hparams` and watch the MNIST charts disappear without any repaints to the hparams demo charts. The post-load callback properly causes scalar charts’ domains to adjust. The refresh button in the dashboard UI properly invalidates and re-fetches data. (Make sure to run with `TB_POLYMER3=1` until that’s the default.) wchargin-branch: dlb-batch-finegrained wchargin-source: 14ec8abde36c4563a4922209d361fc5bd16b6061 --- .../tf_backend/canceller.ts | 9 +- .../data-loader-behavior.ts | 147 ++++++++---------- .../tf-custom-scalar-margin-chart-card.ts | 36 ++++- .../tf-custom-scalar-multi-line-chart-card.ts | 37 ++++- .../tf-distribution-loader.ts | 27 +++- .../tf-histogram-loader.ts | 31 +++- .../tf_hparams_session_group_details/BUILD | 1 + .../tf-hparams-session-group-details.ts | 31 +++- .../scalar/polymer3/tf_scalar_dashboard/BUILD | 1 + .../tf_scalar_dashboard/tf-scalar-card.ts | 22 ++- 10 files changed, 222 insertions(+), 120 deletions(-) diff --git a/tensorboard/components_polymer3/tf_backend/canceller.ts b/tensorboard/components_polymer3/tf_backend/canceller.ts index 727d86818f..dc78a9b13b 100644 --- a/tensorboard/components_polymer3/tf_backend/canceller.ts +++ b/tensorboard/components_polymer3/tf_backend/canceller.ts @@ -13,6 +13,11 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +export interface CancelResult { + value: T; + cancelled: boolean; +} + /** * A class that allows marking promises as cancelled. * @@ -45,9 +50,7 @@ export class Canceller { * a `cancelled` argument. This argument will be `false` unless and * until `cancelAll` is invoked after the creation of this task. */ - public cancellable( - f: (result: {value: T; cancelled: boolean}) => U - ): (T) => U { + public cancellable(f: (result: CancelResult) => U): (T) => U { const originalCancellationCount = this.cancellationCount; return (value) => { const cancelled = this.cancellationCount !== originalCancellationCount; diff --git a/tensorboard/components_polymer3/tf_dashboard_common/data-loader-behavior.ts b/tensorboard/components_polymer3/tf_dashboard_common/data-loader-behavior.ts index 6a4d01bde0..cd3547374b 100644 --- a/tensorboard/components_polymer3/tf_dashboard_common/data-loader-behavior.ts +++ b/tensorboard/components_polymer3/tf_dashboard_common/data-loader-behavior.ts @@ -15,7 +15,7 @@ limitations under the License. import {PolymerElement} from '@polymer/polymer'; import * as _ from 'lodash'; -import {Canceller} from '../tf_backend/canceller'; +import {CancelResult, Canceller} from '../tf_backend/canceller'; import {RequestManager} from '../tf_backend/requestManager'; type CacheKey = string; @@ -34,6 +34,23 @@ export interface DataLoaderBehaviorInterface dataToLoad: Item[]; } +// A function that takes a list of items and asynchronously fetches the +// data for those items. As each item loads, it should invoke the +// `onLoad` callback with an `{item, data}` pair to update the cache. +// After all items have finished loading, it should invoke the +// `onFinish` callback. Conceptually, that this function accepts +// `onLoad` and `onFinish` as arguments is as if it returned an +// Observable-style stream of `{item, data}`-pairs, CPS-transformed. +// +// Used in `DataLoaderBehavior.requestData`. +export interface RequestDataCallback { + ( + items: Item[], + onLoad: (kv: {item: Item; data: Data}) => void, + onFinish: () => void + ): void; +} + export function DataLoaderBehavior( superClass: new () => PolymerElement ): new () => DataLoaderBehaviorInterface { @@ -47,16 +64,16 @@ export function DataLoaderBehavior( */ loadKey = ''; - // List of data to be loaded. By default, a datum is passed to + // List of items to be loaded. By default, items are passed to // `requestData` to fetch data. When the request resolves, invokes // `loadDataCallback` with the datum and its response. dataToLoad: Item[] = []; /** - * A function that takes a datum as an input and returns a unique + * A function that takes an item as an input and returns a unique * identifiable string. Used for caching purposes. */ - getDataLoadName = (datum: Item): CacheKey => String(datum); + getDataLoadName = (item: Item): CacheKey => String(item); /** * A function that takes as inputs: @@ -66,28 +83,11 @@ export function DataLoaderBehavior( * This function will be called when a response from a request to that * data URL is successfully received. */ - loadDataCallback!: (component: this, datum: Item, data: Data) => void; - - public requestManager!: RequestManager; - - // A function that takes a datum as argument and makes the HTTP - // request to fetch the data associated with the datum. It should return - // a promise that either fullfills with the data or rejects with an error. - // If the function doesn't bind 'this', then it will reference the element - // that includes this behavior. - // The default implementation calls this.requestManager.request with - // the value returned by this.getDataLoadUrl(datum) (see below). - // The only place getDataLoadUrl() is called is in the default - // implementation of this method. So if you override this method with - // an implementation that doesn't call getDataLoadUrl, it need not be - // provided. - requestData = (datum: Item) => { - return this.requestManager.request(this.getDataLoadUrl(datum)); - }; - - // A function that takes a datum and returns a string URL for fetching - // data. - getDataLoadUrl!: (datum: Item) => string; + loadDataCallback!: (component: this, item: Item, data: Data) => void; + + // Function that actually loads data from the network. See docs on + // `RequestDataCallback` for details. + requestData: RequestDataCallback; dataLoading = false; @@ -192,62 +192,51 @@ export function DataLoaderBehavior( if (result.cancelled) { return; } - // Read-only property have a special setter. this.dataLoading = true; - // Promises return cacheKeys of the data that were fetched. - const promises = this.dataToLoad - .filter((datum) => { - const cacheKey = this.getDataLoadName(datum); - return !this._dataLoadState.has(cacheKey); - }) - .map((datum) => { - const cacheKey = this.getDataLoadName(datum); - this._dataLoadState.set(cacheKey, LoadState.LOADING); - return this.requestData(datum).then( - this._canceller.cancellable((result) => { - // It was resetted. Do not notify of the response. - if (!result.cancelled) { - this._dataLoadState.set(cacheKey, LoadState.LOADED); - this.loadDataCallback(this, datum, result.value as any); - } - return cacheKey; - }) - ); - }); - return Promise.all(promises) - .then( - this._canceller.cancellable((result) => { - // It was resetted. Do not notify of the data load. - if (!result.cancelled) { - const keysFetched = result.value as any; - const fetched = new Set(keysFetched); - const shouldNotify = this.dataToLoad.some((datum) => - fetched.has(this.getDataLoadName(datum)) - ); - if (shouldNotify) { - this.onLoadFinish(); - } - } - const isDataFetchPending = Array.from( - this._dataLoadState.values() - ).some((loadState) => loadState === LoadState.LOADING); - if (!isDataFetchPending) { - // Read-only property have a special setter. - this.dataLoading = false; - } - }), - // TODO(stephanwlee): remove me when we can use - // Promise.prototype.finally instead - () => {} - ) - .then( - this._canceller.cancellable(({cancelled}) => { - if (cancelled) { - return; + const dirtyItems = this.dataToLoad.filter((datum) => { + const cacheKey = this.getDataLoadName(datum); + return !this._dataLoadState.has(cacheKey); + }); + for (const item of dirtyItems) { + const cacheKey = this.getDataLoadName(item); + this._dataLoadState.set(cacheKey, LoadState.LOADING); + } + const onLoad = this._canceller.cancellable( + (result: CancelResult<{item: Item; data: Data}>) => { + if (result.cancelled) { + return; + } + const {item, data} = result.value; + const cacheKey = this.getDataLoadName(item); + this._dataLoadState.set(cacheKey, LoadState.LOADED); + this.loadDataCallback(this, item, data); + } + ); + const onFinish = this._canceller.cancellable( + (result: CancelResult) => { + // Only notify of data load if the load was not cancelled. + if (!result.cancelled) { + const keysFetched = result.value as any; + const fetched = new Set( + dirtyItems.map((item) => this.getDataLoadName(item)) + ); + const shouldNotify = this.dataToLoad.some((datum) => + fetched.has(this.getDataLoadName(datum)) + ); + if (shouldNotify) { + this.onLoadFinish(); } this._loadDataAsync = null; - }) - ); + } + const isDataFetchPending = Array.from( + this._dataLoadState.values() + ).includes(LoadState.LOADING); + if (!isDataFetchPending) { + this.dataLoading = false; + } + } + ); + this.requestData(dirtyItems, onLoad, () => onFinish(undefined)); }) ); } diff --git a/tensorboard/plugins/custom_scalar/polymer3/tf_custom_scalar_dashboard/tf-custom-scalar-margin-chart-card.ts b/tensorboard/plugins/custom_scalar/polymer3/tf_custom_scalar_dashboard/tf-custom-scalar-margin-chart-card.ts index 92003f3e01..4b3037cbeb 100644 --- a/tensorboard/plugins/custom_scalar/polymer3/tf_custom_scalar_dashboard/tf-custom-scalar-margin-chart-card.ts +++ b/tensorboard/plugins/custom_scalar/polymer3/tf_custom_scalar_dashboard/tf-custom-scalar-margin-chart-card.ts @@ -20,14 +20,17 @@ import * as _ from 'lodash'; import {DomRepeat} from '../../../../components_polymer3/polymer/dom-repeat'; import '../../../../components_polymer3/polymer/irons_and_papers'; import {LegacyElementMixin} from '../../../../components_polymer3/polymer/legacy_element_mixin'; +import {RequestManager} from '../../../../components_polymer3/tf_backend/requestManager'; import {getRouter} from '../../../../components_polymer3/tf_backend/router'; import {addParams} from '../../../../components_polymer3/tf_backend/urlPathHelpers'; import '../../../../components_polymer3/tf_card_heading/tf-card-heading'; +import {RequestDataCallback} from '../../../../components_polymer3/tf_dashboard_common/data-loader-behavior'; import {runsColorScale} from '../../../../components_polymer3/tf_color_scale/colorScale'; import '../../../../components_polymer3/tf_line_chart_data_loader/tf-line-chart-data-loader'; import {TfLineChartDataLoader} from '../../../../components_polymer3/tf_line_chart_data_loader/tf-line-chart-data-loader'; import { SYMBOLS_LIST, + ScalarDatum, Y_TOOLTIP_FORMATTER_PRECISION, multiscaleFormatter, relativeAccessor, @@ -57,6 +60,12 @@ interface StepsMismatch { seriesObject: MarginChartSeries; } +type RunItem = string; +type CustomScalarsDatum = { + regex_valid: boolean; + tag_to_events: Record; +}; + export interface TfCustomScalarMarginChartCard extends HTMLElement { reload(): void; } @@ -72,11 +81,11 @@ class _TfCustomScalarMarginChartCard extends LegacyElementMixin(PolymerElement) active="[[active]]" color-scale="[[_colorScale]]" data-series="[[_seriesNames]]" - get-data-load-url="[[_dataUrl]]" fill-area="[[_fillArea]]" ignore-y-outliers="[[ignoreYOutliers]]" load-key="[[_tagFilter]]" data-to-load="[[runs]]" + request-data="[[_requestData]]" log-scale-active="[[_logScaleActive]]" load-data-callback="[[_createProcessDataFunction(marginChartSeries)]]" request-manager="[[requestManager]]" @@ -315,7 +324,7 @@ class _TfCustomScalarMarginChartCard extends LegacyElementMixin(PolymerElement) ignoreYOutliers: boolean; @property({type: Object}) - requestManager: object; + requestManager: RequestManager; @property({type: Boolean}) showDownloadLinks: boolean; @@ -347,12 +356,23 @@ class _TfCustomScalarMarginChartCard extends LegacyElementMixin(PolymerElement) _logScaleActive: boolean; @property({type: Object}) - _dataUrl: (run: string) => string = (run) => { - const tag = this._tagFilter; - return addParams(getRouter().pluginRoute('custom_scalars', '/scalars'), { - tag, - run, - }); + _requestData: RequestDataCallback = ( + items, + onLoad, + onFinish + ) => { + const router = getRouter(); + const baseUrl = router.pluginRoute('custom_scalars', '/scalars'); + Promise.all( + items.map((item) => { + const run = item; + const tag = this._tagFilter; + const url = addParams(baseUrl, {tag, run}); + return this.requestManager + .request(url) + .then((data) => void onLoad({item, data})); + }) + ).finally(() => void onFinish()); }; @property({type: Object}) diff --git a/tensorboard/plugins/custom_scalar/polymer3/tf_custom_scalar_dashboard/tf-custom-scalar-multi-line-chart-card.ts b/tensorboard/plugins/custom_scalar/polymer3/tf_custom_scalar_dashboard/tf-custom-scalar-multi-line-chart-card.ts index be31c297b2..08ee684453 100644 --- a/tensorboard/plugins/custom_scalar/polymer3/tf_custom_scalar_dashboard/tf-custom-scalar-multi-line-chart-card.ts +++ b/tensorboard/plugins/custom_scalar/polymer3/tf_custom_scalar_dashboard/tf-custom-scalar-multi-line-chart-card.ts @@ -25,10 +25,14 @@ import {getRouter} from '../../../../components_polymer3/tf_backend/router'; import '../../../../components_polymer3/tf_backend/tf-backend'; import {addParams} from '../../../../components_polymer3/tf_backend/urlPathHelpers'; import '../../../../components_polymer3/tf_card_heading/tf-card-heading'; +import {RequestDataCallback} from '../../../../components_polymer3/tf_dashboard_common/data-loader-behavior'; import {runsColorScale} from '../../../../components_polymer3/tf_color_scale/colorScale'; import '../../../../components_polymer3/tf_line_chart_data_loader/tf-line-chart-data-loader'; import {TfLineChartDataLoader} from '../../../../components_polymer3/tf_line_chart_data_loader/tf-line-chart-data-loader'; -import {SYMBOLS_LIST} from '../../../../components_polymer3/vz_chart_helpers/vz-chart-helpers'; +import { + SYMBOLS_LIST, + ScalarDatum, +} from '../../../../components_polymer3/vz_chart_helpers/vz-chart-helpers'; import './tf-custom-scalar-card-style'; import { DataSeries, @@ -40,6 +44,12 @@ export interface TfCustomScalarMultiLineChartCard extends HTMLElement { reload(): void; } +type RunItem = string; +type CustomScalarsDatum = { + regex_valid: boolean; + tag_to_events: Record; +}; + @customElement('tf-custom-scalar-multi-line-chart-card') class _TfCustomScalarMultiLineChartCard extends LegacyElementMixin(PolymerElement) @@ -52,10 +62,10 @@ class _TfCustomScalarMultiLineChartCard active="[[active]]" color-scale="[[_colorScale]]" data-series="[[_seriesNames]]" - get-data-load-url="[[_dataUrl]]" ignore-y-outliers="[[ignoreYOutliers]]" load-key="[[_tagFilter]]" data-to-load="[[runs]]" + request-data="[[_requestData]]" log-scale-active="[[_logScaleActive]]" load-data-callback="[[_createProcessDataFunction()]]" request-manager="[[requestManager]]" @@ -235,12 +245,23 @@ class _TfCustomScalarMultiLineChartCard _logScaleActive: boolean; @property({type: Object}) - _dataUrl: (run: string) => string = (run) => { - const tag = this._tagFilter; - return addParams(getRouter().pluginRoute('custom_scalars', '/scalars'), { - tag, - run, - }); + _requestData: RequestDataCallback = ( + items, + onLoad, + onFinish + ) => { + const router = getRouter(); + const baseUrl = router.pluginRoute('custom_scalars', '/scalars'); + Promise.all( + items.map((item) => { + const run = item; + const tag = this._tagFilter; + const url = addParams(baseUrl, {tag, run}); + return this.requestManager + .request(url) + .then((data) => void onLoad({item, data})); + }) + ).finally(() => void onFinish()); }; @property({type: Object}) diff --git a/tensorboard/plugins/distribution/polymer3/tf_distribution_dashboard/tf-distribution-loader.ts b/tensorboard/plugins/distribution/polymer3/tf_distribution_dashboard/tf-distribution-loader.ts index 2e41bbd4a9..98b0d5efe0 100644 --- a/tensorboard/plugins/distribution/polymer3/tf_distribution_dashboard/tf-distribution-loader.ts +++ b/tensorboard/plugins/distribution/polymer3/tf_distribution_dashboard/tf-distribution-loader.ts @@ -24,7 +24,10 @@ import {getRouter} from '../../../../components_polymer3/tf_backend/router'; import {addParams} from '../../../../components_polymer3/tf_backend/urlPathHelpers'; import '../../../../components_polymer3/tf_card_heading/tf-card-heading'; import {runsColorScale} from '../../../../components_polymer3/tf_color_scale/colorScale'; -import {DataLoaderBehavior} from '../../../../components_polymer3/tf_dashboard_common/data-loader-behavior'; +import { + DataLoaderBehavior, + RequestDataCallback, +} from '../../../../components_polymer3/tf_dashboard_common/data-loader-behavior'; import {VzDistributionChart} from '../vz_distribution_chart/vz-distribution-chart'; import '../vz_distribution_chart/vz-distribution-chart'; @@ -32,6 +35,8 @@ export interface TfDistributionLoader extends HTMLElement { reload(): void; } +type RunTagItem = {run: string; tag: string}; + /** tf-distribution-loader loads an individual distribution from the TensorBoard backend, and renders it into a vz-distribution-chart. @@ -117,13 +122,21 @@ class _TfDistributionLoader @property({type: Object}) getDataLoadName = ({run}) => run; - @property({type: Object}) - getDataLoadUrl = ({tag, run}) => { + requestData: RequestDataCallback = ( + items, + onLoad, + onFinish + ) => { const router = getRouter(); - return addParams(router.pluginRoute('distributions', '/distributions'), { - tag, - run, - }); + const baseUrl = router.pluginRoute('distributions', '/distributions'); + Promise.all( + items.map((item) => { + const url = addParams(baseUrl, {tag: item.tag, run: item.run}); + return this.requestManager + .request(url) + .then((data) => void onLoad({item, data})); + }) + ).finally(() => void onFinish()); }; @property({type: Object}) diff --git a/tensorboard/plugins/histogram/polymer3/tf_histogram_dashboard/tf-histogram-loader.ts b/tensorboard/plugins/histogram/polymer3/tf_histogram_dashboard/tf-histogram-loader.ts index 6c0b9731c5..db6a8497aa 100644 --- a/tensorboard/plugins/histogram/polymer3/tf_histogram_dashboard/tf-histogram-loader.ts +++ b/tensorboard/plugins/histogram/polymer3/tf_histogram_dashboard/tf-histogram-loader.ts @@ -19,11 +19,15 @@ import * as _ from 'lodash'; import {LegacyElementMixin} from '../../../../components_polymer3/polymer/legacy_element_mixin'; import '../../../../components_polymer3/polymer/irons_and_papers'; +import {RequestManager} from '../../../../components_polymer3/tf_backend/requestManager'; import {getRouter} from '../../../../components_polymer3/tf_backend/router'; import {addParams} from '../../../../components_polymer3/tf_backend/urlPathHelpers'; import '../../../../components_polymer3/tf_card_heading/tf-card-heading'; import {runsColorScale} from '../../../../components_polymer3/tf_color_scale/colorScale'; -import {DataLoaderBehavior} from '../../../../components_polymer3/tf_dashboard_common/data-loader-behavior'; +import { + DataLoaderBehavior, + RequestDataCallback, +} from '../../../../components_polymer3/tf_dashboard_common/data-loader-behavior'; import '../vz_histogram_timeseries/vz-histogram-timeseries'; import {VzHistogramTimeseries} from '../vz_histogram_timeseries/vz-histogram-timeseries'; import './histogramCore'; @@ -39,9 +43,11 @@ export interface TfHistogramLoader extends HTMLElement { reload(): void; } +type RunTagItem = {run: string; tag: string}; + @customElement('tf-histogram-loader') class _TfHistogramLoader - extends DataLoaderBehavior<{run: string; tag: string}, VzHistogram[]>( + extends DataLoaderBehavior( LegacyElementMixin(PolymerElement) ) implements TfHistogramLoader { @@ -119,12 +125,23 @@ class _TfHistogramLoader getDataLoadName = ({run}: {run: string; tag: string}): string => run; @property({type: Object}) - getDataLoadUrl = ({tag, run}) => { + requestManager: RequestManager; + + requestData: RequestDataCallback = ( + items, + onLoad, + onFinish + ) => { const router = getRouter(); - return addParams(router.pluginRoute('histograms', '/histograms'), { - tag, - run, - }); + const baseUrl = router.pluginRoute('histograms', '/histograms'); + Promise.all( + items.map((item) => { + const url = addParams(baseUrl, {tag: item.tag, run: item.run}); + return this.requestManager + .request(url) + .then((data) => void onLoad({item, data})); + }) + ).finally(() => void onFinish()); }; @property({type: Object}) diff --git a/tensorboard/plugins/hparams/polymer3/tf_hparams_session_group_details/BUILD b/tensorboard/plugins/hparams/polymer3/tf_hparams_session_group_details/BUILD index 60a9044f83..804ea7b936 100644 --- a/tensorboard/plugins/hparams/polymer3/tf_hparams_session_group_details/BUILD +++ b/tensorboard/plugins/hparams/polymer3/tf_hparams_session_group_details/BUILD @@ -16,6 +16,7 @@ tf_ts_library( "//tensorboard/components_polymer3/polymer:legacy_class", "//tensorboard/components_polymer3/tf_backend", "//tensorboard/components_polymer3/tf_color_scale", + "//tensorboard/components_polymer3/tf_dashboard_common", "//tensorboard/components_polymer3/vz_chart_helpers", "//tensorboard/plugins/hparams/polymer3/tf_hparams_utils", "//tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard:tf_scalar_card", diff --git a/tensorboard/plugins/hparams/polymer3/tf_hparams_session_group_details/tf-hparams-session-group-details.ts b/tensorboard/plugins/hparams/polymer3/tf_hparams_session_group_details/tf-hparams-session-group-details.ts index 31f0fa3fc2..c8ed441be1 100644 --- a/tensorboard/plugins/hparams/polymer3/tf_hparams_session_group_details/tf-hparams-session-group-details.ts +++ b/tensorboard/plugins/hparams/polymer3/tf_hparams_session_group_details/tf-hparams-session-group-details.ts @@ -20,6 +20,7 @@ import * as IronResizableBehavior from '@polymer/iron-resizable-behavior'; import {mixinBehaviors} from '../../../../components_polymer3/polymer/legacy_class'; import '../../../../components_polymer3/polymer/irons_and_papers'; import '../../../../components_polymer3/tf_backend/tf-backend'; +import {RequestDataCallback} from '../../../../components_polymer3/tf_dashboard_common/data-loader-behavior'; import * as tf_hparams_utils from '../tf_hparams_utils/tf-hparams-utils'; import * as tf_color_scale from '../../../../components_polymer3/tf_color_scale/palettes'; import * as vz_chart_helpers from '../../../../components_polymer3/vz_chart_helpers/vz-chart-helpers'; @@ -28,6 +29,11 @@ import '../../../scalar/polymer3/tf_scalar_dashboard/tf-scalar-card'; // TODO: add dependency once the Polymer 3 scalar dashboard is migrated. // import '../tf_scalar_dashboard/tf-scalar-card'; +type RunTagItem = { + run: string; + tag: string; +}; + /** * Shows a session group in more detail. Specifically, shows graphs of the * metrics for the session in the group as a function of the training step. @@ -121,17 +127,28 @@ class TfHparamsSessionGroupDetails extends mixinBehaviors( // '_colorScale'. @property({type: Number}) _sessionGroupNameHash: number; + @property({ type: Object, }) - _requestData = ({tag, run}) => { - const request = { - experimentName: this.experimentName, - sessionName: run, - metricName: tag, - }; - return this.backend.listMetricEvals(request); + _requestData: RequestDataCallback< + RunTagItem, + vz_chart_helpers.ScalarDatum[] + > = (items, onLoad, onFinish) => { + Promise.all( + items.map((item) => { + const request = { + experimentName: this.experimentName, + sessionName: item.run, + metricName: item.tag, + }; + return this.backend + .listMetricEvals(request) + .then((data) => void onLoad({item, data})); + }) + ).finally(() => void onFinish()); }; + @property({ type: Object, }) diff --git a/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/BUILD b/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/BUILD index 5e584831e1..2289b584cc 100644 --- a/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/BUILD +++ b/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/BUILD @@ -54,6 +54,7 @@ tf_ts_library( "//tensorboard/components_polymer3/tf_color_scale", "//tensorboard/components_polymer3/tf_dashboard_common", "//tensorboard/components_polymer3/tf_line_chart_data_loader", + "//tensorboard/components_polymer3/vz_chart_helpers", "//tensorboard/components_polymer3/vz_line_chart2", "@npm//@polymer/decorators", "@npm//@polymer/polymer", diff --git a/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/tf-scalar-card.ts b/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/tf-scalar-card.ts index 85748de707..fd3c0c8941 100644 --- a/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/tf-scalar-card.ts +++ b/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/tf-scalar-card.ts @@ -17,14 +17,19 @@ import {PolymerElement, html} from '@polymer/polymer'; import {customElement, property} from '@polymer/decorators'; import '../../../../components_polymer3/polymer/irons_and_papers'; import {RequestManager} from '../../../../components_polymer3/tf_backend/requestManager'; +import {RequestDataCallback} from '../../../../components_polymer3/tf_dashboard_common/data-loader-behavior'; import {getRouter} from '../../../../components_polymer3/tf_backend/router'; +import {addParams} from '../../../../components_polymer3/tf_backend/urlPathHelpers'; import '../../../../components_polymer3/tf_card_heading/tf-card-heading'; import {runsColorScale} from '../../../../components_polymer3/tf_color_scale/colorScale'; import '../../../../components_polymer3/tf_dashboard_common/tf-downloader'; import '../../../../components_polymer3/tf_line_chart_data_loader/tf-line-chart-data-loader'; +import {ScalarDatum} from '../../../../components_polymer3/vz_chart_helpers/vz-chart-helpers'; import '../../../../components_polymer3/vz_line_chart2/vz-line-chart2'; import {DEFAULT_TOOLTIP_COLUMNS} from '../../../../components_polymer3/vz_line_chart2/vz-line-chart2'; +type RunTagItem = {run: string; tag: string}; + /** * A card that handles loading data (at the right times), rendering a scalar * chart, and providing UI affordances (such as buttons) for scalar data. @@ -252,7 +257,22 @@ export class TfScalarCard extends PolymerElement { // this.requestManager.request( // this.getDataLoadUrl({tag, run, experiment}) @property({type: Object}) - requestData: object; + requestData: RequestDataCallback = ( + items, + onLoad, + onFinish + ) => { + const router = getRouter(); + const baseUrl = router.pluginRoute('scalars', '/scalars'); + Promise.all( + items.map((item) => { + const url = addParams(baseUrl, {tag: item.tag, run: item.run}); + return this.requestManager + .request(url) + .then((data) => void onLoad({item, data})); + }) + ).finally(() => void onFinish()); + }; @property({type: Object}) _getDataLoadName: object = (datum) => this._getSeriesNameFromDatum(datum); From beff777e4462ae2706b4beba14310be3ad49062c Mon Sep 17 00:00:00 2001 From: William Chargin Date: Tue, 18 Aug 2020 10:10:41 -0700 Subject: [PATCH 2/7] scalars: multiplex data fetches within a tag Summary: DO NOT SUBMIT until corresponding internal change is submitted. Test Plan: Manually tested; to be documented. wchargin-branch: scalars-mux-runs wchargin-source: 9459c8cc6bc0b041016dfc689306639a04080217 --- .../tf_backend/requestManager.ts | 28 +++--- tensorboard/plugins/scalar/http_api.md | 22 +++++ .../tf_scalar_dashboard/tf-scalar-card.ts | 40 +++++--- tensorboard/plugins/scalar/scalars_plugin.py | 34 +++++++ .../plugins/scalar/scalars_plugin_test.py | 95 +++++++++++++++++++ 5 files changed, 193 insertions(+), 26 deletions(-) diff --git a/tensorboard/components_polymer3/tf_backend/requestManager.ts b/tensorboard/components_polymer3/tf_backend/requestManager.ts index 954c66c362..7aabcca011 100644 --- a/tensorboard/components_polymer3/tf_backend/requestManager.ts +++ b/tensorboard/components_polymer3/tf_backend/requestManager.ts @@ -92,6 +92,12 @@ export class RequestOptions { } } +// Form data for a POST request as a convenient multidict interface. +// A raw string value is equivalent to a singleton array. +export interface PostData { + [key: string]: string | string[]; +} + export class RequestManager { private _queue: ResolveReject[]; private _maxRetries: number; @@ -108,12 +114,7 @@ export class RequestManager { * postData is provided, this request will use POST, not GET. This is an * object mapping POST keys to string values. */ - public request( - url: string, - postData?: { - [key: string]: string; - } - ): Promise { + public request(url: string, postData?: PostData): Promise { const requestOptions = requestOptionsFromPostData(postData); return this.requestWithOptions(url, requestOptions); } @@ -272,9 +273,7 @@ function buildXMLHttpRequest( return req; } -function requestOptionsFromPostData(postData?: { - [key: string]: string; -}): RequestOptions { +function requestOptionsFromPostData(postData?: PostData): RequestOptions { const result = new RequestOptions(); if (!postData) { result.methodType = HttpMethodType.GET; @@ -285,13 +284,12 @@ function requestOptionsFromPostData(postData?: { return result; } -function formDataFromDictionary(postData: {[key: string]: string}) { +function formDataFromDictionary(postData: PostData) { const formData = new FormData(); - for (let postKey in postData) { - if (postKey) { - // The linter requires 'for in' loops to be filtered by an if - // condition. - formData.append(postKey, postData[postKey]); + for (const [key, maybeValues] of Object.entries(postData)) { + const values = Array.isArray(maybeValues) ? maybeValues : [maybeValues]; + for (const value of values) { + formData.append(key, value); } } return formData; diff --git a/tensorboard/plugins/scalar/http_api.md b/tensorboard/plugins/scalar/http_api.md index 394b57d7a1..fece3ebb11 100644 --- a/tensorboard/plugins/scalar/http_api.md +++ b/tensorboard/plugins/scalar/http_api.md @@ -61,3 +61,25 @@ instead be in CSV format: 1443856985.705543,1448,0.7461960315704346 1443857105.704628,3438,0.5427092909812927 1443857225.705133,5417,0.5457325577735901 + +## `/data/plugin/scalars/scalars_multirun` (POST) + +Accepts form-encoded POST data with a (required) singleton key `tag` and a +repeated key `runs`. Returns a JSON object mapping run names to arrays of the +form returned by `/data/plugin/scalars/scalars`. A run will only be present in +the output if there actually exists data for that run-tag combination. If there +is no data for some or all of the run-tag combinations, no error is raised, but +the response may lack runs requested in the input or be an empty object +entirely. + +Example: + + { + "train": [ + [1443856985.705543, 1448, 0.7461960315704346], + [1443857105.704628, 3438, 0.5427092909812927] + ], + "test": [ + [1443857225.705133, 5417, 0.5457325577735901], + ] + } diff --git a/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/tf-scalar-card.ts b/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/tf-scalar-card.ts index fd3c0c8941..2920160c57 100644 --- a/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/tf-scalar-card.ts +++ b/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/tf-scalar-card.ts @@ -225,14 +225,18 @@ export class TfScalarCard extends PolymerElement { // This function is called when data is received from the backend. @property({type: Object}) - _loadDataCallback: object = (scalarChart, datum, data) => { - const formattedData = data.map((datum) => ({ + _loadDataCallback: object = (scalarChart, item, maybeData) => { + if (maybeData == null) { + console.error('Failed to load data for:', item); + return; + } + const formattedData = maybeData.map((datum) => ({ wall_time: new Date(datum[0] * 1000), step: datum[1], scalar: datum[2], })); - const name = this._getSeriesNameFromDatum(datum); - scalarChart.setSeriesMetadata(name, datum); + const name = this._getSeriesNameFromDatum(item); + scalarChart.setSeriesMetadata(name, item); scalarChart.setSeriesData(name, formattedData); scalarChart.commitChanges(); }; @@ -257,19 +261,33 @@ export class TfScalarCard extends PolymerElement { // this.requestManager.request( // this.getDataLoadUrl({tag, run, experiment}) @property({type: Object}) - requestData: RequestDataCallback = ( + requestData: RequestDataCallback = ( items, onLoad, onFinish ) => { const router = getRouter(); - const baseUrl = router.pluginRoute('scalars', '/scalars'); + const url = router.pluginRoute('scalars', '/scalars_multirun'); + const runsByTag = new Map(); + for (const {tag, run} of items) { + let runs = runsByTag.get(tag); + if (runs == null) { + runsByTag.set(tag, (runs = [])); + } + runs.push(run); + } Promise.all( - items.map((item) => { - const url = addParams(baseUrl, {tag: item.tag, run: item.run}); - return this.requestManager - .request(url) - .then((data) => void onLoad({item, data})); + Array.from(runsByTag.entries()).map(([tag, runs]) => { + return this.requestManager.request(url, {tag, runs}).then((allData) => { + for (const run of runs) { + const item = {tag, run}; + if (Object.prototype.hasOwnProperty.call(allData, run)) { + onLoad({item, data: allData[run]}); + } else { + onLoad({item, data: null}); + } + } + }); }) ).finally(() => void onFinish()); }; diff --git a/tensorboard/plugins/scalar/scalars_plugin.py b/tensorboard/plugins/scalar/scalars_plugin.py index bda455cc1c..a27a85cc60 100644 --- a/tensorboard/plugins/scalar/scalars_plugin.py +++ b/tensorboard/plugins/scalar/scalars_plugin.py @@ -26,6 +26,7 @@ import six from six import StringIO +import werkzeug.exceptions from werkzeug import wrappers from tensorboard import errors @@ -64,6 +65,7 @@ def __init__(self, context): def get_plugin_apps(self): return { "/scalars": self.scalars_route, + "/scalars_multirun": self.scalars_multirun_route, "/tags": self.tags_route, } @@ -115,6 +117,21 @@ def scalars_impl(self, ctx, tag, run, experiment, output_format): else: return (values, "application/json") + def scalars_multirun_impl(self, ctx, tag, runs, experiment): + """Result of the form `(body, mime_type)`.""" + all_scalars = self._data_provider.read_scalars( + ctx, + experiment_id=experiment, + plugin_name=metadata.PLUGIN_NAME, + downsample=self._downsample_to, + run_tag_filter=provider.RunTagFilter(runs=runs, tags=[tag]), + ) + body = { + run: [(x.wall_time, x.step, x.value) for x in run_data[tag]] + for (run, run_data) in all_scalars.items() + } + return (body, "application/json") + @wrappers.Request.application def tags_route(self, request): ctx = plugin_util.context(request.environ) @@ -140,3 +157,20 @@ def scalars_route(self, request): ctx, tag, run, experiment, output_format ) return http_util.Respond(request, body, mime_type) + + @wrappers.Request.application + def scalars_multirun_route(self, request): + """Given a tag and list of runs, return dict of ScalarEvent arrays.""" + if request.method != "POST": + raise werkzeug.exceptions.MethodNotAllowed(["POST"]) + tag = request.form.get("tag") + runs = request.form.getlist("runs") + if tag is None: + raise errors.InvalidArgumentError("tag must be specified") + + ctx = plugin_util.context(request.environ) + experiment = plugin_util.experiment_id(request.environ) + (body, mime_type) = self.scalars_multirun_impl( + ctx, tag, runs, experiment + ) + return http_util.Respond(request, body, mime_type) diff --git a/tensorboard/plugins/scalar/scalars_plugin_test.py b/tensorboard/plugins/scalar/scalars_plugin_test.py index 28f9e8ad01..56d23a06e1 100644 --- a/tensorboard/plugins/scalar/scalars_plugin_test.py +++ b/tensorboard/plugins/scalar/scalars_plugin_test.py @@ -58,6 +58,8 @@ class ScalarsPluginTest(tf.test.TestCase): _RUN_WITH_LEGACY_SCALARS = "_RUN_WITH_LEGACY_SCALARS" _RUN_WITH_SCALARS = "_RUN_WITH_SCALARS" + _RUN_WITH_SCALARS_2 = "_RUN_WITH_SCALARS_2" + _RUN_WITH_SCALARS_3 = "_RUN_WITH_SCALARS_3" _RUN_WITH_HISTOGRAM = "_RUN_WITH_HISTOGRAM" def load_plugin(self, run_names): @@ -99,6 +101,20 @@ def generate_run(self, logdir, run_name): display_name=self._DISPLAY_NAME, description=self._DESCRIPTION, ).numpy() + elif run_name == self._RUN_WITH_SCALARS_2: + summ = summary.op( + self._SCALAR_TAG, + 2 * tf.reduce_sum(data), + display_name=self._DISPLAY_NAME, + description=self._DESCRIPTION, + ).numpy() + elif run_name == self._RUN_WITH_SCALARS_3: + summ = summary.op( + self._SCALAR_TAG, + 3 * tf.reduce_sum(data), + display_name=self._DISPLAY_NAME, + description=self._DESCRIPTION, + ).numpy() elif run_name == self._RUN_WITH_HISTOGRAM: summ = tf.compat.v1.summary.histogram( self._HISTOGRAM_TAG, data @@ -191,6 +207,85 @@ def test_scalars_with_histogram(self): ) self.assertEqual(404, response.status_code) + def test_scalars_multirun(self): + server = self.load_server( + [ + self._RUN_WITH_SCALARS, + self._RUN_WITH_SCALARS_2, + self._RUN_WITH_SCALARS_3, + ] + ) + response = server.post( + "/data/plugin/scalars/scalars_multirun", + data={ + "tag": "%s/scalar_summary" % self._SCALAR_TAG, + "runs": [ + self._RUN_WITH_SCALARS, + # skip _RUN_WITH_SCALARS_2 + self._RUN_WITH_SCALARS_3, + self._RUN_WITH_HISTOGRAM, # no data for this tag; okay + "nonexistent_run", # no data at all; okay + ], + }, + ) + self.assertEqual(200, response.status_code) + self.assertEqual("application/json", response.headers["Content-Type"]) + data = json.loads(response.get_data()) + self.assertCountEqual( + [self._RUN_WITH_SCALARS, self._RUN_WITH_SCALARS_3], data + ) + self.assertLen(data[self._RUN_WITH_SCALARS], self._STEPS) + self.assertLen(data[self._RUN_WITH_SCALARS_3], self._STEPS) + self.assertNotEqual( + data[self._RUN_WITH_SCALARS][0][2], + data[self._RUN_WITH_SCALARS_3][0][2], + ) + + def test_scalars_multirun_single_run(self): + # Checks for any problems with singleton arrays. + server = self.load_server( + [ + self._RUN_WITH_SCALARS, + self._RUN_WITH_SCALARS_2, + self._RUN_WITH_SCALARS_3, + ] + ) + response = server.post( + "/data/plugin/scalars/scalars_multirun", + data={ + "tag": "%s/scalar_summary" % self._SCALAR_TAG, + "runs": [self._RUN_WITH_SCALARS], + }, + ) + self.assertEqual(200, response.status_code) + self.assertEqual("application/json", response.headers["Content-Type"]) + data = json.loads(response.get_data()) + self.assertCountEqual([self._RUN_WITH_SCALARS], data) + self.assertLen(data[self._RUN_WITH_SCALARS], self._STEPS) + + def test_scalars_multirun_no_tag(self): + server = self.load_server([self._RUN_WITH_SCALARS]) + response = server.post( + "/data/plugin/scalars/scalars_multirun", + data={"runs": [self._RUN_WITH_SCALARS, self._RUN_WITH_SCALARS_2]}, + ) + self.assertEqual(400, response.status_code) + self.assertIn( + "tag must be specified", response.get_data().decode("utf-8") + ) + + def test_scalars_multirun_bad_method(self): + server = self.load_server([self._RUN_WITH_SCALARS]) + response = server.get( + "/data/plugin/scalars/scalars_multirun", + query_string={ + "tag": "%s/scalar_summary" % self._SCALAR_TAG, + "runs": [self._RUN_WITH_SCALARS, self._RUN_WITH_SCALARS_3,], + }, + ) + self.assertEqual(405, response.status_code) + self.assertEqual(response.headers["Allow"], "POST") + def test_active_with_legacy_scalars(self): plugin = self.load_plugin([self._RUN_WITH_LEGACY_SCALARS]) self.assertFalse(plugin.is_active()) From 50c7112e57f12e8564a7dc4aa8d8770e4fadd684 Mon Sep 17 00:00:00 2001 From: William Chargin Date: Wed, 19 Aug 2020 10:18:54 -0700 Subject: [PATCH 3/7] [update patch] wchargin-branch: scalars-mux-runs wchargin-source: 2e1bd5c8cd3393ac0a79c64b0c87ea43bf346b8a --- .../polymer3/tf_scalar_dashboard/tf-scalar-card.ts | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/tf-scalar-card.ts b/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/tf-scalar-card.ts index da0bc8ac88..2920160c57 100644 --- a/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/tf-scalar-card.ts +++ b/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/tf-scalar-card.ts @@ -261,17 +261,12 @@ export class TfScalarCard extends PolymerElement { // this.requestManager.request( // this.getDataLoadUrl({tag, run, experiment}) @property({type: Object}) -<<<<<<< HEAD requestData: RequestDataCallback = ( -======= - requestData: RequestDataCallback = ( ->>>>>>> 9d315620295e799c8fa98d603b5f3db81e6d04cd items, onLoad, onFinish ) => { const router = getRouter(); -<<<<<<< HEAD const url = router.pluginRoute('scalars', '/scalars_multirun'); const runsByTag = new Map(); for (const {tag, run} of items) { @@ -293,15 +288,6 @@ export class TfScalarCard extends PolymerElement { } } }); -======= - const baseUrl = router.pluginRoute('scalars', '/scalars'); - Promise.all( - items.map((item) => { - const url = addParams(baseUrl, {tag: item.tag, run: item.run}); - return this.requestManager - .request(url) - .then((data) => void onLoad({item, data})); ->>>>>>> 9d315620295e799c8fa98d603b5f3db81e6d04cd }) ).finally(() => void onFinish()); }; From ef6d05039e86a37e13668529e747159facb69336 Mon Sep 17 00:00:00 2001 From: William Chargin Date: Wed, 19 Aug 2020 10:19:00 -0700 Subject: [PATCH 4/7] scalars: commit chart changes in batch Summary: In #3524, we defined a batch API for chart updates to avoid useless paints that would be immediately overwritten. We now take further advantage of this API to only paint charts once all data has been loaded. Test Plan: On the hparams demo directory, this brings paint time (post-network) for four charts rendering 50 trials each down from about 8 seconds to under 1 second. wchargin-branch: scalars-batch-commit wchargin-source: b0babe18b592f577803eebe6f4102c032fc9246f --- .../tf_line_chart_data_loader/tf-line-chart-data-loader.ts | 1 + .../scalar/polymer3/tf_scalar_dashboard/tf-scalar-card.ts | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorboard/components_polymer3/tf_line_chart_data_loader/tf-line-chart-data-loader.ts b/tensorboard/components_polymer3/tf_line_chart_data_loader/tf-line-chart-data-loader.ts index 449593da72..9941e53bd6 100644 --- a/tensorboard/components_polymer3/tf_line_chart_data_loader/tf-line-chart-data-loader.ts +++ b/tensorboard/components_polymer3/tf_line_chart_data_loader/tf-line-chart-data-loader.ts @@ -207,6 +207,7 @@ class _TfLineChartDataLoader _maybeRenderedInBadState: boolean = false; onLoadFinish() { + this.commitChanges(); if (this.dataToLoad.length > 0 && this._resetDomainOnNextLoad) { // (Don't unset _resetDomainOnNextLoad when we didn't // load any runs: this has the effect that if all our diff --git a/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/tf-scalar-card.ts b/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/tf-scalar-card.ts index 2920160c57..2433bf37e8 100644 --- a/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/tf-scalar-card.ts +++ b/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/tf-scalar-card.ts @@ -238,7 +238,6 @@ export class TfScalarCard extends PolymerElement { const name = this._getSeriesNameFromDatum(item); scalarChart.setSeriesMetadata(name, item); scalarChart.setSeriesData(name, formattedData); - scalarChart.commitChanges(); }; @property({type: Object}) From a80c564b02fb56566dfc0405d5935d0c3c945943 Mon Sep 17 00:00:00 2001 From: William Chargin Date: Wed, 19 Aug 2020 10:52:16 -0700 Subject: [PATCH 5/7] [update patch] wchargin-branch: scalars-mux-runs wchargin-source: 0484e726315d46c282c3d61994ffb9e4dc07c20b --- .../tf_backend/requestManager.ts | 7 ++++-- .../tf_scalar_dashboard/tf-scalar-card.ts | 25 ++++++++++++++++++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/tensorboard/components_polymer3/tf_backend/requestManager.ts b/tensorboard/components_polymer3/tf_backend/requestManager.ts index 7aabcca011..ee2cb36a78 100644 --- a/tensorboard/components_polymer3/tf_backend/requestManager.ts +++ b/tensorboard/components_polymer3/tf_backend/requestManager.ts @@ -92,8 +92,11 @@ export class RequestOptions { } } -// Form data for a POST request as a convenient multidict interface. -// A raw string value is equivalent to a singleton array. +// Form data for a POST request as a convenient multidict interface, +// since the built-in `FormData` type doesn't have a value constructor. +// +// A raw string value is equivalent to a singleton array, and thus an +// empty array value is equivalent to omitting the key entirely. export interface PostData { [key: string]: string | string[]; } diff --git a/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/tf-scalar-card.ts b/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/tf-scalar-card.ts index 2920160c57..8cdabc254a 100644 --- a/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/tf-scalar-card.ts +++ b/tensorboard/plugins/scalar/polymer3/tf_scalar_dashboard/tf-scalar-card.ts @@ -276,8 +276,31 @@ export class TfScalarCard extends PolymerElement { } runs.push(run); } + + // Request at most this many runs at once. + // + // Back-of-the-envelope math: each scalar datum JSON value contains + // two floats and a small-ish integer. Floats are about 18 bytes, + // since f64s have -log_10(2^-53) ~= 16 digits of precision plus + // decimal point and leading zero. Small-ish integers (steps) are + // about 5 bytes. Add JSON overhead `[,,],` and you're looking at + // about 48 bytes per datum. With standard downsampling of + // 1000 points per time series, expect ~50 KB of response payload + // per requested time series. + // + // Requesting 64 time series warrants a ~3 MB response, which seems + // reasonable. + const BATCH_SIZE = 64; + + const requestGroups = []; + for (const [tag, runs] of runsByTag) { + for (let i = 0; i < runs.length; i += BATCH_SIZE) { + requestGroups.push({tag, runs: runs.slice(i, i + BATCH_SIZE)}); + } + } + Promise.all( - Array.from(runsByTag.entries()).map(([tag, runs]) => { + requestGroups.map(({tag, runs}) => { return this.requestManager.request(url, {tag, runs}).then((allData) => { for (const run of runs) { const item = {tag, run}; From aaa17f918a64ad68b2af5f11c4441e6bdcdfe059 Mon Sep 17 00:00:00 2001 From: William Chargin Date: Wed, 19 Aug 2020 12:00:47 -0700 Subject: [PATCH 6/7] [bump ci] wchargin-branch: scalars-mux-runs wchargin-source: f183c2962c3d464b6ac903b63d833f0c926fe162 From 534702f4468060a3971b4d4240707081a18c5c26 Mon Sep 17 00:00:00 2001 From: William Chargin Date: Thu, 20 Aug 2020 13:58:23 -0700 Subject: [PATCH 7/7] [update patch] wchargin-branch: scalars-mux-runs wchargin-source: a4ca83ea5bf6b665052c076df835592105e5eefb --- tensorboard/plugins/scalar/http_api.md | 38 +++++++++++++------ tensorboard/plugins/scalar/scalars_plugin.py | 9 +++-- .../plugins/scalar/scalars_plugin_test.py | 23 +++++++++++ 3 files changed, 56 insertions(+), 14 deletions(-) diff --git a/tensorboard/plugins/scalar/http_api.md b/tensorboard/plugins/scalar/http_api.md index fece3ebb11..59f959cc65 100644 --- a/tensorboard/plugins/scalar/http_api.md +++ b/tensorboard/plugins/scalar/http_api.md @@ -72,14 +72,30 @@ is no data for some or all of the run-tag combinations, no error is raised, but the response may lack runs requested in the input or be an empty object entirely. -Example: - - { - "train": [ - [1443856985.705543, 1448, 0.7461960315704346], - [1443857105.704628, 3438, 0.5427092909812927] - ], - "test": [ - [1443857225.705133, 5417, 0.5457325577735901], - ] - } +Example request: + +```javascript +const formData = new FormData(); +formData.set("tag", "xent/xent_1"); +formData.append("runs", "mnist/lr_1E-03,conv=1,fc=2"); +formData.append("runs", "mnist/lr_1E-03,conv=2,fc=2"); +const response = await fetch( + "/data/plugin/scalars/scalars_multirun", + {method: "POST", body: formData} +); +``` + +Example response: + +```json +{ + "mnist/lr_1E-03,conv=1,fc=2": [ + [1563406328.158425, 0, 3.8424863815307617], + [1563406328.5136807, 5, 5.210817337036133] + ], + "mnist/lr_1E-03,conv=2,fc=2": [ + [1563406405.8505669, 0, 11.278410911560059], + [1563406406.357564, 5, 7.649646759033203] + ] +} +``` diff --git a/tensorboard/plugins/scalar/scalars_plugin.py b/tensorboard/plugins/scalar/scalars_plugin.py index a27a85cc60..6c66d0e6d1 100644 --- a/tensorboard/plugins/scalar/scalars_plugin.py +++ b/tensorboard/plugins/scalar/scalars_plugin.py @@ -163,10 +163,13 @@ def scalars_multirun_route(self, request): """Given a tag and list of runs, return dict of ScalarEvent arrays.""" if request.method != "POST": raise werkzeug.exceptions.MethodNotAllowed(["POST"]) - tag = request.form.get("tag") + tags = request.form.getlist("tag") runs = request.form.getlist("runs") - if tag is None: - raise errors.InvalidArgumentError("tag must be specified") + if len(tags) != 1: + raise errors.InvalidArgumentError( + "tag must be specified exactly once" + ) + tag = tags[0] ctx = plugin_util.context(request.environ) experiment = plugin_util.experiment_id(request.environ) diff --git a/tensorboard/plugins/scalar/scalars_plugin_test.py b/tensorboard/plugins/scalar/scalars_plugin_test.py index 56d23a06e1..88bb876240 100644 --- a/tensorboard/plugins/scalar/scalars_plugin_test.py +++ b/tensorboard/plugins/scalar/scalars_plugin_test.py @@ -263,6 +263,17 @@ def test_scalars_multirun_single_run(self): self.assertCountEqual([self._RUN_WITH_SCALARS], data) self.assertLen(data[self._RUN_WITH_SCALARS], self._STEPS) + def test_scalars_multirun_no_runs(self): + server = self.load_server([self._RUN_WITH_SCALARS]) + response = server.post( + "/data/plugin/scalars/scalars_multirun", + data={"tag": "%s/scalar_summary" % self._SCALAR_TAG}, + ) + self.assertEqual(200, response.status_code) + self.assertEqual("application/json", response.headers["Content-Type"]) + data = json.loads(response.get_data()) + self.assertEqual({}, data) + def test_scalars_multirun_no_tag(self): server = self.load_server([self._RUN_WITH_SCALARS]) response = server.post( @@ -274,6 +285,18 @@ def test_scalars_multirun_no_tag(self): "tag must be specified", response.get_data().decode("utf-8") ) + def test_scalars_multirun_two_tags(self): + server = self.load_server([self._RUN_WITH_SCALARS]) + response = server.post( + "/data/plugin/scalars/scalars_multirun", + data={ + "tag": ["accuracy", "loss"], + "runs": [self._RUN_WITH_SCALARS, self._RUN_WITH_SCALARS_2], + }, + ) + self.assertEqual(400, response.status_code) + self.assertIn("exactly once", response.get_data().decode("utf-8")) + def test_scalars_multirun_bad_method(self): server = self.load_server([self._RUN_WITH_SCALARS]) response = server.get(