projector: read run names from data provider (#4494)

Summary:
The projector plugin no longer depends on the event multiplexer. It
finds run directories by calling `list_runs` to get run names and
adjoining those to the value of the `--logdir` flag. This still only
works with a local logdir (as was also the case before this patch), but
now it works even when the data provider is not backed by a multiplexer.

The data provider interface does not mandate that run names be valid
relative paths under a logdir, but that does hold for the two data
providers that we care about: the multiplexer provider and the RustBoard
provider. Making the projector plugin truly portable will require much
deeper changes, so we don’t bother with anything more principled here.

This will certainly not work with `--logdir_spec`; the old version may
or may not have worked. The docs for `--logdir_spec` clearly disclaim
that it may cause some features to stop working, so I’m okay with this.

Test Plan:
Tested on a logdir whose `projector_config.pbtxt` is under plugin assets
and one whose config is in the main run directory. You can tell that
it’s properly reading the config because otherwise the sprite sheet
won’t show up. The projector plugin no longer contains any occurrences
of the word `multiplexer`, and it still works under `--load_fast`, which
does not create a multiplexer at all. Test sync also passes.

wchargin-branch: projector-data-provider-run-names

tensorboard/plugins/projector/BUILD

@@ -16,8 +16,10 @@ py_library(
     deps = [
         ":metadata",
         ":protos_all_py_pb2",
+        "//tensorboard:context",
         "//tensorboard:expect_numpy_installed",
         "//tensorboard/backend:http_util",
+        "//tensorboard/backend/event_processing:plugin_asset_util",
         "//tensorboard/compat:tensorflow",
         "//tensorboard/plugins:base_plugin",
         "//tensorboard/util:tb_logging",
@@ -62,6 +64,7 @@ py_test(
         "//tensorboard:expect_numpy_installed",
         "//tensorboard:expect_tensorflow_installed",
         "//tensorboard/backend:application",
+        "//tensorboard/backend/event_processing:data_provider",
         "//tensorboard/backend/event_processing:event_multiplexer",
         "//tensorboard/compat/proto:protos_all_py_pb2",
         "//tensorboard/plugins:base_plugin",
@@ -82,6 +85,7 @@ py_test(
         "//tensorboard:expect_numpy_installed",
         "//tensorboard:expect_tensorflow_installed",
         "//tensorboard/backend:application",
+        "//tensorboard/backend/event_processing:data_provider",
         "//tensorboard/backend/event_processing:event_multiplexer",
         "//tensorboard/compat:no_tensorflow",
         "//tensorboard/compat/proto:protos_all_py_pb2",

tensorboard/plugins/projector/projector_plugin.py

@@ -28,6 +28,8 @@
 from google.protobuf import json_format
 from google.protobuf import text_format
 
+from tensorboard import context
+from tensorboard.backend.event_processing import plugin_asset_util
 from tensorboard.backend.http_util import Respond
 from tensorboard.compat import tf
 from tensorboard.plugins import base_plugin
@@ -239,7 +241,7 @@ def __init__(self, context):
         Args:
           context: A base_plugin.TBContext instance.
         """
-        self.multiplexer = context.multiplexer
+        self.data_provider = context.data_provider
         self.logdir = context.logdir
         self.readers = {}
         self._run_paths = None
@@ -282,12 +284,13 @@ def get_plugin_apps(self):
     def is_active(self):
         """Determines whether this plugin is active.
 
-        This plugin is only active if any run has an embedding.
+        This plugin is only active if any run has an embedding, and only
+        when running against a local log directory.
 
         Returns:
           Whether any run has embedding data to show in the projector.
         """
-        if not self.multiplexer:
+        if not self.data_provider or not self.logdir:
             return False
 
         if self._is_active:
@@ -332,8 +335,13 @@ def _determine_is_active(self):
 
     def _update_configs(self):
         """Updates `self._configs` and `self._run_paths`."""
-        if self.multiplexer:
-            run_paths = dict(self.multiplexer.RunPaths())
+        if self.data_provider and self.logdir:
+            # Create a background context; we may not be in a request.
+            ctx = context.RequestContext()
+            run_paths = {
+                run.run_name: os.path.join(self.logdir, run.run_name)
+                for run in self.data_provider.list_runs(ctx, experiment_id="")
+            }
         else:
             run_paths = {}
         run_paths_changed = run_paths != self._run_paths
@@ -513,18 +521,18 @@ def _get_embedding(self, tensor_name, config):
         return None
 
     def _append_plugin_asset_directories(self, run_path_pairs):
-        for run, assets in self.multiplexer.PluginAssets(
-            metadata.PLUGIN_ASSETS_NAME
-        ).items():
+        extra = []
+        plugin_assets_name = metadata.PLUGIN_ASSETS_NAME
+        for (run, logdir) in run_path_pairs:
+            assets = plugin_asset_util.ListAssets(logdir, plugin_assets_name)
             if metadata.PROJECTOR_FILENAME not in assets:
                 continue
             assets_dir = os.path.join(
-                self._run_paths[run],
-                metadata.PLUGINS_DIR,
-                metadata.PLUGIN_ASSETS_NAME,
+                self._run_paths[run], metadata.PLUGINS_DIR, plugin_assets_name
             )
             assets_path_pair = (run, os.path.abspath(assets_dir))
-            run_path_pairs.append(assets_path_pair)
+            extra.append(assets_path_pair)
+        run_path_pairs.extend(extra)
 
     @wrappers.Request.application
     def _serve_file(self, file_path, request):

tensorboard/plugins/projector/projector_plugin_test.py

@@ -30,6 +30,7 @@
 from google.protobuf import text_format
 
 from tensorboard.backend import application
+from tensorboard.backend.event_processing import data_provider
 from tensorboard.backend.event_processing import (
     plugin_event_multiplexer as event_multiplexer,
 )
@@ -274,10 +275,10 @@ def testPluginIsNotActive(self):
         self.assertEqual(2, mock.call_count)
 
     def _SetupWSGIApp(self):
+        logdir = self.log_dir
         multiplexer = event_multiplexer.EventMultiplexer()
-        context = base_plugin.TBContext(
-            logdir=self.log_dir, multiplexer=multiplexer
-        )
+        provider = data_provider.MultiplexerDataProvider(multiplexer, logdir)
+        context = base_plugin.TBContext(logdir=logdir, data_provider=provider)
         self.plugin = projector_plugin.ProjectorPlugin(context)
         wsgi_app = application.TensorBoardWSGI([self.plugin])
         self.server = werkzeug_test.Client(wsgi_app, wrappers.BaseResponse)