diff --git a/velox/common/caching/CachedFactory.h b/velox/common/caching/CachedFactory.h index 22d266ec6235..0cab4de0c07f 100644 --- a/velox/common/caching/CachedFactory.h +++ b/velox/common/caching/CachedFactory.h @@ -146,6 +146,7 @@ template < typename Value, typename Generator, typename Properties = void, + typename Statistics = void, typename Sizer = DefaultSizer, typename Comparator = std::equal_to, typename Hash = std::hash> @@ -178,7 +179,8 @@ class CachedFactory { /// will probably mess with your memory model, so really try to avoid it. CachedPtr generate( const Key& key, - const Properties* properties = nullptr); + const Properties* properties = nullptr, + Statistics* stats = nullptr); /// Looks up the cache entry of the given key if it exists, otherwise returns /// null. @@ -358,17 +360,25 @@ template < typename Value, typename Generator, typename Properties, + typename Statistics, typename Sizer, typename Comparator, typename Hash> -CachedPtr -CachedFactory:: - generate(const Key& key, const Properties* properties) { +CachedPtr CachedFactory< + Key, + Value, + Generator, + Properties, + Statistics, + Sizer, + Comparator, + Hash>:: + generate(const Key& key, const Properties* properties, Statistics* stats) { process::TraceContext trace("CachedFactory::generate"); if (cache_ == nullptr) { return CachedPtr{ /*fromCache=*/false, - (*generator_)(key, properties).release(), + (*generator_)(key, properties, stats).release(), nullptr, std::make_unique(key)}; } @@ -397,7 +407,7 @@ CachedFactory:: } pendingLock.unlock(); // Regenerates in the edge case. - return generate(key, properties); + return generate(key, properties, stats); } pending_.insert(key); @@ -408,7 +418,7 @@ CachedFactory:: pendingCv_.notify_all(); }; - std::unique_ptr generatedValue = (*generator_)(key, properties); + std::unique_ptr generatedValue = (*generator_)(key, properties, stats); const uint64_t valueSize = Sizer()(*generatedValue); Value* rawValue = generatedValue.release(); const bool inserted = addCache(key, rawValue, valueSize); @@ -433,12 +443,19 @@ template < typename Value, typename Generator, typename Properties, + typename Statistics, typename Sizer, typename Comparator, typename Hash> -CachedPtr -CachedFactory::get( - const Key& key) { +CachedPtr CachedFactory< + Key, + Value, + Generator, + Properties, + Statistics, + Sizer, + Comparator, + Hash>::get(const Key& key) { if (cache_ == nullptr) { return {}; } @@ -460,10 +477,19 @@ template < typename Value, typename Generator, typename Properties, + typename Statistics, typename Sizer, typename Comparator, typename Hash> -void CachedFactory:: +void CachedFactory< + Key, + Value, + Generator, + Properties, + Statistics, + Sizer, + Comparator, + Hash>:: retrieveCached( const std::vector& keys, std::vector>>& diff --git a/velox/common/caching/tests/CachedFactoryTest.cpp b/velox/common/caching/tests/CachedFactoryTest.cpp index e8161a979256..eb5a99b19a29 100644 --- a/velox/common/caching/tests/CachedFactoryTest.cpp +++ b/velox/common/caching/tests/CachedFactoryTest.cpp @@ -30,7 +30,8 @@ namespace { struct DoublerGenerator { std::unique_ptr operator()( const int& value, - const void* properties = nullptr) { + const void* properties = nullptr, + void* statistics = nullptr) { ++generated; return std::make_unique(value * 2); } @@ -40,7 +41,8 @@ struct DoublerGenerator { struct IdentityGenerator { std::unique_ptr operator()( const int& value, - const void* properties = nullptr) { + const void* properties = nullptr, + void* statistics = nullptr) { return std::make_unique(value); } }; @@ -113,7 +115,8 @@ TEST(CachedFactoryTest, basicGeneration) { struct DoublerWithExceptionsGenerator { std::unique_ptr operator()( const int& value, - const void* properties = nullptr) { + const void* properties = nullptr, + void* statistics = nullptr) { if (value == 3) { VELOX_FAIL("3 is bad"); } diff --git a/velox/common/file/FileSystems.h b/velox/common/file/FileSystems.h index 7295077a250e..af9e2b82fb66 100644 --- a/velox/common/file/FileSystems.h +++ b/velox/common/file/FileSystems.h @@ -29,6 +29,9 @@ class ConfigBase; } class ReadFile; class WriteFile; +namespace filesystems::File { +class IoStats; +} } // namespace facebook::velox namespace facebook::velox::filesystems { @@ -69,6 +72,8 @@ struct FileOptions { /// S3. std::optional> properties{ std::nullopt}; + + File::IoStats* stats{nullptr}; }; /// Defines directory options diff --git a/velox/connectors/hive/FileHandle.cpp b/velox/connectors/hive/FileHandle.cpp index 7678fb7a6c35..fb8d64066037 100644 --- a/velox/connectors/hive/FileHandle.cpp +++ b/velox/connectors/hive/FileHandle.cpp @@ -41,7 +41,8 @@ std::string groupName(const std::string& filename) { std::unique_ptr FileHandleGenerator::operator()( const std::string& filename, - const FileProperties* properties) { + const FileProperties* properties, + filesystems::File::IoStats* stats) { // We have seen cases where drivers are stuck when creating file handles. // Adding a trace here to spot this more easily in future. process::TraceContext trace("FileHandleGenerator::operator()"); @@ -51,6 +52,7 @@ std::unique_ptr FileHandleGenerator::operator()( MicrosecondTimer timer(&elapsedTimeUs); fileHandle = std::make_unique(); filesystems::FileOptions options; + options.stats = stats; if (properties) { options.fileSize = properties->fileSize; } diff --git a/velox/connectors/hive/FileHandle.h b/velox/connectors/hive/FileHandle.h index 5db30b1d7f4c..0a053742ca56 100644 --- a/velox/connectors/hive/FileHandle.h +++ b/velox/connectors/hive/FileHandle.h @@ -69,7 +69,8 @@ class FileHandleGenerator { : properties_(std::move(properties)) {} std::unique_ptr operator()( const std::string& filename, - const FileProperties* properties); + const FileProperties* properties, + filesystems::File::IoStats* stats); private: const std::shared_ptr properties_; @@ -80,6 +81,7 @@ using FileHandleFactory = CachedFactory< FileHandle, FileHandleGenerator, FileProperties, + filesystems::File::IoStats, FileHandleSizer>; using FileHandleCachedPtr = CachedPtr; diff --git a/velox/connectors/hive/SplitReader.cpp b/velox/connectors/hive/SplitReader.cpp index 02a11f3eedf7..3fd1abb12474 100644 --- a/velox/connectors/hive/SplitReader.cpp +++ b/velox/connectors/hive/SplitReader.cpp @@ -249,8 +249,8 @@ void SplitReader::createReader() { try { fileHandleCachePtr = fileHandleFactory_->generate( hiveSplit_->filePath, - hiveSplit_->properties.has_value() ? &*hiveSplit_->properties - : nullptr); + hiveSplit_->properties.has_value() ? &*hiveSplit_->properties : nullptr, + fsStats_ ? fsStats_.get() : nullptr); VELOX_CHECK_NOT_NULL(fileHandleCachePtr.get()); } catch (const VeloxRuntimeError& e) { if (e.errorCode() == error_code::kFileNotFound && diff --git a/velox/dwio/common/Throttler.cpp b/velox/dwio/common/Throttler.cpp index 2e1431bdc5c8..a99d9fda9ecd 100644 --- a/velox/dwio/common/Throttler.cpp +++ b/velox/dwio/common/Throttler.cpp @@ -270,7 +270,8 @@ uint64_t Throttler::calculateBackoffDurationAndUpdateThrottleCache( std::unique_ptr Throttler::ThrottleSignalGenerator::operator()( const std::string& /*unused*/, - const void* /*unused*/) { + const void* /*unused*/, + void* /*unused*/) { return std::unique_ptr(new ThrottleSignal{1}); } diff --git a/velox/dwio/common/Throttler.h b/velox/dwio/common/Throttler.h index 0ebf1e088205..05bce4ceb2d9 100644 --- a/velox/dwio/common/Throttler.h +++ b/velox/dwio/common/Throttler.h @@ -175,7 +175,8 @@ class Throttler { std::unique_ptr operator()( const std::string& /*unused*/, - const void* /*unused*/); + const void* /*unused*/, + void* /*unused*/); }; using CachedThrottleSignalPtr = CachedPtr; diff --git a/velox/experimental/wave/common/KernelCache.cpp b/velox/experimental/wave/common/KernelCache.cpp index 66aa1658288f..7f8f6702fd92 100644 --- a/velox/experimental/wave/common/KernelCache.cpp +++ b/velox/experimental/wave/common/KernelCache.cpp @@ -94,9 +94,8 @@ class AsyncCompiledKernel : public CompiledKernel { class KernelGenerator { public: - std::unique_ptr operator()( - const std::string, - const KernelGenFunc* gen) { + std::unique_ptr + operator()(const std::string, const KernelGenFunc* gen, void* /*unused*/) { using ModulePromise = folly::Promise; struct PromiseHolder { ModulePromise promise;