Roll back arbitrary transfer cache.

TCMalloc Team · copybara-github · commit 3e01aa28a8d3 · 2020-07-03T10:34:54.000-07:00
PiperOrigin-RevId: 319531352
Change-Id: I1695dd1be9b98d4eb7ed1028f79e989e8cb2a6bd
diff --git a/tcmalloc/experiment_config.h b/tcmalloc/experiment_config.h
@@ -23,6 +23,7 @@ namespace tcmalloc {
 enum class Experiment : int {
   TCMALLOC_TEMERAIRE,
   TCMALLOC_SANS_56_SIZECLASS,
+  TCMALLOC_ARBITRARY_TRANSFER_CACHE,
   TCMALLOC_LARGE_NUM_TO_MOVE,
   TCMALLOC_4K_SIZE_CLASS,
   kMaxExperimentID,
@@ -37,6 +38,7 @@ struct ExperimentConfig {
 inline constexpr ExperimentConfig experiments[] = {
     {Experiment::TCMALLOC_TEMERAIRE, "TCMALLOC_TEMERAIRE"},
     {Experiment::TCMALLOC_SANS_56_SIZECLASS, "TCMALLOC_SANS_56_SIZECLASS"},
+    {Experiment::TCMALLOC_ARBITRARY_TRANSFER_CACHE, "TCMALLOC_ARBITRARY_TRANSFER_CACHE"},
     {Experiment::TCMALLOC_LARGE_NUM_TO_MOVE, "TCMALLOC_LARGE_NUM_TO_MOVE"},
     {Experiment::TCMALLOC_4K_SIZE_CLASS, "TCMALLOC_4K_SIZE_CLASS"},
 };
diff --git a/tcmalloc/transfer_cache.cc b/tcmalloc/transfer_cache.cc
@@ -68,6 +68,10 @@ void TransferCache::Init(size_t cl) {
   // We need at least 2 slots to store list head and tail.
   ASSERT(kMinObjectsToMove >= 2);
 
+  // Cache this value, for performance.
+  arbitrary_transfer_ =
+      IsExperimentActive(Experiment::TCMALLOC_ARBITRARY_TRANSFER_CACHE);
+
   slots_ = nullptr;
   max_capacity_ = 0;
   SizeInfo info = {0, 0};
@@ -140,6 +144,7 @@ bool TransferCache::ShrinkCache() {
     absl::base_internal::SpinLockHolder h(&lock_);
     auto info = slot_info_.load(std::memory_order_relaxed);
     if (info.capacity == 0) return false;
+    if (!arbitrary_transfer_ && info.capacity < N) return false;
 
     N = std::min(N, info.capacity);
     int unused = info.capacity - info.used;
@@ -181,7 +186,7 @@ void TransferCache::InsertRange(absl::Span<void *> batch, int N) {
       tracking::Report(kTCInsertHit, size_class(), 1);
       return;
     }
-  } else {
+  } else if (arbitrary_transfer_) {
     absl::base_internal::SpinLockHolder h(&lock_);
     MakeCacheSpace(N);
     // MakeCacheSpace can drop the lock, so refetch
@@ -220,7 +225,6 @@ void TransferCache::InsertRange(absl::Span<void *> batch, int N) {
       }
 #endif
     }
-    // We don't need to hold the lock here, so release it earlier.
   }
   tracking::Report(kTCInsertMiss, size_class(), 1);
   freelist().InsertRange(batch.data(), N);
@@ -243,7 +247,7 @@ int TransferCache::RemoveRange(void **batch, int N) {
       tracking::Report(kTCRemoveHit, size_class(), 1);
       return N;
     }
-  } else if (info.used >= 0) {
+  } else if (arbitrary_transfer_ && info.used >= 0) {
     absl::base_internal::SpinLockHolder h(&lock_);
     // Refetch with the lock
     info = slot_info_.load(std::memory_order_relaxed);
@@ -255,7 +259,6 @@ int TransferCache::RemoveRange(void **batch, int N) {
     memcpy(batch, entry, sizeof(void *) * fetch);
     tracking::Report(kTCRemoveHit, size_class(), 1);
     if (fetch == N) return N;
-    // We don't need to hold the lock here, so release it earlier.
   }
   tracking::Report(kTCRemoveMiss, size_class(), 1);
   return freelist().RemoveRange(batch + fetch, N - fetch) + fetch;
diff --git a/tcmalloc/transfer_cache.h b/tcmalloc/transfer_cache.h
@@ -47,7 +47,8 @@ class TransferCache {
         max_capacity_(0),
         slot_info_{},
         slots_(nullptr),
-        freelist_do_not_access_directly_() {}
+        freelist_do_not_access_directly_(),
+        arbitrary_transfer_(false) {}
   TransferCache(const TransferCache &) = delete;
   TransferCache &operator=(const TransferCache &) = delete;
 
@@ -131,6 +132,9 @@ class TransferCache {
   }
 
   CentralFreeList freelist_do_not_access_directly_;
+
+  // Cached value of IsExperimentActive(Experiment::TCMALLOC_ARBITRARY_TRANSFER)
+  bool arbitrary_transfer_;
 } ABSL_CACHELINE_ALIGNED;
 
 #else
