From bef9e16675806467f216546c0f16f80ddf909bd2 Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Tue, 10 Nov 2020 19:56:33 +1100 Subject: [PATCH] Disable stacksampler by default It causes problems due to the sheer number of unique metrics emitted, which makes the prometheus endpoint be very expensive / fail a lot. The data is not useful enough to justify the cost. --- common/common/stats.py | 5 +++-- docker-compose.jsonnet | 4 ++-- k8s.jsonnet | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/common/common/stats.py b/common/common/stats.py index 86e7b39..40f783a 100644 --- a/common/common/stats.py +++ b/common/common/stats.py @@ -194,10 +194,11 @@ def install_stacksampler(interval=0.005): We could use user+sys time but that leads to interrupting syscalls, which may affect performance, and we care mostly about user time anyway. """ - if os.environ.get('WUBLOADER_DISABLE_STACKSAMPLER', '').lower() == 'true': - logging.info("Not installing stacksampler - disabled by WUBLOADER_DISABLE_STACKSAMPLER env var") + if os.environ.get('WUBLOADER_ENABLE_STACKSAMPLER', '').lower() != 'true': return + logging.info("Installing stacksampler") + # Note we only start each next timer once the previous timer signal has been processed. # There are two reasons for this: # 1. Avoid handling a signal while already handling a signal, however unlikely, diff --git a/docker-compose.jsonnet b/docker-compose.jsonnet index 1a482ca..1840199 100644 --- a/docker-compose.jsonnet +++ b/docker-compose.jsonnet @@ -151,8 +151,8 @@ env:: { // Uncomment this to set log level to debug // WUBLOADER_LOG_LEVEL: "DEBUG", - // Uncomment this to disable stacksampling performance monitoring - // WUBLOADER_DISABLE_STACKSAMPLER: "true", + // Uncomment this to enable stacksampling performance monitoring + // WUBLOADER_ENABLE_STACKSAMPLER: "true", }, // Now for the actual docker-compose config diff --git a/k8s.jsonnet b/k8s.jsonnet index 257db36..64a4e4a 100644 --- a/k8s.jsonnet +++ b/k8s.jsonnet @@ -86,8 +86,8 @@ env: { // Uncomment this to set log level to debug // WUBLOADER_LOG_LEVEL: "DEBUG", - // Uncomment this to disable stacksampling performance monitoring - // WUBLOADER_DISABLE_STACKSAMPLER: "true", + // Uncomment this to enable stacksampling performance monitoring + // WUBLOADER_ENABLE_STACKSAMPLER: "true", }, // Config for cutter upload locations. See cutter docs for full detail.