summaryrefslogtreecommitdiffstats
path: root/stator
diff options
context:
space:
mode:
authorMichael Manfre2022-12-04 20:08:23 -0500
committerGitHub2022-12-04 18:08:23 -0700
commit3f8045f4128d14558636b1c8a9445c29e8e1583d (patch)
treee36e7b4a928f9b8497283934d60e512ec02a818b /stator
parent258d992deb06a1ffd82ad33004223cf518ce3aa3 (diff)
downloadtakahe-3f8045f4128d14558636b1c8a9445c29e8e1583d.tar.gz
takahe-3f8045f4128d14558636b1c8a9445c29e8e1583d.tar.bz2
takahe-3f8045f4128d14558636b1c8a9445c29e8e1583d.zip
[WIP] Sentry improvements (#108)
Stator clears scope during the main loop to behave more like transactions. Transaction names are set. Sentry tags: * 'takahe.version' * 'takahe.app' values 'web' or 'stator' Added settings: * TAKAHE_SENTRY_SAMPLE_RATE * TAKAHE_SENTRY_TRACES_SAMPLE_RATE
Diffstat (limited to 'stator')
-rw-r--r--stator/runner.py77
1 files changed, 50 insertions, 27 deletions
diff --git a/stator/runner.py b/stator/runner.py
index 5cc0091..ecbaae6 100644
--- a/stator/runner.py
+++ b/stator/runner.py
@@ -7,7 +7,7 @@ from typing import List, Optional, Type
from django.utils import timezone
-from core import exceptions
+from core import exceptions, sentry
from core.models import Config
from stator.models import StatorModel
@@ -38,6 +38,7 @@ class StatorRunner:
self.run_for = run_for
async def run(self):
+ sentry.set_takahe_app("stator")
self.handled = 0
self.started = time.monotonic()
self.last_clean = time.monotonic() - self.schedule_interval
@@ -45,23 +46,32 @@ class StatorRunner:
# For the first time period, launch tasks
print("Running main task loop")
try:
- while True:
- # Do we need to do cleaning?
- if (time.monotonic() - self.last_clean) >= self.schedule_interval:
- # Refresh the config
- Config.system = await Config.aload_system()
- print(f"{self.handled} tasks processed so far")
- print("Running cleaning and scheduling")
- await self.run_scheduling()
+ with sentry.configure_scope() as scope:
+ while True:
+ # Do we need to do cleaning?
+ if (time.monotonic() - self.last_clean) >= self.schedule_interval:
+ # Refresh the config
+ Config.system = await Config.aload_system()
+ print(f"{self.handled} tasks processed so far")
+ print("Running cleaning and scheduling")
+ await self.run_scheduling()
- self.remove_completed_tasks()
- await self.fetch_and_process_tasks()
+ # Clear the cleaning breadcrumbs/extra for the main part of the loop
+ sentry.scope_clear(scope)
- # Are we in limited run mode?
- if self.run_for and (time.monotonic() - self.started) > self.run_for:
- break
- # Prevent busylooping
- await asyncio.sleep(0.5)
+ self.remove_completed_tasks()
+ await self.fetch_and_process_tasks()
+
+ # Are we in limited run mode?
+ if (
+ self.run_for
+ and (time.monotonic() - self.started) > self.run_for
+ ):
+ break
+ # Prevent busylooping
+ await asyncio.sleep(0.5)
+ # Clear the Sentry breadcrumbs and extra for next loop
+ sentry.scope_clear(scope)
except KeyboardInterrupt:
pass
# Wait for tasks to finish
@@ -79,10 +89,11 @@ class StatorRunner:
"""
Do any transition cleanup tasks
"""
- for model in self.models:
- asyncio.create_task(model.atransition_clean_locks())
- asyncio.create_task(model.atransition_schedule_due())
- self.last_clean = time.monotonic()
+ with sentry.start_transaction(op="task", name="stator.run_scheduling"):
+ for model in self.models:
+ asyncio.create_task(model.atransition_clean_locks())
+ asyncio.create_task(model.atransition_schedule_due())
+ self.last_clean = time.monotonic()
async def fetch_and_process_tasks(self):
# Calculate space left for tasks
@@ -106,14 +117,26 @@ class StatorRunner:
"""
Wrapper for atransition_attempt with fallback error handling
"""
- try:
- print(
- f"Attempting transition on {instance._meta.label_lower}#{instance.pk} from state {instance.state}"
+ task_name = f"stator.run_transition:{instance._meta.label_lower}#{{id}} from {instance.state}"
+ with sentry.start_transaction(op="task", name=task_name):
+ sentry.set_context(
+ "instance",
+ {
+ "model": instance._meta.label_lower,
+ "pk": instance.pk,
+ "state": instance.state,
+ "state_age": instance.state_age,
+ },
)
- await instance.atransition_attempt()
- except BaseException as e:
- await exceptions.acapture_exception(e)
- traceback.print_exc()
+
+ try:
+ print(
+ f"Attempting transition on {instance._meta.label_lower}#{instance.pk} from state {instance.state}"
+ )
+ await instance.atransition_attempt()
+ except BaseException as e:
+ await exceptions.acapture_exception(e)
+ traceback.print_exc()
def remove_completed_tasks(self):
"""