diff options
| -rw-r--r-- | tools/edit_monitor/daemon_manager.py | 30 | ||||
| -rw-r--r-- | tools/edit_monitor/daemon_manager_test.py | 4 | ||||
| -rw-r--r-- | tools/edit_monitor/proto/edit_event.proto | 3 |
3 files changed, 23 insertions, 14 deletions
diff --git a/tools/edit_monitor/daemon_manager.py b/tools/edit_monitor/daemon_manager.py index afdefc54e9..328b6ee9a6 100644 --- a/tools/edit_monitor/daemon_manager.py +++ b/tools/edit_monitor/daemon_manager.py @@ -141,17 +141,11 @@ class DaemonManager: # Logging the error and continue. logging.warning("Failed to monitor daemon process with error: %s", e) - if ( - self.max_memory_usage >= memory_threshold - or self.max_cpu_usage >= cpu_threshold - ): - logging.error( - "Daemon process is consuming too much resource, killing..." - ), - self._send_error_event_to_clearcut( - edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_RESOURCE_USAGE - ) - self._terminate_process(self.daemon_process.pid) + if self.max_memory_usage >= memory_threshold: + self._handle_resource_exhausted_error("memory") + + if self.max_cpu_usage >= cpu_threshold: + self._handle_resource_exhausted_error("cpu") logging.info( "Daemon process %d terminated. Max memory usage: %f, Max cpu" @@ -392,6 +386,20 @@ class DaemonManager: return pids + def _handle_resource_exhausted_error(self, resource_type:str): + if resource_type == "memory": + self._send_error_event_to_clearcut( + edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE + ) + else: + self._send_error_event_to_clearcut( + edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_CPU_USAGE + ) + logging.error( + "Daemon process is consuming too much %s, killing...", resource_type + ), + self._terminate_process(self.daemon_process.pid) + def _send_error_event_to_clearcut(self, error_type): edit_monitor_error_event_proto = edit_event_pb2.EditEvent( user_name=self.user_name, diff --git a/tools/edit_monitor/daemon_manager_test.py b/tools/edit_monitor/daemon_manager_test.py index 8d183889f5..12aaab3f80 100644 --- a/tools/edit_monitor/daemon_manager_test.py +++ b/tools/edit_monitor/daemon_manager_test.py @@ -217,7 +217,7 @@ class DaemonManagerTest(unittest.TestCase): self.assert_no_subprocess_running() self._assert_error_event_logged( fake_cclient, - edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_RESOURCE_USAGE, + edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE, ) def test_monitor_daemon_subprocess_killed_high_cpu_usage(self): @@ -235,7 +235,7 @@ class DaemonManagerTest(unittest.TestCase): self.assert_no_subprocess_running() self._assert_error_event_logged( fake_cclient, - edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_RESOURCE_USAGE, + edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_CPU_USAGE, ) @mock.patch('subprocess.check_output') diff --git a/tools/edit_monitor/proto/edit_event.proto b/tools/edit_monitor/proto/edit_event.proto index dc3d3f68bd..9acc2e754b 100644 --- a/tools/edit_monitor/proto/edit_event.proto +++ b/tools/edit_monitor/proto/edit_event.proto @@ -16,8 +16,9 @@ message EditEvent { FAILED_TO_START_EDIT_MONITOR = 1; FAILED_TO_STOP_EDIT_MONITOR = 2; FAILED_TO_REBOOT_EDIT_MONITOR = 3; - KILLED_DUE_TO_EXCEEDED_RESOURCE_USAGE = 4; + KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE = 4; FORCE_CLEANUP = 5; + KILLED_DUE_TO_EXCEEDED_CPU_USAGE = 6; } // Event that logs a single edit |