purge_old_runs() now also deletes files+dirs. Previously it was only dirs that could be purged.

src/purge/purge_old_runs.py

@@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
 
 def purge_old_runs(run_dir: str, max_age_hours: float = 1.0, prefix: str = "myrun_") -> None:
     """
-    Deletes runs in the specified run_dir older than max_age_hours and matching the specified prefix.
+    Deletes files and directories in the specified run_dir older than max_age_hours and matching the specified prefix.
     """
     if not os.path.isabs(run_dir):
         raise ValueError(f"run_dir must be an absolute path: {run_dir}")
@@ -17,26 +17,27 @@ def purge_old_runs(run_dir: str, max_age_hours: float = 1.0, prefix: str = "myru
     now = datetime.datetime.now()
     cutoff = now - datetime.timedelta(hours=max_age_hours)
 
-    for run_id in os.listdir(run_dir):
-        if not run_id.startswith(prefix):
+    for item in os.listdir(run_dir):
+        if not item.startswith(prefix):
             continue  # Skip files and directories that don't match the prefix
 
-        run_path = os.path.join(run_dir, run_id)
-        if not os.path.isdir(run_path):
-            continue  # Skip files
+        item_path = os.path.join(run_dir, item)
 
         try:
-            # Get the modification time of the directory
-            mtime = datetime.datetime.fromtimestamp(os.path.getmtime(run_path))
+            # Get the modification time of the item (file or directory)
+            mtime = datetime.datetime.fromtimestamp(os.path.getmtime(item_path))
 
             if mtime < cutoff:
-                logger.debug(f"Deleting old run: {run_id} from {run_dir}")
-                shutil.rmtree(run_path) # Delete the directory and all its contents
+                logger.debug(f"Deleting old data: {item} from {run_dir}")
+                if os.path.isdir(item_path):
+                    shutil.rmtree(item_path)  # Delete the directory and all its contents
+                else:
+                    os.remove(item_path)  # Delete the file
             else:
-                logger.debug(f"Skipping {run_id} in {run_dir}, last modified: {mtime}")
+                logger.debug(f"Skipping {item} in {run_dir}, last modified: {mtime}")
 
         except Exception as e:
-            logger.error(f"Error processing {run_id} in {run_dir}: {e}")
+            logger.error(f"Error processing {item} in {run_dir}: {e}")
 
 def start_purge_scheduler(run_dir: str, purge_interval_seconds: float=3600, prefix: str = "myrun_") -> None:
     """

src/purge/tests/test_purge_old_runs.py

@@ -14,14 +14,16 @@ class TestPurgeOldRuns(unittest.TestCase):
         os.makedirs(self.test_run_dir, exist_ok=True)
 
         # Create some dummy run directories with different modification times
-        self.create_dummy_run("myrun_run1", hours_old=0.5)
-        self.create_dummy_run("myrun_run2", hours_old=1.5)  # Should be purged
-        self.create_dummy_run("myrun_run3", hours_old=2) # Should be purged
-        self.create_dummy_run("myrun_run4", hours_old=0.25)
-        self.create_dummy_run("myrun_run5", hours_old=1)  # Boundary condition, might be purged
-        self.create_dummy_run("myrun_run6", hours_old=0)  # Today
-        self.create_dummy_run("other_run7", hours_old=1.5) #Should NOT be purged
-        self.create_dummy_file("not_a_run.txt") # a file that should be left alone
+        self.create_dummy_dir("myrun_dir1", hours_old=0.5)
+        self.create_dummy_dir("myrun_dir2", hours_old=1.5)
+        self.create_dummy_dir("myrun_dir3", hours_old=2)
+        self.create_dummy_dir("myrun_dir4", hours_old=0.25)
+        self.create_dummy_dir("myrun_dir5", hours_old=1)
+        self.create_dummy_dir("myrun_dir6", hours_old=0)
+        self.create_dummy_dir("other_dir7", hours_old=1.5) # doesn't have the prefix, so don't delete
+        self.create_dummy_file("other_file.txt", hours_old=5) # doesn't have the prefix, so don't delete
+        self.create_dummy_file("myrun_file1.txt", hours_old=0.25)
+        self.create_dummy_file("myrun_file2.txt", hours_old=1.5)
 
     def tearDown(self):
         """Clean up test environment after each test."""
@@ -29,29 +31,33 @@ class TestPurgeOldRuns(unittest.TestCase):
         if os.path.exists(self.test_run_dir):
             shutil.rmtree(self.test_run_dir)
 
-    def create_dummy_run(self, run_id, hours_old):
+    def create_dummy_dir(self, dirname: str, hours_old: float):
         """Creates a dummy run directory with a specific modification time."""
-        run_path = os.path.join(self.test_run_dir, run_id)
-        os.makedirs(run_path, exist_ok=True)
+        path = os.path.join(self.test_run_dir, dirname)
+        os.makedirs(path, exist_ok=True)
 
         # Set the modification time of the directory
         mtime = time.time() - (hours_old * 3600)  # seconds
-        os.utime(run_path, (mtime, mtime))
+        os.utime(path, (mtime, mtime))
 
-    def create_dummy_file(self, filename):
+    def create_dummy_file(self, filename: str, hours_old: float):
         """Create a dummy file in the test directory."""
-        filepath = os.path.join(self.test_run_dir, filename)
-        with open(filepath, "w") as f:
+        path = os.path.join(self.test_run_dir, filename)
+        with open(path, "w") as f:
             f.write("This is a dummy file.")
 
+        # Set the modification time of the file
+        mtime = time.time() - (hours_old * 3600)  # seconds
+        os.utime(path, (mtime, mtime))
+
     def test_purge_old_runs(self):
         """Tests the purge_old_runs function."""
         max_age_hours = 0.95
         purge_old_runs(self.test_run_dir, max_age_hours=max_age_hours, prefix="myrun_")  # Pass the directory
 
         # Check which runs should have been purged
-        runs_to_keep = ["myrun_run1", "myrun_run4", "myrun_run6", "other_run7","not_a_run.txt"]
-        runs_to_purge = ["myrun_run2", "myrun_run3", "myrun_run5"]
+        runs_to_keep = ["myrun_dir1", "myrun_dir4", "myrun_dir6", "other_dir7", "other_file.txt", "myrun_file1.txt"]
+        runs_to_purge = ["myrun_dir2", "myrun_dir3", "myrun_dir5", "myrun_file2.txt"]
 
         for run_id in runs_to_keep:
             run_path = os.path.join(self.test_run_dir, run_id)
@@ -74,7 +80,7 @@ class TestPurgeOldRuns(unittest.TestCase):
         purge_old_runs(self.test_run_dir, max_age_hours=max_age_hours, prefix="myrun_")  # Pass the directory
 
         # All runs should still exist, including the one with the wrong prefix.
-        expected_runs = ["myrun_run1", "myrun_run2", "myrun_run3", "myrun_run4", "myrun_run5", "myrun_run6", "other_run7", "not_a_run.txt"]
+        expected_runs = ["myrun_dir1", "myrun_dir2", "myrun_dir3", "myrun_dir4", "myrun_dir5", "myrun_dir6", "other_dir7", "other_file.txt", "myrun_file1.txt", "myrun_file2.txt"]
         for run_id in expected_runs:
             run_path = os.path.join(self.test_run_dir, run_id)
             self.assertTrue(os.path.exists(run_path), f"Run {run_id} should not have been purged.")