From e8c2aaf92ae102b7e92cc74b4f4fceba40868bd2 Mon Sep 17 00:00:00 2001 From: Robert Goldmann <deadlocker@gmx.de> Date: Sun, 22 Aug 2021 22:52:40 +0200 Subject: [PATCH] #9 - perform database vaccum after deletion --- src/logic/database/Crud.py | 4 ++++ src/logic/database/DatabaseCleaner.py | 12 +++++++++--- src/logic/routers/GeneralRouter.py | 6 ++++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/logic/database/Crud.py b/src/logic/database/Crud.py index 4f4a4a6..d161865 100644 --- a/src/logic/database/Crud.py +++ b/src/logic/database/Crud.py @@ -184,5 +184,9 @@ def get_total_number_of_measurements(db: Session) -> List[int]: return db.query(Models.Measurement).count() +def perform_vacuum(db: Session): + db.execute('VACUUM') + + def __get_current_datetime(): return datetime.strftime(datetime.now(), DATE_FORMAT) diff --git a/src/logic/database/DatabaseCleaner.py b/src/logic/database/DatabaseCleaner.py index 0ee3e57..38759fe 100644 --- a/src/logic/database/DatabaseCleaner.py +++ b/src/logic/database/DatabaseCleaner.py @@ -14,6 +14,8 @@ LOGGER = logging.getLogger(Constants.APP_NAME) class DatabaseCleaner: MIN_DATE = datetime(year=1970, month=1, day=1).date() + DATE_FORMAT = "%Y-%m-%d" + def __init__(self, retentionPolicies: List[RetentionPolicy], forceBackupAfterCleanup: bool): self._policies = retentionPolicies self._forceBackupAfterCleanup = forceBackupAfterCleanup @@ -28,10 +30,11 @@ class DatabaseCleaner: allSensors = Crud.get_sensors(db, skip=0, limit=1000000) for sensor in allSensors: - LOGGER.debug(f'Cleaning measurements for sensor "{sensor.name}" ' - f'(id: {sensor.id}, device_id: {sensor.device_id})') self._cleanup_measurements_for_sensor(sensor, db, policy, policyStart) + LOGGER.debug('Performing database vacuum...') + Crud.perform_vacuum(db) + LOGGER.info('Database cleanup done') if self._forceBackupAfterCleanup: @@ -45,10 +48,13 @@ class DatabaseCleaner: return minDate = datetime.strptime(firstMeasurement.timestamp, Crud.DATE_FORMAT).date() + LOGGER.debug(f'Cleaning measurements for sensor "{sensor.name}" ' + f'from now to {minDate.strftime(DatabaseCleaner.DATE_FORMAT)} ' + f'(id: {sensor.id}, device_id: {sensor.device_id})') processedDate = policyStart while processedDate > minDate: - LOGGER.debug(f'Cleaning {processedDate.strftime("%Y-%m-%d")}...') + LOGGER.debug(f'Cleaning {processedDate.strftime(DatabaseCleaner.DATE_FORMAT)}...') DatabaseCleaner._cleanup_measurements_for_day(db, processedDate, policy, sensor.id) processedDate = processedDate - timedelta(days=1) diff --git a/src/logic/routers/GeneralRouter.py b/src/logic/routers/GeneralRouter.py index 84b0dcb..8bf908f 100644 --- a/src/logic/routers/GeneralRouter.py +++ b/src/logic/routers/GeneralRouter.py @@ -35,13 +35,15 @@ async def databaseInfo(db: Session = Depends(get_database)): async def databaseCleanup(db: Session = Depends(get_database)): infoBefore = DatabaseInfoProvider.get_database_info(db) - retentionPolicies = SETTINGS['database']['cleanup']['retentionPolicies'] + cleanupSettings = SETTINGS['database']['cleanup'] + + retentionPolicies = cleanupSettings['retentionPolicies'] policies = [] for item in retentionPolicies: policies.append(RetentionPolicy(numberOfMeasurementsPerDay=item['numberOfMeasurementsPerDay'], ageInDays=item['ageInDays'])) - DatabaseCleaner(policies).clean(db, datetime.now().date()) + DatabaseCleaner(policies, cleanupSettings['forceBackupAfterCleanup']).clean(db, datetime.now().date()) infoAfter = DatabaseInfoProvider.get_database_info(db) -- GitLab