Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
StorageLeaf
Manage
Activity
Members
Labels
Plan
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Model registry
Operate
Environments
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
ProjectLeaf
StorageLeaf
Commits
57912746
Commit
57912746
authored
3 years ago
by
Robert Goldmann
Browse files
Options
Downloads
Patches
Plain Diff
#9 - use job scheduler to manually trigger cleanup
parent
2ffae620
No related branches found
No related tags found
No related merge requests found
Changes
3
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
src/logic/DatabaseCleanupService.py
+33
-0
33 additions, 0 deletions
src/logic/DatabaseCleanupService.py
src/logic/database/Schemas.py
+20
-0
20 additions, 0 deletions
src/logic/database/Schemas.py
src/logic/routers/DatabaseRouter.py
+13
-25
13 additions, 25 deletions
src/logic/routers/DatabaseRouter.py
with
66 additions
and
25 deletions
src/logic/DatabaseCleanupService.py
0 → 100644
+
33
−
0
View file @
57912746
from
datetime
import
datetime
from
typing
import
Dict
from
sqlalchemy.orm
import
Session
from
logic.database
import
Schemas
,
DatabaseInfoProvider
from
logic.database.DatabaseCleaner
import
DatabaseCleaner
from
logic.database.RetentionPolicy
import
RetentionPolicy
class
DatabaseCleanupService
:
def
__init__
(
self
,
cleanupSettings
:
Dict
):
self
.
_cleanupSettings
=
cleanupSettings
def
cleanup
(
self
,
db
:
Session
)
->
Schemas
.
DatabaseCleanupInfo
:
infoBefore
=
DatabaseInfoProvider
.
get_database_info
(
db
)
retentionPolicies
=
self
.
_cleanupSettings
[
'
retentionPolicies
'
]
policies
=
[]
for
item
in
retentionPolicies
:
policies
.
append
(
RetentionPolicy
(
numberOfMeasurementsPerDay
=
item
[
'
numberOfMeasurementsPerDay
'
],
ageInDays
=
item
[
'
ageInDays
'
]))
DatabaseCleaner
(
policies
,
self
.
_cleanupSettings
[
'
forceBackupAfterCleanup
'
]).
clean
(
db
,
datetime
.
now
().
date
())
infoAfter
=
DatabaseInfoProvider
.
get_database_info
(
db
)
deletedMeasurements
=
infoBefore
.
number_of_measurements
-
infoAfter
.
number_of_measurements
sizeFreed
=
infoBefore
.
size_on_disk_in_mb
-
infoAfter
.
size_on_disk_in_mb
infoDifference
=
Schemas
.
DatabaseInfo
(
number_of_measurements
=
deletedMeasurements
,
size_on_disk_in_mb
=
sizeFreed
)
return
Schemas
.
DatabaseCleanupInfo
(
status
=
Schemas
.
DatabaseCleanupStatus
.
FINISHED
,
before
=
infoBefore
,
after
=
infoAfter
,
difference
=
infoDifference
)
This diff is collapsed.
Click to expand it.
src/logic/database/Schemas.py
+
20
−
0
View file @
57912746
...
...
@@ -134,3 +134,23 @@ class SensorValue(BaseModel):
class
MultipleMeasurements
(
BaseModel
):
deviceName
:
str
sensors
:
List
[
SensorValue
]
# ===== scheduled jobs =====
class
ScheduledJob
(
BaseModel
):
job_id
:
str
run_frequency
:
str
next_run
:
str
class
Config
:
schema_extra
=
{
'
example
'
:
{
'
job_id
'
:
'
my job id
'
,
'
run_frequency
'
:
'
interval[0:05:00]
'
,
'
next_run
'
:
'
2021-09-30 22:12:09.397935+2:00
'
}
}
class
ScheduledJobs
(
BaseModel
):
jobs
:
List
[
ScheduledJob
]
This diff is collapsed.
Click to expand it.
src/logic/routers/DatabaseRouter.py
+
13
−
25
View file @
57912746
from
datetime
import
datetime
from
fastapi
import
APIRouter
,
Depends
from
fastapi
import
APIRouter
,
Depends
,
HTTPException
from
sqlalchemy.orm
import
Session
from
Settings
import
SETTINGS
from
logic.DatabaseCleanupService
import
DatabaseCleanupService
from
logic.Dependencies
import
get_database
,
check_api_key
from
logic.database
import
Schemas
,
DatabaseInfoProvider
from
logic.database.DatabaseCleaner
import
DatabaseCleaner
,
RetentionPolicy
router
=
APIRouter
(
prefix
=
'
/database
'
,
...
...
@@ -26,29 +24,19 @@ async def databaseInfo(db: Session = Depends(get_database)):
response_model
=
Schemas
.
Status
,
dependencies
=
[
Depends
(
check_api_key
)])
async
def
databaseCleanup
(
db
:
Session
=
Depends
(
get_database
)):
infoBefore
=
DatabaseInfoProvider
.
get_database_info
(
db
)
cleanupSettings
=
SETTINGS
[
'
database
'
][
'
cleanup
'
]
retentionPolicies
=
cleanupSettings
[
'
retentionPolicies
'
]
policies
=
[]
for
item
in
retentionPolicies
:
policies
.
append
(
RetentionPolicy
(
numberOfMeasurementsPerDay
=
item
[
'
numberOfMeasurementsPerDay
'
],
ageInDays
=
item
[
'
ageInDays
'
]))
DatabaseCleaner
(
policies
,
cleanupSettings
[
'
forceBackupAfterCleanup
'
]).
clean
(
db
,
datetime
.
now
().
date
())
infoAfter
=
DatabaseInfoProvider
.
get_database_info
(
db
)
deletedMeasurements
=
infoBefore
.
number_of_measurements
-
infoAfter
.
number_of_measurements
sizeFreed
=
infoBefore
.
size_on_disk_in_mb
-
infoAfter
.
size_on_disk_in_mb
infoDifference
=
Schemas
.
DatabaseInfo
(
number_of_measurements
=
deletedMeasurements
,
size_on_disk_in_mb
=
sizeFreed
)
from
logic
import
JobScheduler
cleanupService
=
DatabaseCleanupService
(
SETTINGS
[
'
database
'
][
'
cleanup
'
])
try
:
JobScheduler
.
SCHEDULER
.
run_manual_job
(
cleanupService
.
cleanup
,
args
=
[
db
])
except
JobScheduler
.
JobAlreadyRunningError
as
e
:
raise
HTTPException
(
status_code
=
400
,
detail
=
str
(
e
))
return
Schemas
.
DatabaseCleanupInfo
(
before
=
infoBefore
,
after
=
infoAfter
,
difference
=
infoDifference
)
return
Schemas
.
Status
(
message
=
'
Successfully triggered database cleanup
'
)
@router.get
(
'
/databaseCleanup
'
,
summary
=
'
Provides the status of the
current
database cleanup
'
,
response_model
=
Schemas
.
DatabaseCleanupInfo
)
summary
=
'
Provides the status of the
all scheduled
database cleanup
jobs
'
,
response_model
=
Schemas
.
ScheduledJobs
)
async
def
databaseCleanup
():
return
Schemas
.
DatabaseCleanupInfo
(
status
=
Schemas
.
DatabaseCleanupStatus
.
UNDEFINED
)
from
logic
import
JobScheduler
return
JobScheduler
.
SCHEDULER
.
get_scheduled_jobs
()
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment