@ -1,22 +1,30 @@
import calendar
import calendar
import logging
import traceback
from rq . serializers import resolve_serializer
from rq . serializers import resolve_serializer
import time
import time
from datetime import datetime , timedelta , timezone
from datetime import datetime , timedelta , timezone
from typing import TYPE_CHECKING , Any , List , Optional , Type , Union
from typing import TYPE_CHECKING , Any , List , Optional , Type , Union
from . timeouts import JobTimeoutException , UnixSignalDeathPenalty
if TYPE_CHECKING :
if TYPE_CHECKING :
from redis import Redis
from redis import Redis
from redis . client import Pipeline
from redis . client import Pipeline
from . utils import as_text
from . utils import as_text
from . connections import resolve_connection
from . connections import resolve_connection
from . defaults import DEFAULT_FAILURE_TTL
from . defaults import DEFAULT_FAILURE_TTL , CALLBACK_TIMEOUT
from . exceptions import InvalidJobOperation , NoSuchJobError
from . exceptions import InvalidJobOperation , NoSuchJobError , AbandonedJobError
from . job import Job , JobStatus
from . job import Job , JobStatus
from . queue import Queue
from . queue import Queue
from . utils import backend_class , current_timestamp
from . utils import backend_class , current_timestamp
logger = logging . getLogger ( " rq.registry " )
class BaseRegistry :
class BaseRegistry :
"""
"""
Base implementation of a job registry , implemented in Redis sorted set .
Base implementation of a job registry , implemented in Redis sorted set .
@ -202,9 +210,10 @@ class StartedJobRegistry(BaseRegistry):
"""
"""
key_template = ' rq:wip: {0} '
key_template = ' rq:wip: {0} '
death_penalty_class = UnixSignalDeathPenalty
def cleanup ( self , timestamp : Optional [ float ] = None ) :
def cleanup ( self , timestamp : Optional [ float ] = None ) :
""" Remove expir ed jobs from registry and add them to FailedJobRegistry.
""" Remove abandon ed jobs from registry and add them to FailedJobRegistry.
Removes jobs with an expiry time earlier than timestamp , specified as
Removes jobs with an expiry time earlier than timestamp , specified as
seconds since the Unix epoch . timestamp defaults to call time if
seconds since the Unix epoch . timestamp defaults to call time if
@ -226,6 +235,14 @@ class StartedJobRegistry(BaseRegistry):
except NoSuchJobError :
except NoSuchJobError :
continue
continue
if job . failure_callback :
try :
with self . death_penalty_class ( CALLBACK_TIMEOUT , JobTimeoutException , job_id = job . id ) :
job . failure_callback ( job , self . connection ,
AbandonedJobError , AbandonedJobError ( ) , traceback . extract_stack ( ) )
except : # noqa
logger . exception ( ' Registry %s : error while executing failure callback ' , self . key )
retry = job . retries_left and job . retries_left > 0
retry = job . retries_left and job . retries_left > 0
if retry :
if retry :
@ -233,8 +250,11 @@ class StartedJobRegistry(BaseRegistry):
job . retry ( queue , pipeline )
job . retry ( queue , pipeline )
else :
else :
exc_string = f " due to { AbandonedJobError . __name__ } "
logger . warning ( f ' { self . __class__ . __name__ } cleanup: Moving job to { FailedJobRegistry . __name__ } '
f ' ( { exc_string } ) ' )
job . set_status ( JobStatus . FAILED )
job . set_status ( JobStatus . FAILED )
job . _exc_info = " Moved to FailedJobRegistry at %s " % datetime . now ( )
job . _exc_info = f" Moved to { FailedJobRegistry . __name__ } , { exc_string } , at { datetime . now ( ) } "
job . save ( pipeline = pipeline , include_meta = False )
job . save ( pipeline = pipeline , include_meta = False )
job . cleanup ( ttl = - 1 , pipeline = pipeline )
job . cleanup ( ttl = - 1 , pipeline = pipeline )
failed_job_registry . add ( job , job . failure_ttl )
failed_job_registry . add ( job , job . failure_ttl )