These have been here since yesterday, Is this some bug? How do I get rid of them?
Please tell me more about your setup:
- What version of Hangfire are you using?
- What storage are you using?
- What storage version are you using?
Hangfire.SqlServer v1.6.17
Hangfire.Core v1.6.17
SQL server 2012 64bit
Dashboard in web project + workers as topshelf windows service
Any chance you’ve disabled the AutomaticRetryAttribute in your dashboard configuration? Could you post here all the configuration logic? Looks like it was disabled when the delete operation was triggered. The only way to remove these entries now is to carefully update the Set table:
DELETE FROM [Set]
WHERE [Key] = N'retries' AND [Value] IN (
111111, -- replace with actual job ids
222222,
333333
)
In web app (dashboard configuration) I have no special mention of AutomaticRetryAttribute. At the time when this happened, we had some issues with jobs so we stopped the windows service and I tried to delete the retries but this happened. I’ll use the query you wrote to update the db
Code in startup class
GlobalConfiguration.Configuration.UseSqlServerStorage(DomainSettings.HangfireConnectionString);
GlobalConfiguration.Configuration.UseConsole();
GlobalJobFilters.Filters.Add(new DisableMultipleQueuedItemsAttribute());
app.UseHangfireDashboard(
"/hangfire",
new DashboardOptions
{
AppPath = "/Jobs",
Authorization = new[]
{
new AuthorizationAttribute()
}
});
In windows service project we have
GlobalConfiguration.Configuration.UseSqlServerStorage(DomainSettings.HangfireConnectionString);
GlobalConfiguration.Configuration.UseNinjectActivator(kernel);
GlobalConfiguration.Configuration.UseConsole();
var options = new BackgroundJobServerOptions
{
Queues = HangfireQueues.QueueList,
WorkerCount = 10
};
_server = new BackgroundJobServer(options);
GlobalJobFilters.Filters.Add(new AutomaticRetryAttribute
{
Attempts = 1
});
GlobalJobFilters.Filters.Add(new LogFailureAttribute());
GlobalJobFilters.Filters.Add(new ProlongExpirationTimeAttribute
{
ExpirationDuration = TimeSpan.FromDays(7)
});
GlobalJobFilters.Filters.Add(new DisableMultipleQueuedItemsAttribute());
LogFailureAttribute:
public class LogFailureAttribute : JobFilterAttribute, IApplyStateFilter
{
private static readonly ILog Logger = LogProvider.GetCurrentClassLogger();
public void OnStateApplied(ApplyStateContext context, IWriteOnlyTransaction transaction)
{
if (context.NewState is FailedState failedState)
{
var msg = $"Background job #{context.BackgroundJob.Id} was failed with an exception.";
Logger.ErrorException(msg, failedState.Exception);
var communicationData = new SystemNotificationCommunicationData
{
Error = $"Hangfire exception JobId: {context.BackgroundJob.Id}",
Details = failedState.Exception?.ToString()
};
var communicationService = ServiceLocator.Current.GetInstance<ICommunicationService>();
communicationService.SendNotificationToSystem(communicationData);
}
}
public void OnStateUnapplied(ApplyStateContext context, IWriteOnlyTransaction transaction)
{
}
}
Disable multiple jobs:
public class DisableMultipleQueuedItemsAttribute : JobFilterAttribute, IServerFilter, IElectStateFilter
{
private static readonly TimeSpan _lockTimeout = TimeSpan.FromSeconds(10);
private static readonly List<string> _releaseStates = new List<string> { DeletedState.StateName, FailedState.StateName };
public void OnPerforming(PerformingContext filterContext)
{
if (!AddFingerprintIfNotExists(filterContext.Connection, filterContext.BackgroundJob))
{
filterContext.Canceled = true;
}
}
public void OnPerformed(PerformedContext filterContext)
{
// If job finished with success we remove lock.
if (filterContext.Exception == null || filterContext.ExceptionHandled)
{
RemoveFingerprint(filterContext.Connection, filterContext.BackgroundJob);
}
}
public void OnStateElection(ElectStateContext context)
{
// If job is going to release state and it isn't in failed state we release the lock.
// if job is already in failed state we don't release lock because lock can be from a running job, failed job already had it's lock removed.
if (_releaseStates.Contains(context.CandidateState.Name) && context.CurrentState != FailedState.StateName)
{
RemoveFingerprint(context.Connection, context.BackgroundJob);
}
}
private static bool AddFingerprintIfNotExists(IStorageConnection connection, BackgroundJob job)
{
using (connection.AcquireDistributedLock(GetFingerprintLockKey(job), _lockTimeout))
{
var fingerprint = connection.GetAllEntriesFromHash(GetFingerprintKey(job));
if (fingerprint != null && fingerprint.ContainsKey("Timestamp"))
{
// Actual fingerprint found, returning.
return false;
}
// Fingerprint does not exist, it is invalid (no `Timestamp` key),
// or it is not actual (timeout expired).
connection.SetRangeInHash(
GetFingerprintKey(job),
new Dictionary<string, string>
{
{ "Timestamp", DateTimeOffset.UtcNow.ToString("o") }
});
return true;
}
}
private static string GetFingerprint(BackgroundJob job)
{
var parameters = string.Empty;
if (job.Job.Args != null)
{
parameters = string.Join(".", job.Job.Args);
}
if (job.Job.Type == null || job.Job.Method == null)
{
return string.Empty;
}
var fingerprint = $"{job.Job.Type.Name}.{job.Job.Method.Name}.{parameters}";
return Helpers.GetSHA256(fingerprint);
}
private static string GetFingerprintKey(BackgroundJob job)
{
return $"Fingerprint:{GetFingerprint(job)}";
}
private static string GetFingerprintLockKey(BackgroundJob job)
{
return $"{GetFingerprintKey(job)}:lock";
}
private static void RemoveFingerprint(IStorageConnection connection, BackgroundJob job)
{
using (connection.AcquireDistributedLock(GetFingerprintLockKey(job), _lockTimeout))
{
using (var transaction = connection.CreateWriteTransaction())
{
transaction.RemoveHash(GetFingerprintKey(job));
transaction.Commit();
}
}
}
}I’ve seen this issue as well. I wonder if it might have something to do with jobs that are terminated?
My biggest complaint is not that the deleted job shows up in the retries queue, but that I can’t remove the deleted job from the UI. It should remove the job from the ‘Set’ table even if the job no longer exists.