Recurring job can't be scheduled, see inner exception for details

bug?
, ,
1

I have an app, which is on live on three different servers, using a loadbalancer for user distribution.
The app uses its own queue and I have added a filter for jobs to keep their original queue in case they fail at some point. But then again, it continues to act like the app is not running. The error is like below;

System.InvalidOperationException: Recurring job can't be scheduled, see inner exception for details.
 ---> Hangfire.Common.JobLoadException: Could not load the job. See inner exception for the details.
 ---> System.IO.FileNotFoundException: Could not resolve assembly 'My.Api'.
   at System.TypeNameParser.ResolveAssembly(String asmName, Func`2 assemblyResolver, Boolean throwOnError, StackCrawlMark& stackMark)
   at System.TypeNameParser.ConstructType(Func`2 assemblyResolver, Func`4 typeResolver, Boolean throwOnError, Boolean ignoreCase, StackCrawlMark& stackMark)
   at System.TypeNameParser.GetType(String typeName, Func`2 assemblyResolver, Func`4 typeResolver, Boolean throwOnError, Boolean ignoreCase, StackCrawlMark& stackMark)
   at System.Type.GetType(String typeName, Func`2 assemblyResolver, Func`4 typeResolver, Boolean throwOnError)
   at Hangfire.Common.TypeHelper.DefaultTypeResolver(String typeName)
   at Hangfire.Storage.InvocationData.DeserializeJob()
   --- End of inner exception stack trace ---
   at Hangfire.Storage.InvocationData.DeserializeJob()
   at Hangfire.RecurringJobEntity..ctor(String recurringJobId, IDictionary`2 recurringJob, ITimeZoneResolver timeZoneResolver, DateTime now)
   --- End of inner exception stack trace ---
   at Hangfire.Server.RecurringJobScheduler.ScheduleRecurringJob(BackgroundProcessContext context, IStorageConnection connection, String recurringJobId, RecurringJobEntity recurringJob, DateTime now)

What can be the issue here? The apps are running. And once I trigger the recurring jobs, they are good to go, until they show the above error.

This is my AppStart file;

private IEnumerable<IDisposable> GetHangfireServers()
    {
        Hangfire.GlobalConfiguration.Configuration
            .SetDataCompatibilityLevel(CompatibilityLevel.Version_170)
            .UseSimpleAssemblyNameTypeSerializer()
            .UseRecommendedSerializerSettings()
            .UseSqlServerStorage(HangfireServer, new SqlServerStorageOptions
            {
                CommandBatchMaxTimeout = TimeSpan.FromMinutes(5),
                SlidingInvisibilityTimeout = TimeSpan.FromMinutes(5),
                QueuePollInterval = TimeSpan.Zero,
                UseRecommendedIsolationLevel = true,
                DisableGlobalLocks = true
            });

        yield return new BackgroundJobServer(new BackgroundJobServerOptions {
            Queues = new[] { "myapp" + GetEnvironmentName() },
            ServerName = "MyApp" + ConfigurationHelper.GetAppSetting("Environment")
        });
    }

public void Configuration(IAppBuilder app)
    {
       
        var container = new Container();
        container.Options.DefaultScopedLifestyle = new AsyncScopedLifestyle();

        RegisterTaskDependencies(container);
        container.RegisterWebApiControllers(System.Web.Http.GlobalConfiguration.Configuration);
        container.Verify();

        var configuration = new HttpConfiguration();
        configuration.DependencyResolver = new SimpleInjectorWebApiDependencyResolver(container);

        /* HANGFIRE CONFIGURATION */
        if (Environment == "Production")
        {
            GlobalJobFilters.Filters.Add(new PreserveOriginalQueueAttribute());
            Hangfire.GlobalConfiguration.Configuration.UseActivator(new SimpleInjectorJobActivator(container));
            Hangfire.GlobalConfiguration.Configuration.UseLogProvider(new Api.HangfireArea.Helpers.CustomLogProvider(container.GetInstance<Core.Modules.LogModule>()));
            app.UseHangfireAspNet(GetHangfireServers);
            app.UseHangfireDashboard("/hangfire", new DashboardOptions
            {
                Authorization = new[] { new DashboardAuthorization() },
                AppPath = GetBackToSiteURL(),
                DisplayStorageConnectionString = false
            });
            AddOrUpdateJobs();
        }
        /* HANGFIRE CONFIGURATION */

        app.UseWebApi(configuration);

        WebApiConfig.Register(configuration);
    }

public static void AddOrUpdateJobs()
{
            var queueName = "myapp" + GetEnvironmentName();
            RecurringJob.AddOrUpdate<HangfireArea.BackgroundJobs.AttachmentCreator>(
                 "MyApp_MyTask",
                 (service) => service.RunMyTask(), 
              "* * * * *", queue: queueName, timeZone: TimeZoneInfo.FindSystemTimeZoneById("Turkey Standard Time"));
  }

What can be the problem here?

2

For anyone who will experience this in the future the solution for me was this; https://stackoverflow.com/questions/66952039/hangfire-recurring-job-can-t-be-scheduled-see-inner-exception-for-details

3

You can create a job Filters that will retry and queue at the same time.
The difference is that you cannot wait to run the job. It will run immediately

public class AutomaticRetryQueueAttribute : JobFilterAttribute, IApplyStateFilter, IElectStateFilter
    {
        private string queue;
        private int attempts;
        private readonly object _lockObject = new object();

        private readonly ILog _logger = LogProvider.For<AutomaticRetryQueueAttribute>();

        public AutomaticRetryQueueAttribute(int Attempts = 10, string Queue = "Default")
        {
            queue = Queue;
            attempts = Attempts;
        }

        public int Attempts
        {
            get { lock (_lockObject) { return attempts; } }
            set
            {
                if (value < 0)
                {
                    throw new ArgumentOutOfRangeException(nameof(value), @"Attempts value must be equal or greater than zero.");
                }

                lock (_lockObject)
                {
                    attempts = value;
                }
            }
        }

        public string Queue
        {
            get { lock (_lockObject) { return queue; } }
            set
            {
                lock (_lockObject)
                {
                    queue = value;
                }
            }
        }

        public void OnStateApplied(ApplyStateContext context, IWriteOnlyTransaction transaction)
        {
            var newState = context.NewState as EnqueuedState;
            if (!string.IsNullOrWhiteSpace(queue) && newState != null && newState.Queue != Queue)
            {
                newState.Queue = String.Format(Queue, context.BackgroundJob.Job.Args.ToArray());
            }

            if (context.NewState is ScheduledState &&
                context.NewState.Reason != null &&
                context.NewState.Reason.StartsWith("Retry attempt"))
            {
                transaction.AddToSet("retries", context.BackgroundJob.Id);
            }
        }

        public void OnStateUnapplied(ApplyStateContext context, IWriteOnlyTransaction transaction)
        {
            if (context.OldStateName == ScheduledState.StateName)
            {
                transaction.RemoveFromSet("retries", context.BackgroundJob.Id);
            }
        }

        public void OnStateElection(ElectStateContext context)
        {
            var failedState = context.CandidateState as FailedState;
            if (failedState == null)
            {
                // This filter accepts only failed job state.
                return;
            }

            if (context.CandidateState is EnqueuedState enqueuedState)
            {
                enqueuedState.Queue = String.Format(Queue, context.BackgroundJob.Job.Args.ToArray());
            }

            var retryAttempt = context.GetJobParameter<int>("RetryCount") + 1;

            if (retryAttempt <= Attempts)
            {
                ScheduleAgainLater(context, retryAttempt, failedState);
            }
            else
            {
                _logger.ErrorException($"Failed to process the job '{context.BackgroundJob.Id}': an exception occurred.", failedState.Exception);
            }
        }

        private void ScheduleAgainLater(ElectStateContext context, int retryAttempt, FailedState failedState)
        {
            context.SetJobParameter("RetryCount", retryAttempt);

            const int maxMessageLength = 50;
            var exceptionMessage = failedState.Exception.Message.Length > maxMessageLength
                ? failedState.Exception.Message.Substring(0, maxMessageLength - 1) + "…"
                : failedState.Exception.Message;

            // If attempt number is less than max attempts, we should
            // schedule the job to run again later.

            var reason = $"Retry attempt {retryAttempt} of {Attempts}: {exceptionMessage}";

            context.CandidateState = (IState)new EnqueuedState { Reason = reason };

            if (context.CandidateState is EnqueuedState enqueuedState)
            {
                enqueuedState.Queue = String.Format(Queue, context.BackgroundJob.Job.Args.ToArray());
            }

            _logger.WarnException($"Failed to process the job '{context.BackgroundJob.Id}': an exception occurred. Retry attempt {retryAttempt} of {Attempts} will be performed.", failedState.Exception);
        }
    }