Long running background process

question
1

We want to manage code written in R from our .net application. We are having our own internal class as below that will be called to enqueue a background job when the user requests is via API.

public class Run
    {
        public static void RunProcess(IJobCancellationToken token)
        {
            Environment.SetEnvironmentVariable("R_LIBS", "D:\\R-3.1.2\\library");
            new RProcess(token).Start();
        }
    }

    public class RProcess
    {
        private static readonly ILog Logger = LogManager.GetLogger("HangfireTest");
        private readonly IJobCancellationToken _token;
        private readonly Process _rProcess;
        private const int TimeoutInSeconds = 3*60;

        public RProcess(IJobCancellationToken token)
        {
            _token = token;
            var processInfo = new ProcessStartInfo
            {
                FileName = "D:\\R-3.1.2\\bin\\x64\\Rscript.exe",
                Arguments = "D:\\projects\\R-sample\\Optimization\\Optimization_Code.R",
                CreateNoWindow = true,
                RedirectStandardOutput = true,
                RedirectStandardError = true,
                UseShellExecute = false
            };

            _rProcess = new Process
            {
                StartInfo = processInfo
            };
        }

        public bool IsNotTimedOut()
        {
            return !_rProcess.HasExited && DateTime.Now.Subtract(_rProcess.StartTime).TotalSeconds < TimeoutInSeconds;
        }

        public void Start()
        {
            Logger.Debug("About to Start Process");
            _rProcess.Start();

            while (IsNotTimedOut())
            {
                try
                {
                    _token.ThrowIfCancellationRequested();
                    Thread.Sleep(2000);
                }
                catch (OperationCanceledException)
                {
                    Kill();
                    throw;
                }
            }
            if (!_rProcess.HasExited)
            {
                Kill();
            }
            else
            {
                var error = "";
                for (var line = _rProcess.StandardError.ReadLine();
                    line != null;
                    line = _rProcess.StandardError.ReadLine())
                {
                    error += line;
                }

                Logger.Debug(error);
            }
        }

        public void Kill()
        {
            if (_rProcess != null && _rProcess.Id != 0)
            {
                Logger.Debug(string.Format("process with id {0} , name: {1} Initiate Kill at: {2}", _rProcess.Id,
                    _rProcess.ProcessName, DateTime.UtcNow.ToLongTimeString()));
                new KillParentFirstStrategy().Kill(_rProcess.Id);
                Logger.Debug(string.Format("process with id {0} , name: {1} Completing Kill at: {2}", _rProcess.Id,
                    _rProcess.ProcessName, DateTime.UtcNow.ToLongTimeString()));
            }
        }
    }

We use the kill strategy whenever the cancellation token has a request for cancellation. The code to kill the process takes about 4-5 seconds based on logs. To test that Hangfire would pick up the processes not complete i tried the following:

On IIS 7.5 :

  • we explicitly “Stop” our Website. All processes we spawned get killed.On Starting the website hangfire picks up the process that was killed and executes it. All works well!!
  • we recycle the app pool associated with our application. Process get killed, though new process are not spawned now, why ? are we missing someting. On the hangfire UI though it says it is processing.
  • We stop the app pool associated with the application. Process get killed, though when we start the app pool again process are not spawned again

Any suggestions as to what we can do to resolve these would be helpful. Is there some configuration we are missing? Hangfire server is started as below:

[assembly: OwinStartup(typeof(Hangfire.MyWebApplication.Startup))]
namespace Hangfire
{
    // Note: For instructions on enabling IIS6 or IIS7 classic mode, 
    // visit http://go.microsoft.com/?LinkId=9394801

    public class WebApiApplication : HttpApplication
    {
        protected void Application_Start()
        {
            AreaRegistration.RegisterAllAreas();

            WebApiConfig.Register(GlobalConfiguration.Configuration);
            FilterConfig.RegisterGlobalFilters(GlobalFilters.Filters);
            RouteConfig.RegisterRoutes(RouteTable.Routes);
            BundleConfig.RegisterBundles(BundleTable.Bundles);
            log4net.Config.XmlConfigurator.Configure();
        }
    }


    namespace MyWebApplication
    {
        public class Startup
        {
            public void Configuration(IAppBuilder app)
            {
                app.UseHangfire(config =>
                {
                    config.UseSqlServerStorage("testHangfire");

                    var options = new BackgroundJobServerOptions
                    {
                        ShutdownTimeout = TimeSpan.FromSeconds(150),
                        WorkerCount = 2
                    };
                    config.UseServer(options);
                    
                });
            }
        }
    }

the client to enqueue jobs is as below:

var backgroundJobClient = new BackgroundJobClient(new SqlServerStorage("testHangfire"));
Logger.Debug("received request to start a new process with query id:" + id);
backgroundJobClient.Enqueue(() => Controllers.Run.RunProcess(JobCancellationToken.Null));