// The error kernel shall handle errors for a given process. // This will be cases where the process itself where unable // to handle the error on it's own, and we might need to // restart the process, or send a message back to the operator // that the action which the message where supposed to trigger, // or that an event where unable to be processed. package steward import ( "fmt" "log" ) // errorKernel is the structure that will hold all the error // handling values and logic. type errorKernel struct { // TODO: The errorKernel should probably have a concept // of error-state which is a map of all the processes, // how many times a process have failed over the same // message etc... } // newErrorKernel will initialize and return a new error kernel func newErrorKernel() *errorKernel { return &errorKernel{ // ringBuffer: newringBuffer(), } } // startErrorKernel will start the error kernel and check if there // have been reveived any errors from any of the processes, and // handle them appropriately. // TODO: Since a process will be locked while waiting to send the error // on the errorCh maybe it makes sense to have a channel inside the // processes error handling with a select so we can send back to the // process if it should continue or not based not based on how severe // the error where. This should be right after sending the error // sending in the process. func (e *errorKernel) startErrorKernel(errorCh chan errProcess) { // TODO: For now it will just print the error messages to the // console. go func() { for { er := <-errorCh // We should be able to handle each error individually and // also concurrently, so the handler is started in it's // own go routine go func() { // TODO: Here we should check the severity of the error, // and also possibly the the error-state of the process // that fails, so we can decide if we should stop and // start a new process to replace to old one, or if we // should just kill the process and send message back to // the operator....or other ? // // Just print the error, and tell the process to continue log.Printf("*** error_kernel: %#v, type=%T\n", er, er) er.errorActionCh <- errActionContinue }() } }() } type errorAction int const ( // errActionJustPrint should just print the error, // and the worker process should continue. errActionContinue errorAction = iota // errActionKillAndSpawnNew should log the error, // stop the current worker process, and spawn a new. errActionKill errorAction = iota // errActionKillAndDie should log the error, stop the // current worker process, and send a message back to // the master supervisor that it was unable to complete // the action of the current message. The error message // should contain a copy of the original message. ) type errProcess struct { // Channel for communicating the action to take back to // to the process who triggered the error errorActionCh chan errorAction // Some informational text infoText string // The process structure that belongs to a given process process process // The message that where in progress when error occured message Message } func (e errProcess) Error() string { return fmt.Sprintf("worker error: proc = %#v, message = %#v", e.process, e.message) }