1
0
Fork 0
mirror of https://github.com/postmannen/ctrl.git synced 2024-12-14 12:37:31 +00:00
ctrl/processes.go

494 lines
14 KiB
Go
Raw Normal View History

2021-02-24 09:58:02 +00:00
package steward
import (
"context"
2021-02-24 09:58:02 +00:00
"fmt"
"log"
"sync"
"time"
"github.com/prometheus/client_golang/prometheus"
2021-02-24 09:58:02 +00:00
)
// processes holds all the information about running processes
type processes struct {
2021-08-11 10:23:37 +00:00
// The main context for subscriber processes.
ctx context.Context
// cancel func to send cancel signal to the subscriber processes context.
cancel context.CancelFunc
// The active spawned processes
2022-04-01 05:09:55 +00:00
// server
server *server
2021-10-08 10:07:10 +00:00
active procsMap
// mutex to lock the map
2021-10-08 20:39:46 +00:00
// mu sync.RWMutex
// The last processID created
lastProcessID int
2021-08-18 10:16:21 +00:00
// The instance global prometheus registry.
metrics *metrics
2022-02-02 07:54:36 +00:00
// Waitgroup to keep track of all the processes started.
2021-08-12 07:21:56 +00:00
wg sync.WaitGroup
2022-01-12 06:42:41 +00:00
// tui
tui *tui
// errorKernel
errorKernel *errorKernel
2022-02-02 07:54:36 +00:00
// configuration
configuration *Configuration
// Signatures
Signatures *signatures
}
// newProcesses will prepare and return a *processes which
// is map containing all the currently running processes.
2022-04-01 05:09:55 +00:00
func newProcesses(ctx context.Context, server *server) *processes {
p := processes{
2022-04-01 05:09:55 +00:00
server: server,
2022-02-02 07:54:36 +00:00
active: *newProcsMap(),
2022-04-01 05:09:55 +00:00
tui: server.tui,
errorKernel: server.errorKernel,
configuration: server.configuration,
Signatures: server.signatures,
2022-04-01 06:51:14 +00:00
metrics: server.metrics,
}
2021-08-16 11:01:12 +00:00
// Prepare the parent context for the subscribers.
2021-08-11 10:23:37 +00:00
ctx, cancel := context.WithCancel(ctx)
2021-10-08 11:42:19 +00:00
// // Start the processes map.
// go func() {
// p.active.run(ctx)
// }()
2021-10-08 10:07:10 +00:00
2021-08-11 10:23:37 +00:00
p.ctx = ctx
p.cancel = cancel
return &p
}
// ----------------------
2022-02-02 07:54:36 +00:00
// ----------------------
type procsMap struct {
2021-11-16 18:07:24 +00:00
procNames map[processName]process
2021-11-16 09:21:44 +00:00
mu sync.Mutex
}
func newProcsMap() *procsMap {
cM := procsMap{
2021-11-16 18:07:24 +00:00
procNames: make(map[processName]process),
}
return &cM
}
// ----------------------
// Start all the subscriber processes.
// Takes an initial process as it's input. All processes
// will be tied to this single process's context.
func (p *processes) Start(proc process) {
2021-08-11 10:23:37 +00:00
// Set the context for the initial process.
proc.ctx = p.ctx
// --- Subscriber services that can be started via flags
{
log.Printf("Starting REQOpProcessList subscriber: %#v\n", proc.node)
sub := newSubject(REQOpProcessList, string(proc.node))
2022-04-01 05:09:55 +00:00
proc := newProcess(proc.ctx, p.server, sub, processKindSubscriber, nil)
go proc.spawnWorker()
}
{
log.Printf("Starting REQOpProcessStart subscriber: %#v\n", proc.node)
sub := newSubject(REQOpProcessStart, string(proc.node))
2022-04-01 05:09:55 +00:00
proc := newProcess(proc.ctx, p.server, sub, processKindSubscriber, nil)
go proc.spawnWorker()
}
2021-09-20 09:53:17 +00:00
{
log.Printf("Starting REQOpProcessStop subscriber: %#v\n", proc.node)
sub := newSubject(REQOpProcessStop, string(proc.node))
2022-04-01 05:09:55 +00:00
proc := newProcess(proc.ctx, p.server, sub, processKindSubscriber, nil)
go proc.spawnWorker()
2021-09-20 09:53:17 +00:00
}
2021-02-24 09:58:02 +00:00
// Start a subscriber for textLogging messages
2021-09-08 16:56:23 +00:00
if proc.configuration.StartSubREQToFileAppend {
proc.startup.subREQToFileAppend(proc)
2021-02-24 09:58:02 +00:00
}
2021-04-06 17:42:03 +00:00
// Start a subscriber for text to file messages
2021-09-08 16:56:23 +00:00
if proc.configuration.StartSubREQToFile {
proc.startup.subREQToFile(proc)
2021-04-06 17:42:03 +00:00
}
2022-03-04 14:02:43 +00:00
// Start a subscriber for text to file messages
if proc.configuration.StartSubREQToFileNACK {
proc.startup.subREQToFileNACK(proc)
}
2021-11-17 12:02:48 +00:00
// Start a subscriber for reading file to copy
if proc.configuration.StartSubREQCopyFileFrom {
proc.startup.subREQCopyFileFrom(proc)
}
2021-11-18 08:34:16 +00:00
// Start a subscriber for writing copied file to disk
if proc.configuration.StartSubREQCopyFileTo {
proc.startup.subREQCopyFileTo(proc)
}
2021-04-05 06:37:24 +00:00
// Start a subscriber for Hello messages
2021-09-08 16:56:23 +00:00
if proc.configuration.StartSubREQHello {
proc.startup.subREQHello(proc)
2021-02-24 09:58:02 +00:00
}
2021-09-08 16:56:23 +00:00
if proc.configuration.StartSubREQErrorLog {
2021-04-06 05:56:49 +00:00
// Start a subscriber for REQErrorLog messages
proc.startup.subREQErrorLog(proc)
}
2021-04-06 04:08:26 +00:00
// Start a subscriber for Ping Request messages
2021-09-08 16:56:23 +00:00
if proc.configuration.StartSubREQPing {
proc.startup.subREQPing(proc)
}
2021-04-06 04:08:26 +00:00
// Start a subscriber for REQPong messages
2021-09-08 16:56:23 +00:00
if proc.configuration.StartSubREQPong {
proc.startup.subREQPong(proc)
}
2021-04-04 09:19:17 +00:00
// Start a subscriber for REQCliCommand messages
2021-09-08 16:56:23 +00:00
if proc.configuration.StartSubREQCliCommand {
proc.startup.subREQCliCommand(proc)
}
// Start a subscriber for CLICommandReply messages
2021-09-08 16:56:23 +00:00
if proc.configuration.StartSubREQToConsole {
proc.startup.subREQToConsole(proc)
}
2022-01-13 07:34:22 +00:00
// Start a subscriber for CLICommandReply messages
if proc.configuration.EnableTUI {
proc.startup.subREQTuiToConsole(proc)
}
if proc.configuration.StartPubREQHello != 0 {
proc.startup.pubREQHello(proc)
}
2021-04-06 17:42:03 +00:00
// Start a subscriber for Http Get Requests
2021-09-08 16:56:23 +00:00
if proc.configuration.StartSubREQHttpGet {
proc.startup.subREQHttpGet(proc)
2021-04-06 17:42:03 +00:00
}
2021-04-13 09:28:52 +00:00
2022-02-11 06:27:51 +00:00
if proc.configuration.StartSubREQHttpGetScheduled {
proc.startup.subREQHttpGetScheduled(proc)
}
2021-09-08 16:56:23 +00:00
if proc.configuration.StartSubREQTailFile {
proc.startup.subREQTailFile(proc)
2021-04-13 09:28:52 +00:00
}
if proc.configuration.StartSubREQCliCommandCont {
proc.startup.subREQCliCommandCont(proc)
}
2021-11-10 10:21:38 +00:00
if proc.configuration.StartSubREQRelay {
proc.startup.subREQRelay(proc)
}
proc.startup.subREQRelayInitial(proc)
proc.startup.subREQToSocket(proc)
2022-02-08 10:49:32 +00:00
proc.startup.subREQPublicKey(proc)
2021-02-24 09:58:02 +00:00
}
2021-08-11 10:23:37 +00:00
// Stop all subscriber processes.
func (p *processes) Stop() {
log.Printf("info: canceling all subscriber processes...\n")
2021-08-11 10:23:37 +00:00
p.cancel()
p.wg.Wait()
log.Printf("info: done canceling all subscriber processes.\n")
2021-08-11 10:23:37 +00:00
}
// ---------------------------------------------------------------------------------------
// Startup holds all the startup methods for subscribers.
2021-08-18 10:16:21 +00:00
type startup struct {
2022-04-01 07:21:50 +00:00
server *server
metrics *metrics
2021-08-18 10:16:21 +00:00
}
2022-04-01 05:09:55 +00:00
func newStartup(server *server) *startup {
2022-04-01 07:21:50 +00:00
s := startup{
server: server,
metrics: server.metrics,
}
2021-08-18 10:16:21 +00:00
return &s
}
func (s startup) subREQHttpGet(p process) {
log.Printf("Starting Http Get subscriber: %#v\n", p.node)
sub := newSubject(REQHttpGet, string(p.node))
2022-04-01 05:09:55 +00:00
proc := newProcess(p.ctx, p.processes.server, sub, processKindSubscriber, nil)
2021-11-09 13:01:42 +00:00
2022-04-01 05:09:55 +00:00
go proc.spawnWorker()
}
2022-02-11 06:27:51 +00:00
func (s startup) subREQHttpGetScheduled(p process) {
log.Printf("Starting Http Get Scheduled subscriber: %#v\n", p.node)
sub := newSubject(REQHttpGetScheduled, string(p.node))
2022-04-01 05:09:55 +00:00
proc := newProcess(p.ctx, p.server, sub, processKindSubscriber, nil)
2022-02-11 06:27:51 +00:00
2022-04-01 05:09:55 +00:00
go proc.spawnWorker()
2022-02-11 06:27:51 +00:00
}
func (s startup) pubREQHello(p process) {
log.Printf("Starting Hello Publisher: %#v\n", p.node)
sub := newSubject(REQHello, p.configuration.CentralNodeName)
2022-04-01 05:09:55 +00:00
proc := newProcess(p.ctx, s.server, sub, processKindPublisher, nil)
// Define the procFunc to be used for the process.
2022-03-31 12:57:30 +00:00
proc.procFunc = func(ctx context.Context, procFuncCh chan Message) error {
ticker := time.NewTicker(time.Second * time.Duration(p.configuration.StartPubREQHello))
for {
d := fmt.Sprintf("Hello from %v\n", p.node)
m := Message{
FileName: "hello.log",
Directory: "hello-messages",
ToNode: Node(p.configuration.CentralNodeName),
FromNode: Node(p.node),
Data: []byte(d),
Method: REQHello,
ACKTimeout: 10,
Retries: 1,
}
2022-03-31 12:57:30 +00:00
sam, err := newSubjectAndMessage(m)
if err != nil {
// In theory the system should drop the message before it reaches here.
2022-04-01 06:43:14 +00:00
p.errorKernel.errSend(p, m, err)
2022-03-31 12:57:30 +00:00
log.Printf("error: ProcessesStart: %v\n", err)
}
proc.toRingbufferCh <- []subjectAndMessage{sam}
select {
case <-ticker.C:
case <-ctx.Done():
er := fmt.Errorf("info: stopped handleFunc for: publisher %v", proc.subject.name())
// sendErrorLogMessage(proc.toRingbufferCh, proc.node, er)
log.Printf("%v\n", er)
return nil
}
}
}
2022-04-01 05:09:55 +00:00
go proc.spawnWorker()
}
func (s startup) subREQToConsole(p process) {
log.Printf("Starting Text To Console subscriber: %#v\n", p.node)
sub := newSubject(REQToConsole, string(p.node))
2022-04-01 05:09:55 +00:00
proc := newProcess(p.ctx, s.server, sub, processKindSubscriber, nil)
go proc.spawnWorker()
2022-01-13 07:34:22 +00:00
}
func (s startup) subREQTuiToConsole(p process) {
log.Printf("Starting Tui To Console subscriber: %#v\n", p.node)
sub := newSubject(REQTuiToConsole, string(p.node))
2022-04-01 05:09:55 +00:00
proc := newProcess(p.ctx, s.server, sub, processKindSubscriber, nil)
go proc.spawnWorker()
}
func (s startup) subREQCliCommand(p process) {
log.Printf("Starting CLICommand Request subscriber: %#v\n", p.node)
sub := newSubject(REQCliCommand, string(p.node))
2022-04-01 05:09:55 +00:00
proc := newProcess(p.ctx, s.server, sub, processKindSubscriber, nil)
go proc.spawnWorker()
}
func (s startup) subREQPong(p process) {
log.Printf("Starting Pong subscriber: %#v\n", p.node)
sub := newSubject(REQPong, string(p.node))
2022-04-01 05:09:55 +00:00
proc := newProcess(p.ctx, s.server, sub, processKindSubscriber, nil)
go proc.spawnWorker()
}
func (s startup) subREQPing(p process) {
log.Printf("Starting Ping Request subscriber: %#v\n", p.node)
sub := newSubject(REQPing, string(p.node))
2022-04-01 05:09:55 +00:00
proc := newProcess(p.ctx, s.server, sub, processKindSubscriber, nil)
go proc.spawnWorker()
}
func (s startup) subREQErrorLog(p process) {
log.Printf("Starting REQErrorLog subscriber: %#v\n", p.node)
sub := newSubject(REQErrorLog, "errorCentral")
2022-04-01 05:09:55 +00:00
proc := newProcess(p.ctx, s.server, sub, processKindSubscriber, nil)
go proc.spawnWorker()
}
2022-01-01 09:13:35 +00:00
// subREQHello is the handler that is triggered when we are receiving a hello
// message. To keep the state of all the hello's received from nodes we need
// to also start a procFunc that will live as a go routine tied to this process,
// where the procFunc will receive messages from the handler when a message is
// received, the handler will deliver the message to the procFunc on the
// proc.procFuncCh, and we can then read that message from the procFuncCh in
// the procFunc running.
func (s startup) subREQHello(p process) {
log.Printf("Starting Hello subscriber: %#v\n", p.node)
sub := newSubject(REQHello, string(p.node))
2022-04-01 05:09:55 +00:00
proc := newProcess(p.ctx, s.server, sub, processKindSubscriber, nil)
// The reason for running the say hello subscriber as a procFunc is that
// a handler are not able to hold state, and we need to hold the state
// of the nodes we've received hello's from in the sayHelloNodes map,
// which is the information we pass along to generate metrics.
2022-03-31 12:57:30 +00:00
proc.procFunc = func(ctx context.Context, procFuncCh chan Message) error {
sayHelloNodes := make(map[Node]struct{})
for {
// Receive a copy of the message sent from the method handler.
var m Message
select {
case m = <-procFuncCh:
case <-ctx.Done():
er := fmt.Errorf("info: stopped handleFunc for: subscriber %v", proc.subject.name())
// sendErrorLogMessage(proc.toRingbufferCh, proc.node, er)
log.Printf("%v\n", er)
return nil
2022-03-31 12:54:39 +00:00
}
2022-03-31 12:57:30 +00:00
// Add an entry for the node in the map
sayHelloNodes[m.FromNode] = struct{}{}
// update the prometheus metrics
2022-04-01 07:21:50 +00:00
s.metrics.promHelloNodesTotal.Set(float64(len(sayHelloNodes)))
s.metrics.promHelloNodesContactLast.With(prometheus.Labels{"nodeName": string(m.FromNode)}).SetToCurrentTime()
2022-03-31 12:57:30 +00:00
}
}
2022-04-01 05:09:55 +00:00
go proc.spawnWorker()
}
2021-04-13 15:15:13 +00:00
func (s startup) subREQToFile(p process) {
log.Printf("Starting text to file subscriber: %#v\n", p.node)
2021-04-13 15:15:13 +00:00
sub := newSubject(REQToFile, string(p.node))
2022-04-01 05:09:55 +00:00
proc := newProcess(p.ctx, s.server, sub, processKindSubscriber, nil)
2021-11-09 13:01:42 +00:00
2022-04-01 05:09:55 +00:00
go proc.spawnWorker()
}
2022-03-04 14:02:43 +00:00
func (s startup) subREQToFileNACK(p process) {
log.Printf("Starting text to file subscriber: %#v\n", p.node)
sub := newSubject(REQToFileNACK, string(p.node))
2022-04-01 05:09:55 +00:00
proc := newProcess(p.ctx, s.server, sub, processKindSubscriber, nil)
2022-03-04 14:02:43 +00:00
2022-04-01 05:09:55 +00:00
go proc.spawnWorker()
2022-03-04 14:02:43 +00:00
}
2021-11-17 12:02:48 +00:00
func (s startup) subREQCopyFileFrom(p process) {
log.Printf("Starting copy file from subscriber: %#v\n", p.node)
sub := newSubject(REQCopyFileFrom, string(p.node))
2022-04-01 05:09:55 +00:00
proc := newProcess(p.ctx, s.server, sub, processKindSubscriber, nil)
2021-11-17 12:02:48 +00:00
2022-04-01 05:09:55 +00:00
go proc.spawnWorker()
2021-11-17 12:02:48 +00:00
}
2021-11-18 08:34:16 +00:00
func (s startup) subREQCopyFileTo(p process) {
log.Printf("Starting copy file to subscriber: %#v\n", p.node)
sub := newSubject(REQCopyFileTo, string(p.node))
2022-04-01 05:09:55 +00:00
proc := newProcess(p.ctx, s.server, sub, processKindSubscriber, nil)
2021-11-18 08:34:16 +00:00
2022-04-01 05:09:55 +00:00
go proc.spawnWorker()
2021-11-18 08:34:16 +00:00
}
func (s startup) subREQToFileAppend(p process) {
log.Printf("Starting text logging subscriber: %#v\n", p.node)
sub := newSubject(REQToFileAppend, string(p.node))
2022-04-01 05:09:55 +00:00
proc := newProcess(p.ctx, s.server, sub, processKindSubscriber, nil)
2021-11-09 13:01:42 +00:00
2022-04-01 05:09:55 +00:00
go proc.spawnWorker()
}
2021-04-13 09:28:52 +00:00
func (s startup) subREQTailFile(p process) {
log.Printf("Starting tail log files subscriber: %#v\n", p.node)
2021-04-13 09:28:52 +00:00
sub := newSubject(REQTailFile, string(p.node))
2022-04-01 05:09:55 +00:00
proc := newProcess(p.ctx, s.server, sub, processKindSubscriber, nil)
2021-11-09 13:01:42 +00:00
2022-04-01 05:09:55 +00:00
go proc.spawnWorker()
2021-04-13 09:28:52 +00:00
}
func (s startup) subREQCliCommandCont(p process) {
log.Printf("Starting cli command with continous delivery: %#v\n", p.node)
sub := newSubject(REQCliCommandCont, string(p.node))
2022-04-01 05:09:55 +00:00
proc := newProcess(p.ctx, s.server, sub, processKindSubscriber, nil)
2021-11-10 10:21:38 +00:00
2022-04-01 05:09:55 +00:00
go proc.spawnWorker()
2021-11-10 10:21:38 +00:00
}
func (s startup) subREQRelay(p process) {
2021-12-06 13:57:02 +00:00
nodeWithRelay := fmt.Sprintf("*.%v", p.node)
log.Printf("Starting Relay: %#v\n", nodeWithRelay)
sub := newSubject(REQRelay, string(nodeWithRelay))
2022-04-01 05:09:55 +00:00
proc := newProcess(p.ctx, s.server, sub, processKindSubscriber, nil)
2022-04-01 05:09:55 +00:00
go proc.spawnWorker()
}
func (s startup) subREQRelayInitial(p process) {
log.Printf("Starting Relay Initial: %#v\n", p.node)
sub := newSubject(REQRelayInitial, string(p.node))
2022-04-01 05:09:55 +00:00
proc := newProcess(p.ctx, s.server, sub, processKindSubscriber, nil)
2021-11-09 13:01:42 +00:00
2022-04-01 05:09:55 +00:00
go proc.spawnWorker()
}
func (s startup) subREQToSocket(p process) {
log.Printf("Starting write to socket subscriber: %#v\n", p.node)
sub := newSubject(REQToSocket, string(p.node))
2022-04-01 05:09:55 +00:00
proc := newProcess(p.ctx, s.server, sub, processKindSubscriber, nil)
2021-11-09 13:01:42 +00:00
2022-04-01 05:09:55 +00:00
go proc.spawnWorker()
}
2021-08-11 10:23:37 +00:00
2022-02-08 10:49:32 +00:00
func (s startup) subREQPublicKey(p process) {
log.Printf("Starting get Public Key subscriber: %#v\n", p.node)
sub := newSubject(REQPublicKey, string(p.node))
2022-04-01 05:09:55 +00:00
proc := newProcess(p.ctx, s.server, sub, processKindSubscriber, nil)
2022-02-08 10:49:32 +00:00
2022-04-01 05:09:55 +00:00
go proc.spawnWorker()
2022-02-08 10:49:32 +00:00
}
2021-08-11 10:23:37 +00:00
// ---------------------------------------------------------------
// Print the content of the processes map.
func (p *processes) printProcessesMap() {
log.Printf("*** Output of processes map :\n")
2021-10-08 10:07:10 +00:00
2021-11-16 09:21:44 +00:00
{
p.active.mu.Lock()
2021-10-08 10:07:10 +00:00
2021-11-16 18:07:24 +00:00
for pName, proc := range p.active.procNames {
log.Printf("* proc - pub/sub: %v, procName in map: %v , id: %v, subject: %v\n", proc.processKind, pName, proc.processID, proc.subject.name())
2021-08-11 10:23:37 +00:00
}
2022-04-01 07:21:50 +00:00
p.metrics.promProcessesTotal.Set(float64(len(p.active.procNames)))
2021-11-16 09:21:44 +00:00
p.active.mu.Unlock()
}
2021-08-11 10:23:37 +00:00
}