2021-02-24 09:58:02 +00:00
|
|
|
package steward
|
|
|
|
|
|
|
|
import (
|
2021-04-07 16:54:08 +00:00
|
|
|
"context"
|
2021-02-24 09:58:02 +00:00
|
|
|
"fmt"
|
2021-03-12 11:08:11 +00:00
|
|
|
"log"
|
2021-03-09 10:58:50 +00:00
|
|
|
"time"
|
2021-03-04 15:27:55 +00:00
|
|
|
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
2021-04-12 08:51:26 +00:00
|
|
|
"github.com/prometheus/client_golang/prometheus/promauto"
|
2021-02-24 09:58:02 +00:00
|
|
|
)
|
|
|
|
|
2021-04-08 11:43:47 +00:00
|
|
|
func (p process) ProcessesStart() {
|
2021-03-26 08:08:47 +00:00
|
|
|
|
|
|
|
// --- Subscriber services that can be started via flags
|
|
|
|
|
2021-04-09 09:30:40 +00:00
|
|
|
// Allways start an REQOpCommand subscriber
|
2021-03-31 10:26:28 +00:00
|
|
|
{
|
2021-04-16 11:43:58 +00:00
|
|
|
log.Printf("Starting REQOpCommand subscriber: %#v\n", p.node)
|
2021-04-08 11:43:47 +00:00
|
|
|
sub := newSubject(REQOpCommand, string(p.node))
|
2021-04-09 09:30:40 +00:00
|
|
|
proc := newProcess(p.natsConn, p.processes, p.toRingbufferCh, p.configuration, sub, p.errorCh, processKindSubscriber, []node{node(p.configuration.CentralNodeName)}, nil)
|
2021-04-08 11:43:47 +00:00
|
|
|
go proc.spawnWorker(p.processes, p.natsConn)
|
2021-03-31 10:26:28 +00:00
|
|
|
}
|
|
|
|
|
2021-02-24 09:58:02 +00:00
|
|
|
// Start a subscriber for textLogging messages
|
2021-04-13 13:54:04 +00:00
|
|
|
if p.configuration.StartSubREQToFileAppend.OK {
|
|
|
|
p.startup.subREQToFileAppend(p)
|
2021-02-24 09:58:02 +00:00
|
|
|
}
|
|
|
|
|
2021-04-06 17:42:03 +00:00
|
|
|
// Start a subscriber for text to file messages
|
2021-04-13 15:15:13 +00:00
|
|
|
if p.configuration.StartSubREQToFile.OK {
|
|
|
|
p.startup.subREQToFile(p)
|
2021-04-06 17:42:03 +00:00
|
|
|
}
|
|
|
|
|
2021-04-05 06:37:24 +00:00
|
|
|
// Start a subscriber for Hello messages
|
2021-04-08 11:43:47 +00:00
|
|
|
if p.configuration.StartSubREQHello.OK {
|
2021-04-09 09:30:40 +00:00
|
|
|
p.startup.subREQHello(p)
|
2021-02-24 09:58:02 +00:00
|
|
|
}
|
2021-02-24 14:43:31 +00:00
|
|
|
|
2021-04-08 11:43:47 +00:00
|
|
|
if p.configuration.StartSubREQErrorLog.OK {
|
2021-04-06 05:56:49 +00:00
|
|
|
// Start a subscriber for REQErrorLog messages
|
2021-04-09 09:30:40 +00:00
|
|
|
p.startup.subREQErrorLog(p)
|
2021-02-24 14:43:31 +00:00
|
|
|
}
|
2021-03-09 10:58:50 +00:00
|
|
|
|
2021-04-06 04:08:26 +00:00
|
|
|
// Start a subscriber for Ping Request messages
|
2021-04-08 11:43:47 +00:00
|
|
|
if p.configuration.StartSubREQPing.OK {
|
2021-04-09 09:30:40 +00:00
|
|
|
p.startup.subREQPing(p)
|
2021-03-26 08:08:47 +00:00
|
|
|
}
|
|
|
|
|
2021-04-06 04:08:26 +00:00
|
|
|
// Start a subscriber for REQPong messages
|
2021-04-08 11:43:47 +00:00
|
|
|
if p.configuration.StartSubREQPong.OK {
|
2021-04-09 09:30:40 +00:00
|
|
|
p.startup.subREQPong(p)
|
2021-03-26 08:08:47 +00:00
|
|
|
}
|
|
|
|
|
2021-04-04 09:19:17 +00:00
|
|
|
// Start a subscriber for REQCliCommand messages
|
2021-04-08 11:43:47 +00:00
|
|
|
if p.configuration.StartSubREQCliCommand.OK {
|
2021-04-09 09:30:40 +00:00
|
|
|
p.startup.subREQCliCommand(p)
|
2021-03-26 08:08:47 +00:00
|
|
|
}
|
|
|
|
|
2021-04-05 04:54:18 +00:00
|
|
|
// Start a subscriber for Not In Order Cli Command Request messages
|
2021-04-08 11:43:47 +00:00
|
|
|
if p.configuration.StartSubREQnCliCommand.OK {
|
2021-04-09 09:30:40 +00:00
|
|
|
p.startup.subREQnCliCommand(p)
|
2021-03-26 08:08:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Start a subscriber for CLICommandReply messages
|
2021-04-13 15:22:25 +00:00
|
|
|
if p.configuration.StartSubREQToConsole.OK {
|
|
|
|
p.startup.subREQToConsole(p)
|
2021-03-26 08:08:47 +00:00
|
|
|
}
|
|
|
|
|
2021-04-08 11:43:47 +00:00
|
|
|
if p.configuration.StartPubREQHello != 0 {
|
2021-04-09 09:30:40 +00:00
|
|
|
p.startup.pubREQHello(p)
|
2021-03-09 10:58:50 +00:00
|
|
|
}
|
2021-04-06 17:42:03 +00:00
|
|
|
|
|
|
|
// Start a subscriber for Http Get Requests
|
2021-04-08 11:43:47 +00:00
|
|
|
if p.configuration.StartSubREQHttpGet.OK {
|
2021-04-09 09:30:40 +00:00
|
|
|
p.startup.subREQHttpGet(p)
|
2021-04-06 17:42:03 +00:00
|
|
|
}
|
2021-04-13 09:28:52 +00:00
|
|
|
|
|
|
|
if p.configuration.StartSubREQTailFile.OK {
|
|
|
|
p.startup.subREQTailFile(p)
|
|
|
|
}
|
2021-02-24 09:58:02 +00:00
|
|
|
}
|
2021-04-08 11:43:47 +00:00
|
|
|
|
2021-04-09 09:30:40 +00:00
|
|
|
// ---------------------------------------------------------------------------------------
|
2021-04-08 11:43:47 +00:00
|
|
|
|
2021-04-09 09:30:40 +00:00
|
|
|
type startup struct{}
|
|
|
|
|
|
|
|
func (s startup) subREQHttpGet(p process) {
|
|
|
|
|
2021-04-16 11:43:58 +00:00
|
|
|
log.Printf("Starting Http Get subscriber: %#v\n", p.node)
|
2021-04-09 09:30:40 +00:00
|
|
|
sub := newSubject(REQHttpGet, string(p.node))
|
|
|
|
proc := newProcess(p.natsConn, p.processes, p.toRingbufferCh, p.configuration, sub, p.errorCh, processKindSubscriber, p.configuration.StartSubREQHttpGet.Values, nil)
|
|
|
|
// fmt.Printf("*** %#v\n", proc)
|
|
|
|
go proc.spawnWorker(p.processes, p.natsConn)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s startup) pubREQHello(p process) {
|
2021-04-16 11:43:58 +00:00
|
|
|
log.Printf("Starting Hello Publisher: %#v\n", p.node)
|
2021-04-09 09:30:40 +00:00
|
|
|
|
|
|
|
sub := newSubject(REQHello, p.configuration.CentralNodeName)
|
|
|
|
proc := newProcess(p.natsConn, p.processes, p.toRingbufferCh, p.configuration, sub, p.errorCh, processKindPublisher, []node{}, nil)
|
|
|
|
|
|
|
|
// Define the procFunc to be used for the process.
|
|
|
|
proc.procFunc = procFunc(
|
|
|
|
func(ctx context.Context) error {
|
|
|
|
ticker := time.NewTicker(time.Second * time.Duration(p.configuration.StartPubREQHello))
|
|
|
|
for {
|
|
|
|
// fmt.Printf("--- DEBUG : procFunc call:kind=%v, Subject=%v, toNode=%v\n", proc.processKind, proc.subject, proc.subject.ToNode)
|
|
|
|
|
|
|
|
d := fmt.Sprintf("Hello from %v\n", p.node)
|
|
|
|
|
|
|
|
m := Message{
|
2021-05-28 14:50:48 +00:00
|
|
|
ToNode: node(p.configuration.CentralNodeName),
|
2021-04-09 09:30:40 +00:00
|
|
|
FromNode: node(p.node),
|
|
|
|
Data: []string{d},
|
|
|
|
Method: REQHello,
|
|
|
|
}
|
|
|
|
|
|
|
|
sam, err := newSAM(m)
|
|
|
|
if err != nil {
|
|
|
|
// In theory the system should drop the message before it reaches here.
|
|
|
|
log.Printf("error: ProcessesStart: %v\n", err)
|
|
|
|
}
|
|
|
|
proc.toRingbufferCh <- []subjectAndMessage{sam}
|
|
|
|
|
|
|
|
select {
|
|
|
|
case <-ticker.C:
|
|
|
|
case <-ctx.Done():
|
|
|
|
er := fmt.Errorf("info: stopped handleFunc for: %v", proc.subject.name())
|
|
|
|
sendErrorLogMessage(proc.toRingbufferCh, proc.node, er)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
})
|
|
|
|
go proc.spawnWorker(p.processes, p.natsConn)
|
|
|
|
}
|
|
|
|
|
2021-04-13 15:22:25 +00:00
|
|
|
func (s startup) subREQToConsole(p process) {
|
2021-04-16 11:43:58 +00:00
|
|
|
log.Printf("Starting Text To Console subscriber: %#v\n", p.node)
|
2021-04-13 15:22:25 +00:00
|
|
|
sub := newSubject(REQToConsole, string(p.node))
|
|
|
|
proc := newProcess(p.natsConn, p.processes, p.toRingbufferCh, p.configuration, sub, p.errorCh, processKindSubscriber, p.configuration.StartSubREQToConsole.Values, nil)
|
2021-04-09 09:30:40 +00:00
|
|
|
go proc.spawnWorker(p.processes, p.natsConn)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s startup) subREQnCliCommand(p process) {
|
2021-04-16 11:43:58 +00:00
|
|
|
log.Printf("Starting CLICommand Not Sequential Request subscriber: %#v\n", p.node)
|
2021-04-09 09:30:40 +00:00
|
|
|
sub := newSubject(REQnCliCommand, string(p.node))
|
|
|
|
proc := newProcess(p.natsConn, p.processes, p.toRingbufferCh, p.configuration, sub, p.errorCh, processKindSubscriber, p.configuration.StartSubREQnCliCommand.Values, nil)
|
|
|
|
go proc.spawnWorker(p.processes, p.natsConn)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s startup) subREQCliCommand(p process) {
|
2021-04-16 11:43:58 +00:00
|
|
|
log.Printf("Starting CLICommand Request subscriber: %#v\n", p.node)
|
2021-04-09 09:30:40 +00:00
|
|
|
sub := newSubject(REQCliCommand, string(p.node))
|
|
|
|
proc := newProcess(p.natsConn, p.processes, p.toRingbufferCh, p.configuration, sub, p.errorCh, processKindSubscriber, p.configuration.StartSubREQCliCommand.Values, nil)
|
|
|
|
go proc.spawnWorker(p.processes, p.natsConn)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s startup) subREQPong(p process) {
|
2021-04-16 11:43:58 +00:00
|
|
|
log.Printf("Starting Pong subscriber: %#v\n", p.node)
|
2021-04-09 09:30:40 +00:00
|
|
|
sub := newSubject(REQPong, string(p.node))
|
|
|
|
proc := newProcess(p.natsConn, p.processes, p.toRingbufferCh, p.configuration, sub, p.errorCh, processKindSubscriber, p.configuration.StartSubREQPong.Values, nil)
|
|
|
|
go proc.spawnWorker(p.processes, p.natsConn)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s startup) subREQPing(p process) {
|
2021-04-16 11:43:58 +00:00
|
|
|
log.Printf("Starting Ping Request subscriber: %#v\n", p.node)
|
2021-04-09 09:30:40 +00:00
|
|
|
sub := newSubject(REQPing, string(p.node))
|
|
|
|
proc := newProcess(p.natsConn, p.processes, p.toRingbufferCh, p.configuration, sub, p.errorCh, processKindSubscriber, p.configuration.StartSubREQPing.Values, nil)
|
|
|
|
go proc.spawnWorker(p.processes, p.natsConn)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s startup) subREQErrorLog(p process) {
|
2021-04-16 11:43:58 +00:00
|
|
|
log.Printf("Starting REQErrorLog subscriber: %#v\n", p.node)
|
2021-04-09 09:30:40 +00:00
|
|
|
sub := newSubject(REQErrorLog, "errorCentral")
|
|
|
|
proc := newProcess(p.natsConn, p.processes, p.toRingbufferCh, p.configuration, sub, p.errorCh, processKindSubscriber, p.configuration.StartSubREQErrorLog.Values, nil)
|
|
|
|
go proc.spawnWorker(p.processes, p.natsConn)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s startup) subREQHello(p process) {
|
2021-04-16 11:43:58 +00:00
|
|
|
log.Printf("Starting Hello subscriber: %#v\n", p.node)
|
2021-04-09 09:30:40 +00:00
|
|
|
sub := newSubject(REQHello, string(p.node))
|
|
|
|
proc := newProcess(p.natsConn, p.processes, p.toRingbufferCh, p.configuration, sub, p.errorCh, processKindSubscriber, p.configuration.StartSubREQHello.Values, nil)
|
|
|
|
proc.procFuncCh = make(chan Message)
|
|
|
|
|
|
|
|
// The reason for running the say hello subscriber as a procFunc is that
|
|
|
|
// a handler are not able to hold state, and we need to hold the state
|
|
|
|
// of the nodes we've received hello's from in the sayHelloNodes map,
|
|
|
|
// which is the information we pass along to generate metrics.
|
|
|
|
proc.procFunc = func(ctx context.Context) error {
|
|
|
|
sayHelloNodes := make(map[node]struct{})
|
2021-04-12 08:51:26 +00:00
|
|
|
|
|
|
|
promHelloNodes := promauto.NewGauge(prometheus.GaugeOpts{
|
2021-04-12 14:49:16 +00:00
|
|
|
Name: "hello_nodes_total",
|
2021-04-12 08:51:26 +00:00
|
|
|
Help: "The current number of total nodes who have said hello",
|
|
|
|
})
|
|
|
|
|
2021-04-12 14:40:18 +00:00
|
|
|
promHelloNodesNameVec := promauto.NewGaugeVec(prometheus.GaugeOpts{
|
2021-04-12 14:49:16 +00:00
|
|
|
Name: "hello_node_last_hello",
|
2021-04-12 14:40:18 +00:00
|
|
|
Help: "Name of the nodes who have said hello",
|
|
|
|
}, []string{"nodeName"},
|
|
|
|
)
|
|
|
|
|
2021-04-09 09:30:40 +00:00
|
|
|
for {
|
|
|
|
// Receive a copy of the message sent from the method handler.
|
|
|
|
var m Message
|
|
|
|
|
|
|
|
select {
|
|
|
|
case m = <-proc.procFuncCh:
|
|
|
|
case <-ctx.Done():
|
|
|
|
er := fmt.Errorf("info: stopped handleFunc for: %v", proc.subject.name())
|
|
|
|
sendErrorLogMessage(proc.toRingbufferCh, proc.node, er)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2021-04-12 14:40:18 +00:00
|
|
|
// Add an entry for the node in the map
|
2021-04-09 09:30:40 +00:00
|
|
|
sayHelloNodes[m.FromNode] = struct{}{}
|
|
|
|
|
|
|
|
// update the prometheus metrics
|
2021-04-12 08:51:26 +00:00
|
|
|
promHelloNodes.Set(float64(len(sayHelloNodes)))
|
2021-04-12 14:40:18 +00:00
|
|
|
promHelloNodesNameVec.With(prometheus.Labels{"nodeName": string(m.FromNode)}).SetToCurrentTime()
|
|
|
|
|
2021-04-09 09:30:40 +00:00
|
|
|
}
|
2021-04-08 11:43:47 +00:00
|
|
|
}
|
2021-04-09 09:30:40 +00:00
|
|
|
go proc.spawnWorker(p.processes, p.natsConn)
|
|
|
|
}
|
|
|
|
|
2021-04-13 15:15:13 +00:00
|
|
|
func (s startup) subREQToFile(p process) {
|
2021-04-16 11:43:58 +00:00
|
|
|
log.Printf("Starting text to file subscriber: %#v\n", p.node)
|
2021-04-13 15:15:13 +00:00
|
|
|
sub := newSubject(REQToFile, string(p.node))
|
|
|
|
proc := newProcess(p.natsConn, p.processes, p.toRingbufferCh, p.configuration, sub, p.errorCh, processKindSubscriber, p.configuration.StartSubREQToFile.Values, nil)
|
2021-04-09 09:30:40 +00:00
|
|
|
// fmt.Printf("*** %#v\n", proc)
|
|
|
|
go proc.spawnWorker(p.processes, p.natsConn)
|
|
|
|
}
|
|
|
|
|
2021-04-13 13:54:04 +00:00
|
|
|
func (s startup) subREQToFileAppend(p process) {
|
2021-04-16 11:43:58 +00:00
|
|
|
log.Printf("Starting text logging subscriber: %#v\n", p.node)
|
2021-04-13 13:54:04 +00:00
|
|
|
sub := newSubject(REQToFileAppend, string(p.node))
|
|
|
|
proc := newProcess(p.natsConn, p.processes, p.toRingbufferCh, p.configuration, sub, p.errorCh, processKindSubscriber, p.configuration.StartSubREQToFileAppend.Values, nil)
|
2021-04-09 09:30:40 +00:00
|
|
|
// fmt.Printf("*** %#v\n", proc)
|
|
|
|
go proc.spawnWorker(p.processes, p.natsConn)
|
2021-04-08 11:43:47 +00:00
|
|
|
}
|
2021-04-13 09:28:52 +00:00
|
|
|
|
|
|
|
func (s startup) subREQTailFile(p process) {
|
2021-04-16 11:43:58 +00:00
|
|
|
log.Printf("Starting tail log files subscriber: %#v\n", p.node)
|
2021-04-13 09:28:52 +00:00
|
|
|
sub := newSubject(REQTailFile, string(p.node))
|
|
|
|
proc := newProcess(p.natsConn, p.processes, p.toRingbufferCh, p.configuration, sub, p.errorCh, processKindSubscriber, p.configuration.StartSubREQTailFile.Values, nil)
|
|
|
|
// fmt.Printf("*** %#v\n", proc)
|
|
|
|
go proc.spawnWorker(p.processes, p.natsConn)
|
|
|
|
}
|