rewrote the textlogger subscriber, fixed * in allowed publishers, cleaned up logging

commandOrEventType.go

@@ -7,7 +7,6 @@
 package steward
 
 import (
-	"fmt"
 	"log"
 )
 
@@ -61,17 +60,13 @@ type CommandOrEventAvailable struct {
 	topics map[CommandOrEvent]struct{}
 }
 
-func (co CommandOrEventAvailable) CheckIfExists(c CommandOrEvent) bool {
+func (co CommandOrEventAvailable) CheckIfExists(c CommandOrEvent, subject Subject) bool {
 	_, ok := co.topics[c]
 	if ok {
-		fmt.Println("-------------------------------------------------------------")
-		log.Printf("THE COMMAND OR EVENT EXISTS: %v\n", c)
-		fmt.Println("-------------------------------------------------------------")
+		log.Printf("info: CommandOrEventAvailable.CheckIfExists: command or event found: %v, for %v\n", c, subject.name())
 		return true
 	} else {
-		fmt.Println("-------------------------------------------------------------")
-		log.Printf("THE COMMAND OR EVENT DO NOT EXIST: %v\n", c)
-		fmt.Println("-------------------------------------------------------------")
+		log.Printf("error: CommandOrEventAvailable.CheckIfExists: command or event not found: %v, for %v\n", c, subject.name())
 		return false
 	}
 }

configurationAndFlags.go

@@ -29,6 +29,8 @@ type Configuration struct {
 	DefaultMessageRetries int
 	// Make the current node send hello messages to central at given interval in seconds
 	PublisherServiceSayhello int
+	// Publisher data folder
+	SubscribersDataFolder string
 }
 
 func NewConfiguration() *Configuration {
@@ -46,6 +48,7 @@ func newConfigurationDefaults() Configuration {
 		DefaultMessageTimeout:    10,
 		DefaultMessageRetries:    1,
 		PublisherServiceSayhello: 30,
+		SubscribersDataFolder:    "./data",
 	}
 	return c
 }
@@ -76,6 +79,8 @@ func (c *Configuration) CheckFlags() {
 	flag.IntVar(&c.DefaultMessageTimeout, "defaultMessageTimeout", fc.DefaultMessageTimeout, "default message timeout in seconds. This can be overridden on the message level")
 	flag.IntVar(&c.DefaultMessageRetries, "defaultMessageRetries", fc.DefaultMessageRetries, "default amount of retries that will be done before a message is thrown away, and out of the system")
 	flag.IntVar(&c.PublisherServiceSayhello, "publisherServiceSayhello", fc.PublisherServiceSayhello, "Make the current node send hello messages to central at given interval in seconds")
+	flag.StringVar(&c.SubscribersDataFolder, "subscribersDataFolder", fc.SubscribersDataFolder, "The data folder where subscribers are allowed to write their data if needed")
+
 	flag.Parse()
 
 	if err := c.WriteConfigFile(); err != nil {
@@ -93,7 +98,7 @@ func (c *Configuration) ReadConfigFile() (Configuration, error) {
 
 	f, err := os.OpenFile(fp, os.O_RDONLY, 0600)
 	if err != nil {
-		return Configuration{}, fmt.Errorf("error: failed to open file %v", err)
+		return Configuration{}, fmt.Errorf("error: ReadConfigFile: failed to open file: %v", err)
 	}
 	defer f.Close()
 
@@ -122,7 +127,7 @@ func (c *Configuration) WriteConfigFile() error {
 
 	f, err := os.OpenFile(fp, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
-		return fmt.Errorf("error: failed to open file %v", err)
+		return fmt.Errorf("error: WriteConfigFile: failed to open file: %v", err)
 	}
 	defer f.Close()
 

data/TextLogging.EventACK.central-ship1

@@ -0,0 +1,22 @@
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing

data/TextLogging.EventACK.central-ship2

@@ -0,0 +1,6 @@
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing
+some message sent from a ship for testing

etc/config.toml

@@ -7,3 +7,4 @@ NodeName = "central"
 ProfilingPort = ""
 PromHostAndPort = ":2112"
 PublisherServiceSayhello = 0
+SubscribersDataFolder = "./data"

getmessagefromfile.go

@@ -98,7 +98,7 @@ func readTruncateMessageFile(fileName string) ([]byte, error) {
 
 	f, err := os.OpenFile(fileName, os.O_APPEND|os.O_RDWR, os.ModeAppend)
 	if err != nil {
-		log.Printf("Failed to open file %v\n", err)
+		log.Printf("error: readTruncateMessageFile: Failed to open file: %v\n", err)
 		return nil, err
 	}
 	defer f.Close()

publisher.go

@@ -49,7 +49,7 @@ func (s *server) processNewMessages(dbFileName string, newSAM chan []subjectAndM
 				log.Printf("error: the method do not exist: %v\n", sam.Message.Method)
 				continue
 			}
-			if !s.commandOrEventAvailable.CheckIfExists(sam.Subject.CommandOrEvent) {
+			if !s.commandOrEventAvailable.CheckIfExists(sam.Subject.CommandOrEvent, sam.Subject) {
 				log.Printf("error: the command or evnt do not exist: %v\n", sam.Subject.CommandOrEvent)
 				continue
 			}
@@ -69,7 +69,7 @@ func (s *server) processNewMessages(dbFileName string, newSAM chan []subjectAndM
 			// Are there already a process for that subject, put the
 			// message on that processes incomming message channel.
 			if ok {
-				log.Printf("info: found the specific subject: %v\n", subjName)
+				log.Printf("info: processNewMessages: found the specific subject: %v\n", subjName)
 				s.processes[pn].subject.messageCh <- m
 
 				// If no process to handle the specific subject exist,
@@ -77,7 +77,7 @@ func (s *server) processNewMessages(dbFileName string, newSAM chan []subjectAndM
 			} else {
 				// If a publisher process do not exist for the given subject, create it, and
 				// by using the goto at the end redo the process for this specific message.
-				log.Printf("info: did not find that specific subject, starting new process for subject: %v\n", subjName)
+				log.Printf("info: processNewMessages: did not find that specific subject, starting new process for subject: %v\n", subjName)
 
 				sub := newSubject(sam.Subject.Method, sam.Subject.CommandOrEvent, sam.Subject.ToNode)
 				proc := s.processPrepareNew(sub, s.errorKernel.errorCh, processKindPublisher, nil)

ringbuffer.go

@@ -110,7 +110,7 @@ func (r *ringBuffer) fillBuffer(inCh chan subjectAndMessage, samValueBucket stri
 	for v := range inCh {
 
 		// Check if the command or event exists in commandOrEvent.go
-		if !coeAvailable.CheckIfExists(v.CommandOrEvent) {
+		if !coeAvailable.CheckIfExists(v.CommandOrEvent, v.Subject) {
 			log.Printf("error: the event or command type do not exist, so this message will not be put on the buffer to be processed. Check the syntax used in the json file for the message. Allowed values are : %v\n", coeAvailableValues)
 
 			fmt.Println()
@@ -159,8 +159,6 @@ func (r *ringBuffer) fillBuffer(inCh chan subjectAndMessage, samValueBucket stri
 		// Increment index, and store the new value to the database.
 		r.mu.Lock()
 		r.totalMessagesIndex++
-		fmt.Printf("*** NEXT INDEX NUMBER INCREMENTED: %v\n", r.totalMessagesIndex)
-		fmt.Println("---------------------------------------------------------")
 		r.dbUpdate(r.db, indexValueBucket, "index", []byte(strconv.Itoa(r.totalMessagesIndex)))
 		r.mu.Unlock()
 	}
@@ -193,15 +191,12 @@ func (r *ringBuffer) processBufferMessages(samValueBucket string, outCh chan sam
 			// Listen on the done channel here , so a go routine handling the
 			// message will be able to signal back here that the message have
 			// been processed, and that we then can delete it out of the K/V Store.
-			fmt.Printf("#.#.#.#.#.#.#.# Before DONE: %v\n", v)
 			<-v.Data.done
-			log.Printf("info: done with message %v\n", v.ID)
-			fmt.Printf("#.#.#.#.#.#.#.# Got DONE: %v\n", v)
+			log.Printf("info: processBufferMessages: done with message, deleting key from bucket, %v\n", v.ID)
 
 			// Since we are now done with the specific message we can delete
 			// it out of the K/V Store.
 			r.deleteKeyFromBucket(samValueBucket, strconv.Itoa(v.ID))
-			log.Printf("info: deleting key %v from bucket\n", v.ID)
 
 			r.permStore <- fmt.Sprintf("%v : %+v\n", time.Now().UTC(), v)
 
@@ -316,7 +311,7 @@ func (r *ringBuffer) getIndexValue(indexBucket string) int {
 		log.Printf("error: getIndexValue: strconv.Atoi : %v\n", err)
 	}
 
-	fmt.Printf("**** RETURNING INDEX, WITH VALUE = %v\n", index)
+	fmt.Printf("ringBuffer.getIndexValue: got index value = %v\n", index)
 
 	return index
 }
@@ -377,7 +372,7 @@ func (r *ringBuffer) startPermanentStore() {
 	const storeFile string = "store.log"
 	f, err := os.OpenFile(storeFile, os.O_APPEND|os.O_RDWR|os.O_CREATE, 0600)
 	if err != nil {
-		log.Printf("Failed to open file %v\n", err)
+		log.Printf("error: startPermanentStore: failed to open file: %v\n", err)
 	}
 	defer f.Close()
 

server.go

@@ -6,6 +6,7 @@ import (
 	"encoding/gob"
 	"fmt"
 	"log"
+	"os"
 	"sync"
 	"time"
 
@@ -93,6 +94,22 @@ func NewServer(c *Configuration) (*server, error) {
 		defaultMessageRetries:   c.DefaultMessageRetries,
 	}
 
+	// Create the default data folder for where subscribers should
+	// write it's data if needed.
+
+	// Check if data folder exist, and create it if needed.
+	if _, err := os.Stat(c.SubscribersDataFolder); os.IsNotExist(err) {
+		if c.SubscribersDataFolder == "" {
+			return nil, fmt.Errorf("error: subscribersDataFolder value is empty, you need to provide the config or the flag value at startup %v: %v", c.SubscribersDataFolder, err)
+		}
+		err := os.Mkdir(c.SubscribersDataFolder, 0700)
+		if err != nil {
+			return nil, fmt.Errorf("error: failed to create directory %v: %v", c.SubscribersDataFolder, err)
+		}
+
+		log.Printf("info: Creating subscribers data folder at %v\n", c.SubscribersDataFolder)
+	}
+
 	return s, nil
 
 }
@@ -113,11 +130,6 @@ func (s *server) Start() {
 	// Start the checking the input file for new messages from operator.
 	go s.getMessagesFromFile("./", "inmsg.txt", s.newMessagesCh)
 
-	// Start the textLogging service that will run on the subscribers
-	// TODO: This should only be started if the flag value provided when
-	// starting asks to subscribe to TextLogging events.
-	go s.subscriberServices.startTextLogging()
-
 	// if enabled, start the sayHello I'm here service at the given interval
 	if s.publisherServices.sayHelloPublisher.interval != 0 {
 		go s.publisherServices.sayHelloPublisher.start(s.newMessagesCh, node(s.nodeName))
@@ -286,7 +298,7 @@ func (s *server) messageDeliverNats(proc process, message Message) {
 
 		// If the message is an ACK type of message we must check that a
 		// reply, and if it is not we don't wait here at all.
-		fmt.Printf("---- MESSAGE : %v\n", message)
+		fmt.Printf("info: messageDeliverNats: preparing to send message: %v\n", message)
 		if proc.subject.CommandOrEvent == CommandACK || proc.subject.CommandOrEvent == EventACK {
 			// Wait up until 10 seconds for a reply,
 			// continue and resend if to reply received.
@@ -345,43 +357,31 @@ func (s *server) subscriberHandler(natsConn *nats.Conn, thisNode string, msg *na
 	// method etc.
 	switch {
 	case proc.subject.CommandOrEvent == CommandACK || proc.subject.CommandOrEvent == EventACK:
-		log.Printf("info: subscriberHandler: message.CommandOrEvent received was = %v, preparing to call handler\n", proc.subject.CommandOrEvent)
+		log.Printf("info: subscriberHandler: ACK Message received received, preparing to call handler: %v\n", proc.subject.name())
 		mf, ok := s.methodsAvailable.CheckIfExists(message.Method)
 		if !ok {
 			// TODO: Check how errors should be handled here!!!
 			log.Printf("error: subscriberHandler: method type not available: %v\n", proc.subject.CommandOrEvent)
 		}
-		fmt.Printf("*** DEBUG: BEFORE CALLING HANDLER: ACK\n")
 
 		out := []byte("not allowed from " + message.FromNode)
 		var err error
 
 		// Check if we are allowed to receive from that host
-		_, arOK := proc.allowedReceivers[message.FromNode]
-		//_, arOK2 := proc.allowedReceivers[message.FromNode]
+		_, arOK1 := proc.allowedReceivers[message.FromNode]
+		_, arOK2 := proc.allowedReceivers["*"]
 
-		if arOK {
-			out, err = mf.handler(s, message, thisNode)
+		if arOK1 || arOK2 {
+			out, err = mf.handler(s, proc, message, thisNode)
 
 			if err != nil {
 				// TODO: Send to error kernel ?
 				log.Printf("error: subscriberHandler: failed to execute event: %v\n", err)
-			} else {
-				log.Printf("--- info: we don't allow receiving from: %v\n", message.FromNode)
 			}
+		} else {
+			log.Printf("info: we don't allow receiving from: %v, %v\n", message.FromNode, proc.subject)
 		}
 
-		// if message.FromNode != "central" {
-		// 	log.Printf("--- info: we don't allow receiving from: %v\n", message.FromNode)
-		//
-		// 	out, err = mf.handler(s, message, thisNode)
-		//
-		// 	if err != nil {
-		// 		// TODO: Send to error kernel ?
-		// 		log.Printf("error: subscriberHandler: failed to execute event: %v\n", err)
-		// 	}
-		// }
-
 		// Send a confirmation message back to the publisher
 		natsConn.Publish(msg.Reply, out)
 
@@ -392,7 +392,7 @@ func (s *server) subscriberHandler(natsConn *nats.Conn, thisNode string, msg *na
 			sendErrorLogMessage(s.newMessagesCh, node(thisNode), err)
 		}
 	case proc.subject.CommandOrEvent == CommandNACK || proc.subject.CommandOrEvent == EventNACK:
-		log.Printf("info: subscriberHandler: message.CommandOrEvent received was = %v, preparing to call handler\n", proc.subject.CommandOrEvent)
+		log.Printf("info: subscriberHandler: ACK Message received received, preparing to call handler: %v\n", proc.subject.name())
 		mf, ok := s.methodsAvailable.CheckIfExists(message.Method)
 		if !ok {
 			// TODO: Check how errors should be handled here!!!
@@ -400,8 +400,7 @@ func (s *server) subscriberHandler(natsConn *nats.Conn, thisNode string, msg *na
 		}
 		// since we don't send a reply for a NACK message, we don't care about the
 		// out return when calling mf.handler
-		fmt.Printf("*** DEBUG: BEFORE CALLING HANDLER: NACK\n")
-		_, err := mf.handler(s, message, thisNode)
+		_, err := mf.handler(s, proc, message, thisNode)
 
 		if err != nil {
 			// TODO: Send to error kernel ?

subscriber-services.go

@@ -1,19 +1,10 @@
 package steward
 
-import (
-	"fmt"
-	"log"
-	"os"
-)
-
 // subscriberServices will hold all the helper services needed for
 // the different subcribers. Example of a help service can be a log
 // subscriber needs a way to write logs locally or send them to some
 // other central logging system.
 type subscriberServices struct {
-	// Where we should put the data to write to a log file
-	logCh chan []byte
-
 	// sayHelloNodes are the register where the register where nodes
 	// who have sent an sayHello are stored. Since the sayHello
 	// subscriber is a handler that will be just be called when a
@@ -25,7 +16,6 @@ type subscriberServices struct {
 //newSubscriberServices will prepare and return a *subscriberServices
 func newSubscriberServices() *subscriberServices {
 	s := subscriberServices{
-		logCh:         make(chan []byte),
 		sayHelloNodes: make(map[node]struct{}),
 	}
 
@@ -33,26 +23,3 @@ func newSubscriberServices() *subscriberServices {
 }
 
 // ---
-
-// startTextLogging will open a file ready for writing log messages to,
-// and the input for writing to the file is given via the logCh argument.
-func (s *subscriberServices) startTextLogging() {
-	fileName := "./textlogging.log"
-
-	f, err := os.OpenFile(fileName, os.O_APPEND|os.O_RDWR|os.O_CREATE, os.ModeAppend)
-	if err != nil {
-		log.Printf("Failed to open file %v\n", err)
-		return
-	}
-	defer f.Close()
-
-	for b := range s.logCh {
-		fmt.Printf("***** Trying to write to file : %s\n\n", b)
-		_, err := f.Write(b)
-		f.Sync()
-		if err != nil {
-			log.Printf("Failed to open file %v\n", err)
-		}
-	}
-
-}

subscriberMethodTypes.go

@@ -35,7 +35,9 @@ package steward
 import (
 	"fmt"
 	"log"
+	"os"
 	"os/exec"
+	"path/filepath"
 
 	"github.com/prometheus/client_golang/prometheus"
 )
@@ -120,7 +122,7 @@ func (ma MethodsAvailable) CheckIfExists(m Method) (methodHandler, bool) {
 // ------------------------------------------------------------
 
 type methodHandler interface {
-	handler(server *server, message Message, node string) ([]byte, error)
+	handler(server *server, proc process, message Message, node string) ([]byte, error)
 	getKind() CommandOrEvent
 }
 
@@ -134,7 +136,7 @@ func (m methodCommandCLICommand) getKind() CommandOrEvent {
 	return m.commandOrEvent
 }
 
-func (m methodCommandCLICommand) handler(s *server, message Message, node string) ([]byte, error) {
+func (m methodCommandCLICommand) handler(s *server, proc process, message Message, node string) ([]byte, error) {
 	// Since the command to execute is at the first position in the
 	// slice we need to slice it out. The arguments are at the
 	// remaining positions.
@@ -161,9 +163,29 @@ func (m methodEventTextLogging) getKind() CommandOrEvent {
 	return m.commandOrEvent
 }
 
-func (m methodEventTextLogging) handler(s *server, message Message, node string) ([]byte, error) {
+func (m methodEventTextLogging) handler(s *server, proc process, message Message, node string) ([]byte, error) {
+	sub := Subject{
+		ToNode:         string(message.ToNode),
+		CommandOrEvent: proc.subject.CommandOrEvent,
+		Method:         message.Method,
+	}
+
+	logFile := filepath.Join(s.configuration.SubscribersDataFolder, string(sub.name())+"-"+string(message.FromNode))
+	f, err := os.OpenFile(logFile, os.O_APPEND|os.O_RDWR|os.O_CREATE, os.ModeAppend)
+	if err != nil {
+		log.Printf("error: methodEventTextLogging.handler: failed to open file: %v\n", err)
+		return nil, err
+	}
+	defer f.Close()
+
 	for _, d := range message.Data {
-		s.subscriberServices.logCh <- []byte(d)
+		_, err := f.Write([]byte(d))
+		f.Sync()
+		if err != nil {
+			log.Printf("error: methodEventTextLogging.handler: failed to write to file: %v\n", err)
+		}
+
+		//s.subscriberServices.logCh <- []byte(d)
 	}
 
 	outMsg := []byte("confirmed from: " + node + ": " + fmt.Sprint(message.ID))
@@ -180,7 +202,7 @@ func (m methodEventSayHello) getKind() CommandOrEvent {
 	return m.commandOrEvent
 }
 
-func (m methodEventSayHello) handler(s *server, message Message, node string) ([]byte, error) {
+func (m methodEventSayHello) handler(s *server, proc process, message Message, node string) ([]byte, error) {
 	log.Printf("<--- Received hello from %v \n", message.FromNode)
 	// Since the handler is only called to handle a specific type of message we need
 	// to store it elsewhere, and choice for now is under s.metrics.sayHelloNodes
@@ -208,9 +230,7 @@ func (m methodEventErrorLog) getKind() CommandOrEvent {
 	return m.commandOrEvent
 }
 
-func (m methodEventErrorLog) handler(s *server, message Message, node string) ([]byte, error) {
-	log.Printf("----------------------------------------------------------------------------..\n")
-	log.Printf("Received error from: %v, containing: %v", message.FromNode, message.Data)
-	log.Printf("----------------------------------------------------------------------------..\n")
+func (m methodEventErrorLog) handler(s *server, proc process, message Message, node string) ([]byte, error) {
+	log.Printf("<--- Received error from: %v, containing: %v", message.FromNode, message.Data)
 	return nil, nil
 }

subscribers.go

@@ -37,7 +37,7 @@ func (s *server) subscribersStart() {
 	{
 		fmt.Printf("Starting textlogging subscriber: %#v\n", s.nodeName)
 		sub := newSubject(TextLogging, EventACK, s.nodeName)
-		proc := s.processPrepareNew(sub, s.errorKernel.errorCh, processKindSubscriber, []node{"central"})
+		proc := s.processPrepareNew(sub, s.errorKernel.errorCh, processKindSubscriber, []node{"*"})
 		// fmt.Printf("*** %#v\n", proc)
 		go s.spawnWorkerProcess(proc)
 	}