mirrors/ctrl
mirrors/ctrl
1
0
Fork 0
mirror of https://github.com/postmannen/ctrl.git synced 2024-12-14 12:37:31 +00:00
Code Activity

updated package info

Browse source 
updated references
removed tui client
removed ringbuffer persist store
removed ringbuffer
enabled audit logging
moved audit logging to message readers
disabled goreleaser
update readme, cbor, zstd
removed request type ping and pong
update readme
testing with cmd.WaitDelay for clicommand
fixed readme
removed ringbuffer flag
default serialization set to cbor, default compression set to zstd, fixed race,
removed event type ack and nack, also removed from subject. Fixed file stat error for copy log file
removed remaining elements of the event type
removed comments
renamed toRingbufferCh to samToSendCh
renamed directSAMSCh ro samSendLocalCh
removed handler interface
agpl3 license
added license-change.md
This commit is contained in:
postmannen 2023-10-04 22:58:42 +02:00
parent c3b41d2954
commit 69995f76ca
42 changed files with 1109 additions and 2932 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
.goreleaser.yaml → .notinuse-goreleaser.yaml
View file

@ -8,8 +8,8 @@ builds:
- CGO_ENABLED=0
goos:
- linux
main: ./cmd/steward/main.go
binary: steward
main: ./cmd/ctrl/main.go
binary: ctrl
checksum:
name_template: "checksums.txt"
snapshot:
@ -36,23 +36,23 @@ dockers:
- "--label=org.opencontainers.image.version={{.Version}}"
nfpms:
- vendor: Raa Labs AS
homepage: https://github.com/raalabs/steward
homepage: https://github.com/postmannen/ctrl
maintainer: Bjørn Tore Svinningen <postmannen@gmail.com>
description: |-
Infrastructure controller
license: MIT
formats:
- rpm
bindir: /usr/local/steward/
bindir: /usr/local/ctrl/
contents:
- src: LICENSE
dst: /usr/share/doc/steward/LICENSE
dst: /usr/share/doc/ctrl/LICENSE
file_info:
owner: root
group: root
mode: 0644
- src: README.md
dst: /usr/share/doc/steward/README.md
dst: /usr/share/doc/ctrl/README.md
file_info:
owner: root
group: root

11
Dockerfile
View file

@ -5,9 +5,9 @@ RUN apk --no-cache add build-base git gcc
RUN mkdir -p /build
COPY ./ /build/
WORKDIR /build/cmd/steward/
WORKDIR /build/cmd/ctrl/
RUN go version
RUN go build -o steward
RUN go build -o ctrl
# final stage
FROM alpine
@ -15,7 +15,7 @@ FROM alpine
RUN apk update && apk add curl && apk add nmap
WORKDIR /app
COPY --from=build-env /build/cmd/steward/steward /app/
COPY --from=build-env /build/cmd/ctrl/ctrl /app/
ENV RING_BUFFER_PERSIST_STORE "1"
ENV RING_BUFFER_SIZE "1000"
@ -48,7 +48,6 @@ ENV COMPRESSION ""
ENV SERIALIZATION ""
ENV SET_BLOCK_PROFILE_RATE "0"
ENV ENABLE_SOCKET "1"
ENV ENABLE_TUI "0"
ENV ENABLE_SIGNATURE_CHECK "0"
ENV ENABLE_ACL_CHECK "0"
ENV IS_CENTRAL_AUTH "0"
@ -75,7 +74,7 @@ ENV START_SUB_REQ_HTTP_GET_SCHEDULED ""
ENV START_SUB_REQ_TAIL_FILE ""
ENV START_SUB_REQ_CLI_COMMAND_CONT ""
CMD ["ash","-c","env CONFIGFOLDER=./etc/ /app/steward\
CMD ["ash","-c","env CONFIGFOLDER=./etc/ /app/ctrl\
-ringBufferPersistStore=${RING_BUFFER_PERSIST_STORE}\
-ringBufferSize=${RING_BUFFER_SIZE}\
-socketFolder=${SOCKET_FOLDER}\
@ -106,7 +105,6 @@ CMD ["ash","-c","env CONFIGFOLDER=./etc/ /app/steward\
-serialization=${SERIALIZATION}\
-setBlockProfileRate=${SET_BLOCK_PROFILE_RATE}\
-enableSocket=${ENABLE_SOCKET}\
-enableTUI=${ENABLE_TUI}\
-enableSignatureCheck=${ENABLE_SIGNATURE_CHECK}\
-enableAclCheck=${ENABLE_ACL_CHECK}\
-isCentralAuth=${IS_CENTRAL_AUTH}\
@ -122,7 +120,6 @@ CMD ["ash","-c","env CONFIGFOLDER=./etc/ /app/steward\
-startSubREQCopySrc=${START_SUB_REQ_COPY_SRC}\
-startSubREQCopyDst=${START_SUB_REQ_COPY_DST}\
-startSubREQToFileNACK=${START_SUB_REQ_TO_FILE_NACK}\
-startSubREQPing=${START_SUB_REQ_PING}\
-startSubREQPong=${START_SUB_REQ_PONG}\
-startSubREQCliCommand=${START_SUB_REQ_CLI_COMMAND}\
-startSubREQToConsole=${START_SUB_REQ_TO_CONSOLE}\

674
LICENSE
View file

@ -1,20 +1,660 @@
Copyright (c) 2022 Bjørn Tore Svinningen, RaaLabs.
# GNU AFFERO GENERAL PUBLIC LICENSE
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
Version 3, 19 November 2007
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
Copyright (C) 2007 Free Software Foundation, Inc.
<https://fsf.org/>
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Everyone is permitted to copy and distribute verbatim copies of this
license document, but changing it is not allowed.
## Preamble
The GNU Affero General Public License is a free, copyleft license for
software and other kinds of works, specifically designed to ensure
cooperation with the community in the case of network server software.
The licenses for most software and other practical works are designed
to take away your freedom to share and change the works. By contrast,
our General Public Licenses are intended to guarantee your freedom to
share and change all versions of a program--to make sure it remains
free software for all its users.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
them if you wish), that you receive source code or can get it if you
want it, that you can change the software or use pieces of it in new
free programs, and that you know you can do these things.
Developers that use our General Public Licenses protect your rights
with two steps: (1) assert copyright on the software, and (2) offer
you this License which gives you legal permission to copy, distribute
and/or modify the software.
A secondary benefit of defending all users' freedom is that
improvements made in alternate versions of the program, if they
receive widespread use, become available for other developers to
incorporate. Many developers of free software are heartened and
encouraged by the resulting cooperation. However, in the case of
software used on network servers, this result may fail to come about.
The GNU General Public License permits making a modified version and
letting the public access it on a server without ever releasing its
source code to the public.
The GNU Affero General Public License is designed specifically to
ensure that, in such cases, the modified source code becomes available
to the community. It requires the operator of a network server to
provide the source code of the modified version running there to the
users of that server. Therefore, public use of a modified version, on
a publicly accessible server, gives the public access to the source
code of the modified version.
An older license, called the Affero General Public License and
published by Affero, was designed to accomplish similar goals. This is
a different license, not a version of the Affero GPL, but Affero has
released a new version of the Affero GPL which permits relicensing
under this license.
The precise terms and conditions for copying, distribution and
modification follow.
## TERMS AND CONDITIONS
### 0. Definitions
"This License" refers to version 3 of the GNU Affero General Public
License.
"Copyright" also means copyright-like laws that apply to other kinds
of works, such as semiconductor masks.
"The Program" refers to any copyrightable work licensed under this
License. Each licensee is addressed as "you". "Licensees" and
"recipients" may be individuals or organizations.
To "modify" a work means to copy from or adapt all or part of the work
in a fashion requiring copyright permission, other than the making of
an exact copy. The resulting work is called a "modified version" of
the earlier work or a work "based on" the earlier work.
A "covered work" means either the unmodified Program or a work based
on the Program.
To "propagate" a work means to do anything with it that, without
permission, would make you directly or secondarily liable for
infringement under applicable copyright law, except executing it on a
computer or modifying a private copy. Propagation includes copying,
distribution (with or without modification), making available to the
public, and in some countries other activities as well.
To "convey" a work means any kind of propagation that enables other
parties to make or receive copies. Mere interaction with a user
through a computer network, with no transfer of a copy, is not
conveying.
An interactive user interface displays "Appropriate Legal Notices" to
the extent that it includes a convenient and prominently visible
feature that (1) displays an appropriate copyright notice, and (2)
tells the user that there is no warranty for the work (except to the
extent that warranties are provided), that licensees may convey the
work under this License, and how to view a copy of this License. If
the interface presents a list of user commands or options, such as a
menu, a prominent item in the list meets this criterion.
### 1. Source Code
The "source code" for a work means the preferred form of the work for
making modifications to it. "Object code" means any non-source form of
a work.
A "Standard Interface" means an interface that either is an official
standard defined by a recognized standards body, or, in the case of
interfaces specified for a particular programming language, one that
is widely used among developers working in that language.
The "System Libraries" of an executable work include anything, other
than the work as a whole, that (a) is included in the normal form of
packaging a Major Component, but which is not part of that Major
Component, and (b) serves only to enable use of the work with that
Major Component, or to implement a Standard Interface for which an
implementation is available to the public in source code form. A
"Major Component", in this context, means a major essential component
(kernel, window system, and so on) of the specific operating system
(if any) on which the executable work runs, or a compiler used to
produce the work, or an object code interpreter used to run it.
The "Corresponding Source" for a work in object code form means all
the source code needed to generate, install, and (for an executable
work) run the object code and to modify the work, including scripts to
control those activities. However, it does not include the work's
System Libraries, or general-purpose tools or generally available free
programs which are used unmodified in performing those activities but
which are not part of the work. For example, Corresponding Source
includes interface definition files associated with source files for
the work, and the source code for shared libraries and dynamically
linked subprograms that the work is specifically designed to require,
such as by intimate data communication or control flow between those
subprograms and other parts of the work.
The Corresponding Source need not include anything that users can
regenerate automatically from other parts of the Corresponding Source.
The Corresponding Source for a work in source code form is that same
work.
### 2. Basic Permissions
All rights granted under this License are granted for the term of
copyright on the Program, and are irrevocable provided the stated
conditions are met. This License explicitly affirms your unlimited
permission to run the unmodified Program. The output from running a
covered work is covered by this License only if the output, given its
content, constitutes a covered work. This License acknowledges your
rights of fair use or other equivalent, as provided by copyright law.
You may make, run and propagate covered works that you do not convey,
without conditions so long as your license otherwise remains in force.
You may convey covered works to others for the sole purpose of having
them make modifications exclusively for you, or provide you with
facilities for running those works, provided that you comply with the
terms of this License in conveying all material for which you do not
control copyright. Those thus making or running the covered works for
you must do so exclusively on your behalf, under your direction and
control, on terms that prohibit them from making any copies of your
copyrighted material outside their relationship with you.
Conveying under any other circumstances is permitted solely under the
conditions stated below. Sublicensing is not allowed; section 10 makes
it unnecessary.
### 3. Protecting Users' Legal Rights From Anti-Circumvention Law
No covered work shall be deemed part of an effective technological
measure under any applicable law fulfilling obligations under article
11 of the WIPO copyright treaty adopted on 20 December 1996, or
similar laws prohibiting or restricting circumvention of such
measures.
When you convey a covered work, you waive any legal power to forbid
circumvention of technological measures to the extent such
circumvention is effected by exercising rights under this License with
respect to the covered work, and you disclaim any intention to limit
operation or modification of the work as a means of enforcing, against
the work's users, your or third parties' legal rights to forbid
circumvention of technological measures.
### 4. Conveying Verbatim Copies
You may convey verbatim copies of the Program's source code as you
receive it, in any medium, provided that you conspicuously and
appropriately publish on each copy an appropriate copyright notice;
keep intact all notices stating that this License and any
non-permissive terms added in accord with section 7 apply to the code;
keep intact all notices of the absence of any warranty; and give all
recipients a copy of this License along with the Program.
You may charge any price or no price for each copy that you convey,
and you may offer support or warranty protection for a fee.
### 5. Conveying Modified Source Versions
You may convey a work based on the Program, or the modifications to
produce it from the Program, in the form of source code under the
terms of section 4, provided that you also meet all of these
conditions:
- a) The work must carry prominent notices stating that you modified
it, and giving a relevant date.
- b) The work must carry prominent notices stating that it is
released under this License and any conditions added under
section 7. This requirement modifies the requirement in section 4
to "keep intact all notices".
- c) You must license the entire work, as a whole, under this
License to anyone who comes into possession of a copy. This
License will therefore apply, along with any applicable section 7
additional terms, to the whole of the work, and all its parts,
regardless of how they are packaged. This License gives no
permission to license the work in any other way, but it does not
invalidate such permission if you have separately received it.
- d) If the work has interactive user interfaces, each must display
Appropriate Legal Notices; however, if the Program has interactive
interfaces that do not display Appropriate Legal Notices, your
work need not make them do so.
A compilation of a covered work with other separate and independent
works, which are not by their nature extensions of the covered work,
and which are not combined with it such as to form a larger program,
in or on a volume of a storage or distribution medium, is called an
"aggregate" if the compilation and its resulting copyright are not
used to limit the access or legal rights of the compilation's users
beyond what the individual works permit. Inclusion of a covered work
in an aggregate does not cause this License to apply to the other
parts of the aggregate.
### 6. Conveying Non-Source Forms
You may convey a covered work in object code form under the terms of
sections 4 and 5, provided that you also convey the machine-readable
Corresponding Source under the terms of this License, in one of these
ways:
- a) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by the
Corresponding Source fixed on a durable physical medium
customarily used for software interchange.
- b) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by a
written offer, valid for at least three years and valid for as
long as you offer spare parts or customer support for that product
model, to give anyone who possesses the object code either (1) a
copy of the Corresponding Source for all the software in the
product that is covered by this License, on a durable physical
medium customarily used for software interchange, for a price no
more than your reasonable cost of physically performing this
conveying of source, or (2) access to copy the Corresponding
Source from a network server at no charge.
- c) Convey individual copies of the object code with a copy of the
written offer to provide the Corresponding Source. This
alternative is allowed only occasionally and noncommercially, and
only if you received the object code with such an offer, in accord
with subsection 6b.
- d) Convey the object code by offering access from a designated
place (gratis or for a charge), and offer equivalent access to the
Corresponding Source in the same way through the same place at no
further charge. You need not require recipients to copy the
Corresponding Source along with the object code. If the place to
copy the object code is a network server, the Corresponding Source
may be on a different server (operated by you or a third party)
that supports equivalent copying facilities, provided you maintain
clear directions next to the object code saying where to find the
Corresponding Source. Regardless of what server hosts the
Corresponding Source, you remain obligated to ensure that it is
available for as long as needed to satisfy these requirements.
- e) Convey the object code using peer-to-peer transmission,
provided you inform other peers where the object code and
Corresponding Source of the work are being offered to the general
public at no charge under subsection 6d.
A separable portion of the object code, whose source code is excluded
from the Corresponding Source as a System Library, need not be
included in conveying the object code work.
A "User Product" is either (1) a "consumer product", which means any
tangible personal property which is normally used for personal,
family, or household purposes, or (2) anything designed or sold for
incorporation into a dwelling. In determining whether a product is a
consumer product, doubtful cases shall be resolved in favor of
coverage. For a particular product received by a particular user,
"normally used" refers to a typical or common use of that class of
product, regardless of the status of the particular user or of the way
in which the particular user actually uses, or expects or is expected
to use, the product. A product is a consumer product regardless of
whether the product has substantial commercial, industrial or
non-consumer uses, unless such uses represent the only significant
mode of use of the product.
"Installation Information" for a User Product means any methods,
procedures, authorization keys, or other information required to
install and execute modified versions of a covered work in that User
Product from a modified version of its Corresponding Source. The
information must suffice to ensure that the continued functioning of
the modified object code is in no case prevented or interfered with
solely because modification has been made.
If you convey an object code work under this section in, or with, or
specifically for use in, a User Product, and the conveying occurs as
part of a transaction in which the right of possession and use of the
User Product is transferred to the recipient in perpetuity or for a
fixed term (regardless of how the transaction is characterized), the
Corresponding Source conveyed under this section must be accompanied
by the Installation Information. But this requirement does not apply
if neither you nor any third party retains the ability to install
modified object code on the User Product (for example, the work has
been installed in ROM).
The requirement to provide Installation Information does not include a
requirement to continue to provide support service, warranty, or
updates for a work that has been modified or installed by the
recipient, or for the User Product in which it has been modified or
installed. Access to a network may be denied when the modification
itself materially and adversely affects the operation of the network
or violates the rules and protocols for communication across the
network.
Corresponding Source conveyed, and Installation Information provided,
in accord with this section must be in a format that is publicly
documented (and with an implementation available to the public in
source code form), and must require no special password or key for
unpacking, reading or copying.
### 7. Additional Terms
"Additional permissions" are terms that supplement the terms of this
License by making exceptions from one or more of its conditions.
Additional permissions that are applicable to the entire Program shall
be treated as though they were included in this License, to the extent
that they are valid under applicable law. If additional permissions
apply only to part of the Program, that part may be used separately
under those permissions, but the entire Program remains governed by
this License without regard to the additional permissions.
When you convey a copy of a covered work, you may at your option
remove any additional permissions from that copy, or from any part of
it. (Additional permissions may be written to require their own
removal in certain cases when you modify the work.) You may place
additional permissions on material, added by you to a covered work,
for which you have or can give appropriate copyright permission.
Notwithstanding any other provision of this License, for material you
add to a covered work, you may (if authorized by the copyright holders
of that material) supplement the terms of this License with terms:
- a) Disclaiming warranty or limiting liability differently from the
terms of sections 15 and 16 of this License; or
- b) Requiring preservation of specified reasonable legal notices or
author attributions in that material or in the Appropriate Legal
Notices displayed by works containing it; or
- c) Prohibiting misrepresentation of the origin of that material,
or requiring that modified versions of such material be marked in
reasonable ways as different from the original version; or
- d) Limiting the use for publicity purposes of names of licensors
or authors of the material; or
- e) Declining to grant rights under trademark law for use of some
trade names, trademarks, or service marks; or
- f) Requiring indemnification of licensors and authors of that
material by anyone who conveys the material (or modified versions
of it) with contractual assumptions of liability to the recipient,
for any liability that these contractual assumptions directly
impose on those licensors and authors.
All other non-permissive additional terms are considered "further
restrictions" within the meaning of section 10. If the Program as you
received it, or any part of it, contains a notice stating that it is
governed by this License along with a term that is a further
restriction, you may remove that term. If a license document contains
a further restriction but permits relicensing or conveying under this
License, you may add to a covered work material governed by the terms
of that license document, provided that the further restriction does
not survive such relicensing or conveying.
If you add terms to a covered work in accord with this section, you
must place, in the relevant source files, a statement of the
additional terms that apply to those files, or a notice indicating
where to find the applicable terms.
Additional terms, permissive or non-permissive, may be stated in the
form of a separately written license, or stated as exceptions; the
above requirements apply either way.
### 8. Termination
You may not propagate or modify a covered work except as expressly
provided under this License. Any attempt otherwise to propagate or
modify it is void, and will automatically terminate your rights under
this License (including any patent licenses granted under the third
paragraph of section 11).
However, if you cease all violation of this License, then your license
from a particular copyright holder is reinstated (a) provisionally,
unless and until the copyright holder explicitly and finally
terminates your license, and (b) permanently, if the copyright holder
fails to notify you of the violation by some reasonable means prior to
60 days after the cessation.
Moreover, your license from a particular copyright holder is
reinstated permanently if the copyright holder notifies you of the
violation by some reasonable means, this is the first time you have
received notice of violation of this License (for any work) from that
copyright holder, and you cure the violation prior to 30 days after
your receipt of the notice.
Termination of your rights under this section does not terminate the
licenses of parties who have received copies or rights from you under
this License. If your rights have been terminated and not permanently
reinstated, you do not qualify to receive new licenses for the same
material under section 10.
### 9. Acceptance Not Required for Having Copies
You are not required to accept this License in order to receive or run
a copy of the Program. Ancillary propagation of a covered work
occurring solely as a consequence of using peer-to-peer transmission
to receive a copy likewise does not require acceptance. However,
nothing other than this License grants you permission to propagate or
modify any covered work. These actions infringe copyright if you do
not accept this License. Therefore, by modifying or propagating a
covered work, you indicate your acceptance of this License to do so.
### 10. Automatic Licensing of Downstream Recipients
Each time you convey a covered work, the recipient automatically
receives a license from the original licensors, to run, modify and
propagate that work, subject to this License. You are not responsible
for enforcing compliance by third parties with this License.
An "entity transaction" is a transaction transferring control of an
organization, or substantially all assets of one, or subdividing an
organization, or merging organizations. If propagation of a covered
work results from an entity transaction, each party to that
transaction who receives a copy of the work also receives whatever
licenses to the work the party's predecessor in interest had or could
give under the previous paragraph, plus a right to possession of the
Corresponding Source of the work from the predecessor in interest, if
the predecessor has it or can get it with reasonable efforts.
You may not impose any further restrictions on the exercise of the
rights granted or affirmed under this License. For example, you may
not impose a license fee, royalty, or other charge for exercise of
rights granted under this License, and you may not initiate litigation
(including a cross-claim or counterclaim in a lawsuit) alleging that
any patent claim is infringed by making, using, selling, offering for
sale, or importing the Program or any portion of it.
### 11. Patents
A "contributor" is a copyright holder who authorizes use under this
License of the Program or a work on which the Program is based. The
work thus licensed is called the contributor's "contributor version".
A contributor's "essential patent claims" are all patent claims owned
or controlled by the contributor, whether already acquired or
hereafter acquired, that would be infringed by some manner, permitted
by this License, of making, using, or selling its contributor version,
but do not include claims that would be infringed only as a
consequence of further modification of the contributor version. For
purposes of this definition, "control" includes the right to grant
patent sublicenses in a manner consistent with the requirements of
this License.
Each contributor grants you a non-exclusive, worldwide, royalty-free
patent license under the contributor's essential patent claims, to
make, use, sell, offer for sale, import and otherwise run, modify and
propagate the contents of its contributor version.
In the following three paragraphs, a "patent license" is any express
agreement or commitment, however denominated, not to enforce a patent
(such as an express permission to practice a patent or covenant not to
sue for patent infringement). To "grant" such a patent license to a
party means to make such an agreement or commitment not to enforce a
patent against the party.
If you convey a covered work, knowingly relying on a patent license,
and the Corresponding Source of the work is not available for anyone
to copy, free of charge and under the terms of this License, through a
publicly available network server or other readily accessible means,
then you must either (1) cause the Corresponding Source to be so
available, or (2) arrange to deprive yourself of the benefit of the
patent license for this particular work, or (3) arrange, in a manner
consistent with the requirements of this License, to extend the patent
license to downstream recipients. "Knowingly relying" means you have
actual knowledge that, but for the patent license, your conveying the
covered work in a country, or your recipient's use of the covered work
in a country, would infringe one or more identifiable patents in that
country that you have reason to believe are valid.
If, pursuant to or in connection with a single transaction or
arrangement, you convey, or propagate by procuring conveyance of, a
covered work, and grant a patent license to some of the parties
receiving the covered work authorizing them to use, propagate, modify
or convey a specific copy of the covered work, then the patent license
you grant is automatically extended to all recipients of the covered
work and works based on it.
A patent license is "discriminatory" if it does not include within the
scope of its coverage, prohibits the exercise of, or is conditioned on
the non-exercise of one or more of the rights that are specifically
granted under this License. You may not convey a covered work if you
are a party to an arrangement with a third party that is in the
business of distributing software, under which you make payment to the
third party based on the extent of your activity of conveying the
work, and under which the third party grants, to any of the parties
who would receive the covered work from you, a discriminatory patent
license (a) in connection with copies of the covered work conveyed by
you (or copies made from those copies), or (b) primarily for and in
connection with specific products or compilations that contain the
covered work, unless you entered into that arrangement, or that patent
license was granted, prior to 28 March 2007.
Nothing in this License shall be construed as excluding or limiting
any implied license or other defenses to infringement that may
otherwise be available to you under applicable patent law.
### 12. No Surrender of Others' Freedom
If conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot convey a
covered work so as to satisfy simultaneously your obligations under
this License and any other pertinent obligations, then as a
consequence you may not convey it at all. For example, if you agree to
terms that obligate you to collect a royalty for further conveying
from those to whom you convey the Program, the only way you could
satisfy both those terms and this License would be to refrain entirely
from conveying the Program.
### 13. Remote Network Interaction; Use with the GNU General Public License
Notwithstanding any other provision of this License, if you modify the
Program, your modified version must prominently offer all users
interacting with it remotely through a computer network (if your
version supports such interaction) an opportunity to receive the
Corresponding Source of your version by providing access to the
Corresponding Source from a network server at no charge, through some
standard or customary means of facilitating copying of software. This
Corresponding Source shall include the Corresponding Source for any
work covered by version 3 of the GNU General Public License that is
incorporated pursuant to the following paragraph.
Notwithstanding any other provision of this License, you have
permission to link or combine any covered work with a work licensed
under version 3 of the GNU General Public License into a single
combined work, and to convey the resulting work. The terms of this
License will continue to apply to the part which is the covered work,
but the work with which it is combined will remain governed by version
3 of the GNU General Public License.
### 14. Revised Versions of this License
The Free Software Foundation may publish revised and/or new versions
of the GNU Affero General Public License from time to time. Such new
versions will be similar in spirit to the present version, but may
differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the Program
specifies that a certain numbered version of the GNU Affero General
Public License "or any later version" applies to it, you have the
option of following the terms and conditions either of that numbered
version or of any later version published by the Free Software
Foundation. If the Program does not specify a version number of the
GNU Affero General Public License, you may choose any version ever
published by the Free Software Foundation.
If the Program specifies that a proxy can decide which future versions
of the GNU Affero General Public License can be used, that proxy's
public statement of acceptance of a version permanently authorizes you
to choose that version for the Program.
Later license versions may give you additional or different
permissions. However, no additional obligations are imposed on any
author or copyright holder as a result of your choosing to follow a
later version.
### 15. Disclaimer of Warranty
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT
WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND
PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE
DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR
CORRECTION.
### 16. Limitation of Liability
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR
CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES
ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT
NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR
LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM
TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER
PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
### 17. Interpretation of Sections 15 and 16
If the disclaimer of warranty and limitation of liability provided
above cannot be given local legal effect according to their terms,
reviewing courts shall apply local law that most closely approximates
an absolute waiver of all civil liability in connection with the
Program, unless a warranty or assumption of liability accompanies a
copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS
## How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these
terms.
To do so, attach the following notices to the program. It is safest to
attach them to the start of each source file to most effectively state
the exclusion of warranty; and each file should have at least the
"copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
ctrl Copyright (C) 2024 Bjørn Tore Svinningen
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper
mail.
If your software can interact with users remotely through a computer
network, you should also make sure that it provides a way for users to
get its source. For example, if your program is a web application, its
interface could display a "Source" link that leads users to an archive
of the code. There are many ways you could offer source, and different
solutions will be better for different programs; see section 13 for
the specific requirements.
You should also get your employer (if you work as a programmer) or
school, if any, to sign a "copyright disclaimer" for the program, if
necessary. For more information on this, and how to apply and follow
the GNU AGPL, see <https://www.gnu.org/licenses/>.

12
LICENSE-CHANGE.md Normal file
View file

@ -0,0 +1,12 @@
# License change from MIT to AGPL v3.0
Up until commit 5c23fa409ff8f33c201a8876eb1b558030c735e0 on Jun 30 2023 the code in this repository was named Steward and was written for RaaLabs with an MIT license.
Old License info:
MIT, Copyright (c) 2022 Bjørn Tore Svinningen, RaaLabs.
commit 5c23fa409ff8f33c201a8876eb1b558030c735e0
Author: postmannen <postmannen@gmail.com>
Date: Fri Jun 30 05:28:42 2023 +0200
New License info:
After the above mentioned commit the service was forked and renamed to "ctrl" and the license was changed to an AGPL V3.0 License. More information in LICENSE file.

317
README.md
View file

@ -1,30 +1,21 @@
# Ctrl
## Foreword
ctrl is a fork of the code I earlier wrote for RaaLabs called Steward. The original repo was public and written under an MIT license, but in 2023 the original repo was made private, and are no longer avaialable to the public. The goal of this repo is to provide an actively maintained version for improved reliability and stability.
NB: Filing of issues and bug fixes are highly appreciated. Feature requests will genereally not be followed up simply because I don't have the time to review it at this time :)
## Intro
ctrl is a Command & Control backend system for Servers, IOT and Edge platforms where the network link for reaching them can be reliable like local networks, or totally unreliable like satellite links. An end node can even be offline when you give it a command, and ctrl will make sure that the command is delivered when the node comes online.
ctrl is a Command & Control backend system for Servers, IOT and Edge platforms. Simply put, control anything that runs an operating system.
Example use cases:
- Send a specific message to one or many end nodes that will instruct to run scripts or a series of shell commands to change config, restart services and control those systems.
- Send command to one or many end nodes that will instruct to run scripts or a series of shell commands to change config, restart services and control those systems.
- Gather IOT/OT data from both secure and not secure devices and systems, and transfer them encrypted in a secure way over the internet to your central system for handling those data.
- Collect metrics or monitor end nodes and store the result on a central ctrl instance, or pass those data on to another central system for handling metrics or monitoring data.
- Distribute certificates.
As long as you can do something as an operator on in a shell on a system you can do the same with ctrl in a secure way to one or all end nodes (servers) in one go with one single message/command.
**NB** Expect the main branch to have breaking changes. If stability is needed, use the released packages, and read the release notes where changes will be explained.
- [Ctrl](#ctrl)
- [Foreword](#foreword)
- [Intro](#intro)
- [What is it ?](#what-is-it-)
- [Example](#example)
- [Disclaimer](#disclaimer)
- [Overview](#overview)
- [Example of message flow](#example-of-message-flow)
@ -52,7 +43,7 @@ As long as you can do something as an operator on in a shell on a system you can
- [How to send the reply to another node](#how-to-send-the-reply-to-another-node)
- [Use local as the toNode nodename](#use-local-as-the-tonode-nodename)
- [method timeout](#method-timeout)
- [Example](#example)
- [Example for method timeout](#example-for-method-timeout)
- [Schedule a Method in a message to be run several times](#schedule-a-method-in-a-message-to-be-run-several-times)
- [Request Methods](#request-methods)
- [REQOpProcessList](#reqopprocesslist)
@ -66,13 +57,13 @@ As long as you can do something as an operator on in a shell on a system you can
- [REQHello](#reqhello)
- [REQCopySrc](#reqcopysrc)
- [REQErrorLog](#reqerrorlog)
- [Request Methods used for reply messages](#request-methods-used-for-reply-messages)
- [REQNone](#reqnone)
- [REQToConsole](#reqtoconsole)
- [REQToFileAppend](#reqtofileappend)
- [Write to a socket file](#write-to-a-socket-file)
- [REQToFile](#reqtofile)
- [REQToFileNACK](#reqtofilenack)
- [ReqCliCommand](#reqclicommand-1)
- [Write to socket file](#write-to-socket-file)
- [ReqCliCommand as reply method](#reqclicommand-as-reply-method)
- [Errors reporting](#errors-reporting)
- [Prometheus metrics](#prometheus-metrics)
- [Security / Authorization](#security--authorization)
@ -100,19 +91,15 @@ As long as you can do something as an operator on in a shell on a system you can
- [Howto](#howto)
- [Options for running](#options-for-running)
- [How to Run](#how-to-run)
- [Run ctrl in the simplest possible way for testing](#run-ctrl-in-the-simplest-possible-way-for-testing)
- [Nats-server](#nats-server)
- [Install ctrl](#install-ctrl)
- [Build from source](#build-from-source)
- [Download a release binary](#download-a-release-binary)
- [Nats-server](#nats-server)
- [Build ctrl from source](#build-ctrl-from-source)
- [Get it up and running](#get-it-up-and-running)
- [Send messages with ctrl](#send-messages-with-ctrl)
- [Example for starting ctrl with some more options set](#example-for-starting-ctrl-with-some-more-options-set)
- [Example for starting ctrl with some more options set](#example-for-starting-ctrl-with-some-more-options-set)
- [Nkey Authentication](#nkey-authentication)
- [nats-server (the message broker)](#nats-server-the-message-broker)
- [Nats-server config with nkey authentication example](#nats-server-config-with-nkey-authentication-example)
- [Nkey from ED25519 SSH key](#nkey-from-ed25519-ssh-key)
- [Message fields explanation](#message-fields-explanation)
- [Nkey from ED25519 SSH key](#nkey-from-ed25519-ssh-key)
- [How to send a Message](#how-to-send-a-message)
- [Send to socket with netcat](#send-to-socket-with-netcat)
- [Sending a command from one Node to Another Node](#sending-a-command-from-one-node-to-another-node)
@ -125,16 +112,11 @@ As long as you can do something as an operator on in a shell on a system you can
- [Naming](#naming)
- [Subject](#subject)
- [Complete subject example](#complete-subject-example)
- [TODO](#todo)
- [Add Op option the remove messages from the queue on nodes](#add-op-option-the-remove-messages-from-the-queue-on-nodes)
- [Appendix-A](#appendix-a)
- [Appendix-B](#appendix-b)
- [History](#history)
## What is it ?
## Example
Command And Control anything like Servers, Containers, VM's or others by creating and sending messages with methods who will describe what to do. ctrl will then take the responsibility for making sure that the message are delivered to the receiver, and that the method specified are executed with the given parameters defined. An example of a message.
An example of a **request method** to feed into the system. All fields are explained in detail further down in the document.
An example of a **request** message to copy into ctrl's **readfolder**.
```json
[
@ -209,10 +191,10 @@ If one process hangs on a long running message method it will not affect the res
### Publisher
1. A message in valid format is appended to one of the input methods. Available inputs are Unix Socket listener, TCP listener, and File Reader.
1. The message is picked up by the system and put on a FIFO ringbuffer.
1. The method type of the message is checked, a subject is created based on the content of the message, and a publisher process to handle the message type for that specific receiving node is started if it does not exist.
1. The message is then serialized to binary format, and sent to the subscriber on the receiving node.
1. If the message is expected to be ACK'ed by the subcriber then the publisher will wait for an ACK if the message was delivered. If an ACK was not received within the defined timeout the message will be resent. The amount of retries are defined within the message.
2. The message is picked up by the system.
3. The method type of the message is checked, a subject is created based on the content of the message, and a publisher process to handle the message type for that specific receiving node is started if it does not exist.
4. The message is then serialized to binary format, and sent to the subscriber on the receiving node.
5. If the message is expected to be ACK'ed by the subcriber then the publisher will wait for an ACK if the message was delivered. If an ACK was not received within the defined timeout the message will be resent. The amount of retries are defined within the message.
### Subscriber
@ -246,13 +228,13 @@ New Request Messages in Json/Yaml format can be delivered by the user to ctrl in
### Error messages from nodes
- Error messages will be sent back to the central error handler upon failure on a node.
- Error messages will be sent back to the central error handler and the originating node upon failure.
```log
Tue Sep 21 09:17:55 2021, info: toNode: ship2, fromNode: central, method: REQOpProcessList: max retries reached, check if node is up and running and if it got a subscriber for the given REQ type
```
The error logs can be read on the central server in the directory `<ctrl-home>/data/errorLog`.
The error logs can be read on the central server in the directory `<ctrl-home>/data/errorLog`, and in the log of the instance the source node.
### Message handling and threads
@ -279,7 +261,7 @@ If a message are **ACK** or **NACK** type are defined by the value of the **ACKT
- Default timeouts to wait for ACK messages and max attempts to retry sending a message are specified upon startup. This can be overridden on the message level.
- Timeouts can be specified on both the **message**, and the **method**.
- Timeouts can be specified on both the **message** delivery, and the **method**.
- A message can have a timeout used for used for when to resend and how many retries.
- If the method triggers a shell command, the command can have its own timeout, allowing process timeout for long/stuck commands, or for telling how long the command is supposed to run.
@ -330,7 +312,7 @@ Instead of solely depending in the ack timeout the **RetryWait** can be used. Re
]
```
This is the same as the previos example, but it will also wait another 10 seconds after it noticed that an ACK was not received before the message is retried.
This is the same as the previous example, but it will also wait another 10 seconds after it noticed that an ACK was not received before the message is retried.
The flow will be like this:
@ -451,7 +433,7 @@ As an example. If You want to place a message on the startup folder of **node1**
#### Use local as the toNode nodename
Since messages used in startup folder are ment to be delivered locally we can simply things a bit by setting the **toNode** field value of the message to **local**.
Since messages used in startup folder are ment to be delivered locally we can simplify things a bit by setting the **toNode** field value of the message to **local**.
```json
[
@ -477,7 +459,7 @@ We can also make the request method run for as long as the ctrl instance itself
This can make sense if you for example wan't to continously ping a host, or continously run a script on a node.
##### Example
##### Example for method timeout
```json
[
@ -750,8 +732,6 @@ Method for receiving error logs for Central error logger.
**NB**: This is **not** to be used by users. Use **REQToFileAppend** instead.
### Request Methods used for reply messages
#### REQNone
Don't send a reply message.
@ -801,6 +781,8 @@ If the value of the **directory** field is not prefixed with `./` or `/` the dir
]
```
##### Write to a socket file
If there is already a file at the specified path with the specified name, and if that file is a socket, then the request method will automatically switch to socket communication and write to the socket instead of normal file writing.
#### REQToFile
@ -822,17 +804,11 @@ If the value of the **directory** field is not prefixed with `./` or `/` the dir
]
```
##### Write to socket file
If there is already a file at the specified path with the specified name, and if that file is a socket, then the request method will automatically switch to socket communication and write to the socket instead of normal file writing.
#### REQToFileNACK
Same as REQToFile, but will not send an ACK when a message is delivered.
#### ReqCliCommand
**ReqCliCommand** is a bit special in that it can be used as both **method** and **replyMethod**
The final result, if any, of the replyMethod will be sent to the central server.
#### ReqCliCommand as reply method
By using the `{{ctrl_DATA}}` you can grab the output of your initial request method, and then use it as input in your reply method.
@ -1016,15 +992,9 @@ The location of the config file are given via an env variable at startup (defaul
The different fields and their type in the config file. The fields of the config file can also be set by providing flag values at startup. Use the `-help` flag to get all the options.
Check [Appendix-A](#appendix-a) for a list of the flags/config options, and their usage.
### How to Run
#### Run ctrl in the simplest possible way for testing
**NB** Running ctrl like this is perfect for testing in a local test environment, but is not something you would wan't to do in production.
##### Nats-server
#### Nats-server
Download the **nats-server** from <https://github.com/nats-io/nats-server/releases/>
@ -1040,19 +1010,13 @@ Start the nats server listening on local interfaces and port 4222.
`./nats-server -D`
##### Install ctrl
You can either build ctrl from source or download from release which is a compiled binary.
##### Build from source
#### Build ctrl from source
ctrl is written in Go, so you need Go installed to compile it. You can get Go at <https://golang.org/dl/>.
When Go is installed:
Clone the repository:
`git clone https://github.com/RaaLabs/ctrl.git`.
`git clone https://github.com/postmannen/ctrl.git`.
Change directory and build:
@ -1061,10 +1025,6 @@ cd ./ctrl/cmd/ctrl
go build -o ctrl
```
###### Download a release binary
Release binaries for several architechures are available at <https://github.com/RaaLabs/ctrl/releases>
##### Get it up and running
**NB:** Remember to run the nats setup above before running the ctrl binary.
@ -1103,7 +1063,7 @@ Example on Linux:
`nc -NU ./tmp/ctrl.sock < reqnone.msg`
#### Example for starting ctrl with some more options set
##### Example for starting ctrl with some more options set
A complete example to start a central node called `central`.
@ -1208,14 +1168,10 @@ The official docs for nkeys can be found here <https://docs.nats.io/nats-server/
More example configurations for the nats-server are located in the [doc](https://github.com/RaaLabs/ctrl/tree/main/doc) folder in this repository.
##### Nkey from ED25519 SSH key
#### Nkey from ED25519 SSH key
ctrl can also use an existing SSH ED25519 private key for authentication with the flag **-nkeyFromED25519SSHKeyFile="full-path-to-ssh-private-key"**. The SSH key will be converted to an Nkey if the option is used. The Seed and the User file will be stored in the **socketFolder** which by default is at **./tmp**
### Message fields explanation
Check [Appendix-B](#appendix-b) for message fields and their explanation.
### How to send a Message
The API for sending a message from one node to another node is by sending a structured JSON or YAML object into a listener port in of of the following ways.
@ -1223,6 +1179,7 @@ The API for sending a message from one node to another node is by sending a stru
- unix socket called `ctrl.sock`. By default lives in the `./tmp` directory
- tcpListener, specify host:port with startup flag, or config file.
- httpListener, specify host:port with startup flag, or config file.
- readfolder, copy messages to send directly into the folder.
#### Send to socket with netcat
@ -1364,7 +1321,7 @@ Or in YAML:
You can save the content to myfile.JSON and append it to the `socket` file:
- `nc -U ./ctrl.sock < example/toShip1-REQCliCommand.json`
- `cp <message-name> <pathto>/readfolder`
## Concepts/Ideas
@ -1404,204 +1361,10 @@ For CliCommand message to a node named "ship1" of type Event and it wants an Ack
`ship1.REQCliCommand.EventACK`
## TODO
## History
### Add Op option the remove messages from the queue on nodes
ctrl is the continuation of the code I earlier wrote for RaaLabs called Steward. The original repo was public with a MIT license, but in October 2023 the original repo was made private, and are no longer avaialable to the public. The goal of this repo is to provide an actively maintained, reliable and stable version. This is also a playground for myself to test out ideas an features for such a service as described earlier.
If messages have been sent, and not picked up by a node it might make sense to have some method to clear messages on a node. This could either be done by message ID, and/or time duration.
This started out as an idea I had for how to control infrastructure. This is the continuation of the same idea, and a project I'm working on free of charge in my own spare time, so please be gentle :)
## Appendix-A
```Go
// RingBufferPermStore enable or disable the persisting of
// messages being processed to local db.
RingBufferPersistStore bool
// RingBufferSize
RingBufferSize int
// The configuration folder on disk
ConfigFolder string
// The folder where the socket file should live
SocketFolder string
// TCP Listener for sending messages to the system
TCPListener string
// HTTP Listener for sending messages to the system
HTTPListener string
// The folder where the database should live
DatabaseFolder string
// some unique string to identify this Edge unit
NodeName string
// the address of the message broker
BrokerAddress string
// NatsConnOptTimeout the timeout for trying the connect to nats broker
NatsConnOptTimeout int
// nats connect retry
NatsConnectRetryInterval int
// NatsReconnectJitter in milliseconds
NatsReconnectJitter int
// NatsReconnectJitterTLS in seconds
NatsReconnectJitterTLS int
// REQKeysRequestUpdateInterval in seconds
REQKeysRequestUpdateInterval int
// REQAclRequestUpdateInterval in seconds
REQAclRequestUpdateInterval int
// The number of the profiling port
ProfilingPort string
// host and port for prometheus listener, e.g. localhost:2112
PromHostAndPort string
// set to true if this is the node that should receive the error log's from other nodes
DefaultMessageTimeout int
// Default value for how long can a request method max be allowed to run.
DefaultMethodTimeout int
// default amount of retries that will be done before a message is thrown away, and out of the system
DefaultMessageRetries int
// Publisher data folder
SubscribersDataFolder string
// central node to receive messages published from nodes
CentralNodeName string
// Path to the certificate of the root CA
RootCAPath string
// Full path to the NKEY's seed file
NkeySeedFile string
// NkeyPublicKey
NkeyPublicKey string `toml:"-"`
// The host and port to expose the data folder
ExposeDataFolder string
// Timeout for error messages
ErrorMessageTimeout int
// Retries for error messages.
ErrorMessageRetries int
// Compression
Compression string
// Serialization
Serialization string
// SetBlockProfileRate for block profiling
SetBlockProfileRate int
// EnableSocket for enabling the creation of a ctrl.sock file
EnableSocket bool
// EnableTUI will enable the Terminal User Interface
EnableTUI bool
// EnableSignatureCheck
EnableSignatureCheck bool
// EnableAclCheck
EnableAclCheck bool
// IsCentralAuth
IsCentralAuth bool
// EnableDebug will also enable printing all the messages received in the errorKernel
// to STDERR.
EnableDebug bool
// KeepPublishersAliveFor number of seconds.
// Timer that will be used for when to remove the sub process
// publisher. The timer is reset each time a message is published with
// the process, so the sub process publisher will not be removed until
// it have not received any messages for the given amount of time.
KeepPublishersAliveFor int
// Make the current node send hello messages to central at given interval in seconds
StartPubREQHello int
// Enable the updates of public keys
EnableKeyUpdates bool
// Enable the updates of acl's
EnableAclUpdates bool
// Start the central error logger.
IsCentralErrorLogger bool
// Subscriber for hello messages
StartSubREQHello bool
// Subscriber for text logging
StartSubREQToFileAppend bool
// Subscriber for writing to file
StartSubREQToFile bool
// Subscriber for writing to file without ACK
StartSubREQToFileNACK bool
// Subscriber for reading files to copy
StartSubREQCopySrc bool
// Subscriber for writing copied files to disk
StartSubREQCopyDst bool
// Subscriber for Echo Request
StartSubREQPing bool
// Subscriber for Echo Reply
StartSubREQPong bool
// Subscriber for CLICommandRequest
StartSubREQCliCommand bool
// Subscriber for REQToConsole
StartSubREQToConsole bool
// Subscriber for REQHttpGet
StartSubREQHttpGet bool
// Subscriber for REQHttpGetScheduled
StartSubREQHttpGetScheduled bool
// Subscriber for tailing log files
StartSubREQTailFile bool
// Subscriber for continously delivery of output from cli commands.
StartSubREQCliCommandCont bool
```
## Appendix-B
```go
// The node to send the message to.
ToNode Node `json:"toNode" yaml:"toNode"`
// ToNodes to specify several hosts to send message to in the
// form of an slice/array.
// The ToNodes field is only a concept that exists when messages
// are injected f.ex. on a socket, and there they are directly
//converted into separate node messages for each node, and from
// there the ToNodes field is not used any more within the system.
// With other words, a message that exists within ctrl is always
// for just for a single node.
ToNodes []Node `json:"toNodes,omitempty" yaml:"toNodes,omitempty"`
// The actual data in the message. This is typically where we
// specify the cli commands to execute on a node, and this is
// also the field where we put the returned data in a reply
// message.
Data []string `json:"data" yaml:"data"`
// Method, what request type to use, like REQCliCommand, REQHttpGet..
Method Method `json:"method" yaml:"method"`
// Additional arguments that might be needed when executing the
// method. Can be f.ex. an ip address if it is a tcp sender, or the
// shell command to execute in a cli session.
MethodArgs []string `json:"methodArgs" yaml:"methodArgs"`
// ReplyMethod, is the method to use for the reply message.
// By default the reply method will be set to log to file, but
// you can override it setting your own here.
ReplyMethod Method `json:"replyMethod" yaml:"replyMethod"`
// Additional arguments that might be needed when executing the reply
// method. Can be f.ex. an ip address if it is a tcp sender, or the
// shell command to execute in a cli session.
ReplyMethodArgs []string `json:"replyMethodArgs" yaml:"replyMethodArgs"`
// IsReply are used to tell that this is a reply message. By default
// the system sends the output of a request method back to the node
// the message originated from. If it is a reply method we want the
// result of the reply message to be sent to the central server, so
// we can use this value if set to swap the toNode, and fromNode
// fields.
IsReply bool `json:"isReply" yaml:"isReply"`
// From what node the message originated
FromNode Node
// ACKTimeout for waiting for an ack message
ACKTimeout int `json:"ACKTimeout" yaml:"ACKTimeout"`
// Resend retries
Retries int `json:"retries" yaml:"retries"`
// The ACK timeout of the new message created via a request event.
ReplyACKTimeout int `json:"replyACKTimeout" yaml:"replyACKTimeout"`
// The retries of the new message created via a request event.
ReplyRetries int `json:"replyRetries" yaml:"replyRetries"`
// Timeout for long a process should be allowed to operate
MethodTimeout int `json:"methodTimeout" yaml:"methodTimeout"`
// Timeout for long a process should be allowed to operate
ReplyMethodTimeout int `json:"replyMethodTimeout" yaml:"replyMethodTimeout"`
// Directory is a string that can be used to create the
//directory structure when saving the result of some method.
// For example "syslog","metrics", or "metrics/mysensor"
// The type is typically used in the handler of a method.
Directory string `json:"directory" yaml:"directory"`
// FileName is used to be able to set a wanted name
// on a file being saved as the result of data being handled
// by a method handler.
FileName string `json:"fileName" yaml:"fileName"`
// PreviousMessage are used for example if a reply message is
// generated and we also need a copy of the details of the the
// initial request message.
PreviousMessage *Message
```
NB: Filing of issues and bug fixes are highly appreciated. Feature requests will genereally not be followed up simply because I don't have the time to review it at this time :

2
central_auth_acl_handling.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
import (
"crypto/sha256"

2
central_auth_key_handling.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
import (
"bytes"

2
central_auth_parser.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
import (
"strings"

2
central_auth_test.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
import (
"fmt"

10
cmd/steward/main.go → cmd/ctrl/main.go
View file

@ -10,14 +10,14 @@ import (
_ "net/http/pprof"
"github.com/RaaLabs/steward"
"github.com/pkg/profile"
"github.com/postmannen/ctrl"
)
// Use ldflags to set version
// env CONFIGFOLDER=./etc/ go run -ldflags "-X main.version=v0.1.10" --race ./cmd/steward/.
// env CONFIGFOLDER=./etc/ go run -ldflags "-X main.version=v0.1.10" --race ./cmd/ctrl/.
// or
// env GOOS=linux GOARCH=amd64 go build -ldflags "-X main.version=v0.1.10" -o steward
// env GOOS=linux GOARCH=amd64 go build -ldflags "-X main.version=v0.1.10" -o ctrl
var version string
func main() {
@ -26,7 +26,7 @@ func main() {
//defer profile.Start(profile.TraceProfile, profile.ProfilePath(".")).Stop()
//defer profile.Start(profile.MemProfile, profile.MemProfileRate(1)).Stop()
c := steward.NewConfiguration()
c := ctrl.NewConfiguration()
err := c.CheckFlags(version)
if err != nil {
log.Printf("%v\n", err)
@ -45,7 +45,7 @@ func main() {
runtime.SetBlockProfileRate(c.SetBlockProfileRate)
}
s, err := steward.NewServer(c, version)
s, err := ctrl.NewServer(c, version)
if err != nil {
log.Printf("%v\n", err)
os.Exit(1)

69
configuration_flags.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
import (
"flag"
@ -17,11 +17,6 @@ import (
// an if check should be added to the checkConfigValues function
// to set default values when reading from config file.
type Configuration struct {
// RingBufferPersistStore, enable or disable the persisting of
// messages being processed to local db.
RingBufferPersistStore bool `comment:"RingBufferPersistStore, enable or disable the persisting of messages being processed to local db"`
// RingBufferSize
RingBufferSize int `comment:"RingBufferSize"`
// ConfigFolder, the location for the configuration folder on disk
ConfigFolder string `comment:"ConfigFolder, the location for the configuration folder on disk"`
// The folder where the socket file should live
@ -86,10 +81,8 @@ type Configuration struct {
Serialization string `comment:"Serialization, supports cbor or gob,default is gob. Enable cbor by setting the string value cbor"`
// SetBlockProfileRate for block profiling
SetBlockProfileRate int `comment:"SetBlockProfileRate for block profiling"`
// EnableSocket for enabling the creation of a steward.sock file
EnableSocket bool `comment:"EnableSocket for enabling the creation of a steward.sock file"`
// EnableTUI will enable the Terminal User Interface
EnableTUI bool `comment:"EnableTUI will enable the Terminal User Interface"`
// EnableSocket for enabling the creation of a ctrl.sock file
EnableSocket bool `comment:"EnableSocket for enabling the creation of a ctrl.sock file"`
// EnableSignatureCheck to enable signature checking
EnableSignatureCheck bool `comment:"EnableSignatureCheck to enable signature checking"`
// EnableAclCheck to enable ACL checking
@ -131,10 +124,6 @@ type Configuration struct {
// Start subscriber for writing copied files to disk
StartSubREQCopyDst bool `comment:"Start subscriber for writing copied files to disk"`
// Start subscriber for Echo Request
StartSubREQPing bool `comment:"Start subscriber for Echo Request"`
// Start subscriber for Echo Reply
StartSubREQPong bool `comment:"Start subscriber for Echo Reply"`
// Start subscriber for CLICommandRequest
StartSubREQCliCommand bool `comment:"Start subscriber for CLICommandRequest"`
// Start subscriber for REQToConsole
StartSubREQToConsole bool `comment:"Start subscriber for REQToConsole"`
@ -187,7 +176,6 @@ type ConfigurationFromFile struct {
Serialization *string
SetBlockProfileRate *int
EnableSocket *bool
EnableTUI *bool
EnableSignatureCheck *bool
EnableAclCheck *bool
IsCentralAuth *bool
@ -206,8 +194,6 @@ type ConfigurationFromFile struct {
StartSubREQToFileNACK *bool
StartSubREQCopySrc *bool
StartSubREQCopyDst *bool
StartSubREQPing *bool
StartSubREQPong *bool
StartSubREQCliCommand *bool
StartSubREQToConsole *bool
StartSubREQHttpGet *bool
@ -226,8 +212,6 @@ func NewConfiguration() *Configuration {
func newConfigurationDefaults() Configuration {
c := Configuration{
ConfigFolder: "./etc/",
RingBufferPersistStore: true,
RingBufferSize: 1000,
SocketFolder: "./tmp",
ReadFolder: "./readfolder",
EnableReadFolder: true,
@ -248,18 +232,17 @@ func newConfigurationDefaults() Configuration {
DefaultMessageRetries: 1,
DefaultMethodTimeout: 10,
SubscribersDataFolder: "./data",
CentralNodeName: "",
CentralNodeName: "central",
RootCAPath: "",
NkeySeedFile: "",
NkeyFromED25519SSHKeyFile: "",
ExposeDataFolder: "",
ErrorMessageTimeout: 60,
ErrorMessageRetries: 10,
Compression: "",
Serialization: "",
Compression: "z",
Serialization: "cbor",
SetBlockProfileRate: 0,
EnableSocket: true,
EnableTUI: false,
EnableSignatureCheck: false,
EnableAclCheck: false,
IsCentralAuth: false,
@ -269,8 +252,8 @@ func newConfigurationDefaults() Configuration {
KeepPublishersAliveFor: 10,
StartPubREQHello: 30,
EnableKeyUpdates: true,
EnableAclUpdates: true,
EnableKeyUpdates: false,
EnableAclUpdates: false,
IsCentralErrorLogger: false,
StartSubREQHello: true,
StartSubREQToFileAppend: true,
@ -278,8 +261,6 @@ func newConfigurationDefaults() Configuration {
StartSubREQToFileNACK: true,
StartSubREQCopySrc: true,
StartSubREQCopyDst: true,
StartSubREQPing: true,
StartSubREQPong: true,
StartSubREQCliCommand: true,
StartSubREQToConsole: true,
StartSubREQHttpGet: true,
@ -296,16 +277,6 @@ func checkConfigValues(cf ConfigurationFromFile) Configuration {
var conf Configuration
cd := newConfigurationDefaults()
if cf.RingBufferSize == nil {
conf.RingBufferSize = cd.RingBufferSize
} else {
conf.RingBufferSize = *cf.RingBufferSize
}
if cf.RingBufferPersistStore == nil {
conf.RingBufferPersistStore = cd.RingBufferPersistStore
} else {
conf.RingBufferPersistStore = *cf.RingBufferPersistStore
}
if cf.ConfigFolder == nil {
conf.ConfigFolder = cd.ConfigFolder
} else {
@ -466,11 +437,6 @@ func checkConfigValues(cf ConfigurationFromFile) Configuration {
} else {
conf.EnableSocket = *cf.EnableSocket
}
if cf.EnableTUI == nil {
conf.EnableTUI = cd.EnableTUI
} else {
conf.EnableTUI = *cf.EnableTUI
}
if cf.EnableSignatureCheck == nil {
conf.EnableSignatureCheck = cd.EnableSignatureCheck
} else {
@ -561,16 +527,6 @@ func checkConfigValues(cf ConfigurationFromFile) Configuration {
} else {
conf.StartSubREQCopyDst = *cf.StartSubREQCopyDst
}
if cf.StartSubREQPing == nil {
conf.StartSubREQPing = cd.StartSubREQPing
} else {
conf.StartSubREQPing = *cf.StartSubREQPing
}
if cf.StartSubREQPong == nil {
conf.StartSubREQPong = cd.StartSubREQPong
} else {
conf.StartSubREQPong = *cf.StartSubREQPong
}
if cf.StartSubREQCliCommand == nil {
conf.StartSubREQCliCommand = cd.StartSubREQCliCommand
} else {
@ -633,9 +589,7 @@ func (c *Configuration) CheckFlags(version string) error {
*c = fc
//flag.StringVar(&c.ConfigFolder, "configFolder", fc.ConfigFolder, "Defaults to ./usr/local/steward/etc/. *NB* This flag is not used, if your config file are located somwhere else than default set the location in an env variable named CONFIGFOLDER")
flag.BoolVar(&c.RingBufferPersistStore, "ringBufferPersistStore", fc.RingBufferPersistStore, "true/false for enabling the persisting of ringbuffer to disk")
flag.IntVar(&c.RingBufferSize, "ringBufferSize", fc.RingBufferSize, "size of the ringbuffer")
//flag.StringVar(&c.ConfigFolder, "configFolder", fc.ConfigFolder, "Defaults to ./usr/local/ctrl/etc/. *NB* This flag is not used, if your config file are located somwhere else than default set the location in an env variable named CONFIGFOLDER")
flag.StringVar(&c.SocketFolder, "socketFolder", fc.SocketFolder, "folder who contains the socket file. Defaults to ./tmp/. If other folder is used this flag must be specified at startup.")
flag.StringVar(&c.ReadFolder, "readFolder", fc.ReadFolder, "folder who contains the readfolder. Defaults to ./readfolder/. If other folder is used this flag must be specified at startup.")
flag.StringVar(&c.TCPListener, "tcpListener", fc.TCPListener, "start up a TCP listener in addition to the Unix Socket, to give messages to the system. e.g. localhost:8888. No value means not to start the listener, which is default. NB: You probably don't want to start this on any other interface than localhost")
@ -665,8 +619,7 @@ func (c *Configuration) CheckFlags(version string) error {
flag.StringVar(&c.Compression, "compression", fc.Compression, "compression method to use. defaults to no compression, z = zstd, g = gzip. Undefined value will default to no compression")
flag.StringVar(&c.Serialization, "serialization", fc.Serialization, "Serialization method to use. defaults to gob, other values are = cbor. Undefined value will default to gob")
flag.IntVar(&c.SetBlockProfileRate, "setBlockProfileRate", fc.SetBlockProfileRate, "Enable block profiling by setting the value to f.ex. 1. 0 = disabled")
flag.BoolVar(&c.EnableSocket, "enableSocket", fc.EnableSocket, "true/false, for enabling the creation of a steward.sock file")
flag.BoolVar(&c.EnableTUI, "enableTUI", fc.EnableTUI, "true/false for enabling the Terminal User Interface")
flag.BoolVar(&c.EnableSocket, "enableSocket", fc.EnableSocket, "true/false, for enabling the creation of ctrl.sock file")
flag.BoolVar(&c.EnableSignatureCheck, "enableSignatureCheck", fc.EnableSignatureCheck, "true/false *TESTING* enable signature checking.")
flag.BoolVar(&c.EnableAclCheck, "enableAclCheck", fc.EnableAclCheck, "true/false *TESTING* enable Acl checking.")
flag.BoolVar(&c.IsCentralAuth, "isCentralAuth", fc.IsCentralAuth, "true/false, *TESTING* is this the central auth server")
@ -690,8 +643,6 @@ func (c *Configuration) CheckFlags(version string) error {
flag.BoolVar(&c.StartSubREQToFileNACK, "startSubREQToFileNACK", fc.StartSubREQToFileNACK, "true/false")
flag.BoolVar(&c.StartSubREQCopySrc, "startSubREQCopySrc", fc.StartSubREQCopySrc, "true/false")
flag.BoolVar(&c.StartSubREQCopyDst, "startSubREQCopyDst", fc.StartSubREQCopyDst, "true/false")
flag.BoolVar(&c.StartSubREQPing, "startSubREQPing", fc.StartSubREQPing, "true/false")
flag.BoolVar(&c.StartSubREQPong, "startSubREQPong", fc.StartSubREQPong, "true/false")
flag.BoolVar(&c.StartSubREQCliCommand, "startSubREQCliCommand", fc.StartSubREQCliCommand, "true/false")
flag.BoolVar(&c.StartSubREQToConsole, "startSubREQToConsole", fc.StartSubREQToConsole, "true/false")
flag.BoolVar(&c.StartSubREQHttpGet, "startSubREQHttpGet", fc.StartSubREQHttpGet, "true/false")

12
doc/example-messages/ping.json
View file

@ -1,12 +0,0 @@
[
{
"directory": "ping",
"fileName":"somefile.ping.log",
"toNode": "ship1",
"data": [""],
"method":"REQPing",
"ACKTimeout":3,
"retries":3,
"methodTimeout": 10
}
]

6
doc/kubernetes/steward.yaml
View file

@ -108,8 +108,6 @@ spec:
value: "0"
- name: ENABLE_SOCKET
value: "1"
- name: ENABLE_TUI
value: "0"
- name: ENABLE_SIGNATURE_CHECK
value: "0"
- name: ENABLE_ACL_CHECK
@ -162,7 +160,7 @@ spec:
- name: MESSAGE_FULL_PATH
value: "/app/message.yaml"
- name: SOCKET_FULL_PATH
value: "/app/tmp/steward.sock"
value: "/app/tmp/ctrl.sock"
- name: INTERVAL
value: "10"
name: stewardwriter1
@ -183,7 +181,7 @@ spec:
- name: MESSAGE_FULL_PATH
value: "/app/message.yaml"
- name: SOCKET_FULL_PATH
value: "/app/tmp/steward.sock"
value: "/app/tmp/ctrl.sock"
- name: INTERVAL
value: "10"
name: stewardwriter2

2
errorkernel.go
View file

@ -5,7 +5,7 @@
// that the action which the message where supposed to trigger
// failed, or that an event where unable to be processed.
package steward
package ctrl
import (
"context"

28
event_type.go
View file

@ -1,28 +0,0 @@
// NB:
// When adding new constants for the Method or event
// types, make sure to also add them to the map
// <Method/Event>Available since the this will be used
// to check if the message values are valid later on.
package steward
// Event describes on the message level if this is
// an ACK or NACK kind of message in the Subject name.
// This field is mainly used to be able to spawn up different
// worker processes based on the Subject name.
// This type is used in both building the subject name, and
// also inside the Message type to describe what kind like
// ACK or NACK it is.
type Event string
const (
// EventACK, wait for the return of an ACK message.
// The sender will wait for an ACK reply message
// to decide if it was succesfully delivered or not.
// If no ACK was received within the timeout, the
// message will be resent the nr. of times specified
// in retries field of the message.
EventACK Event = "EventACK"
// Same as above, but No ACK.
EventNACK Event = "EventNACK"
)

17
go.mod
View file

@ -1,23 +1,21 @@
module github.com/RaaLabs/steward
module github.com/postmannen/ctrl
go 1.20
go 1.21
require (
github.com/fsnotify/fsnotify v1.6.0
github.com/fxamacker/cbor/v2 v2.4.0
github.com/gdamore/tcell/v2 v2.6.0
github.com/fxamacker/cbor/v2 v2.5.0
github.com/go-playground/validator/v10 v10.10.1
github.com/google/uuid v1.3.0
github.com/hpcloud/tail v1.0.0
github.com/jinzhu/copier v0.3.5
github.com/klauspost/compress v1.16.3
github.com/jinzhu/copier v0.4.0
github.com/klauspost/compress v1.17.0
github.com/nats-io/nats-server/v2 v2.8.4
github.com/nats-io/nats.go v1.25.0
github.com/nats-io/nkeys v0.4.4
github.com/pelletier/go-toml/v2 v2.0.7
github.com/pkg/profile v1.7.0
github.com/prometheus/client_golang v1.14.0
github.com/rivo/tview v0.0.0-20230330183452-5796b0cd5c1f
go.etcd.io/bbolt v1.3.7
golang.org/x/crypto v0.7.0
golang.org/x/exp v0.0.0-20230321023759-10a507213a29
@ -28,14 +26,11 @@ require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/felixge/fgprof v0.9.3 // indirect
github.com/gdamore/encoding v1.0.0 // indirect
github.com/go-playground/locales v0.14.0 // indirect
github.com/go-playground/universal-translator v0.18.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/pprof v0.0.0-20230323073829-e72429f035bd // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/minio/highwayhash v1.0.2 // indirect
github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a // indirect
@ -43,10 +38,8 @@ require (
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/x448/float16 v0.8.4 // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/term v0.6.0 // indirect
golang.org/x/text v0.8.0 // indirect
golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
google.golang.org/protobuf v1.30.0 // indirect

46
go.sum
View file

@ -13,12 +13,8 @@ github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88=
github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo=
github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko=
github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
github.com/gdamore/tcell/v2 v2.6.0 h1:OKbluoP9VYmJwZwq/iLb4BxwKcwGthaa1YNBJIyCySg=
github.com/gdamore/tcell/v2 v2.6.0/go.mod h1:be9omFATkdr0D9qewWW3d+MEvl5dha+Etb5y65J2H8Y=
github.com/fxamacker/cbor/v2 v2.5.0 h1:oHsG0V/Q6E/wqTS2O1Cozzsy69nqCiguo5Q1a1ADivE=
github.com/fxamacker/cbor/v2 v2.5.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo=
github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
@ -34,6 +30,7 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
github.com/google/pprof v0.0.0-20230323073829-e72429f035bd h1:r8yyd+DJDmsUhGrRBxH5Pj7KeFK5l+Y3FsgT8keqKtk=
github.com/google/pprof v0.0.0-20230323073829-e72429f035bd/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk=
@ -42,10 +39,10 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg=
github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY=
github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
@ -56,10 +53,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
@ -90,12 +83,6 @@ github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI
github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
github.com/rivo/tview v0.0.0-20230330183452-5796b0cd5c1f h1:vpjWdGBgikHYD4ruBvDINMxwdh5UWVck9yOyrwFktMo=
github.com/rivo/tview v0.0.0-20230330183452-5796b0cd5c1f/go.mod h1:nVwGv4MP47T0jvlk7KuTTjjuSmrGO4JF0iaiNt4bufE=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
@ -110,56 +97,37 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug=
golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U=
golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=

18
message_and_subject.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
import (
"fmt"
@ -18,7 +18,7 @@ type Message struct {
// are injected f.ex. on a socket, and there they are directly
//converted into separate node messages for each node, and from
// there the ToNodes field is not used any more within the system.
// With other words, a message that exists within Steward is always
// With other words, a message that exists within ctrl is always
// for just for a single node.
ToNodes []Node `json:"toNodes,omitempty" yaml:"toNodes,omitempty"`
// The Unique ID of the message
@ -85,12 +85,6 @@ type Message struct {
// Schedule
Schedule []int `json:"schedule" yaml:"schedule"`
// done is used to signal when a message is fully processed.
// This is used for signaling back to the ringbuffer that we are
// done with processing a message, and the message can be removed
// from the ringbuffer and into the time series log.
done chan struct{}
// ctx for the specifix message. Used for for example canceling
// scheduled messages.
// NB: Commented out this field for specific message context
@ -110,8 +104,6 @@ type Node string
type Subject struct {
// node, the name of the node to receive the message.
ToNode string `json:"node" yaml:"toNode"`
// Event, event type like EventACK or EventNACK.
Event Event `json:"event" yaml:"event"`
// method, what is this message doing, etc. CLICommand, Syslog, etc.
Method Method `json:"method" yaml:"method"`
// messageCh is used by publisher kind processes to read new messages
@ -129,7 +121,7 @@ type Subject struct {
func newSubject(method Method, node string) Subject {
// Get the Event type for the Method.
ma := method.GetMethodsAvailable()
mh, ok := ma.CheckIfExists(method)
_, ok := ma.CheckIfExists(method)
//mh, ok := ma.Methodhandlers[method]
if !ok {
log.Printf("error: newSubject: no Event type specified for the method: %v\n", method)
@ -138,7 +130,6 @@ func newSubject(method Method, node string) Subject {
return Subject{
ToNode: node,
Event: mh.getKind(),
Method: method,
messageCh: make(chan Message),
}
@ -154,7 +145,6 @@ func newSubjectNoVerifyHandler(method Method, node string) Subject {
return Subject{
ToNode: node,
Event: EventACK,
Method: method,
messageCh: make(chan Message),
}
@ -165,5 +155,5 @@ type subjectName string
// Return a value of the subjectName for the subject as used with nats subject.
func (s Subject) name() subjectName {
return subjectName(fmt.Sprintf("%s.%s.%s", s.ToNode, s.Method, s.Event))
return subjectName(fmt.Sprintf("%s.%s", s.ToNode, s.Method))
}

35
message_readers.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
import (
"bytes"
@ -16,10 +16,10 @@ import (
"gopkg.in/yaml.v3"
)
// readStartupFolder will check the <workdir>/startup folder when Steward
// readStartupFolder will check the <workdir>/startup folder when ctrl
// starts for messages to process.
// The purpose of the startup folder is that we can define messages on a
// node that will be run when Steward starts up.
// node that will be run when ctrl starts up.
// Messages defined in the startup folder should have the toNode set to
// self, and the from node set to where we want the answer sent. The reason
// for this is that all replies normally pick up the host from the original
@ -113,7 +113,7 @@ func (s *server) readStartupFolder() {
er = fmt.Errorf("%v", string(j))
s.errorKernel.errSend(s.processInitial, Message{}, er, logInfo)
s.directSAMSCh <- sams
s.samSendLocalCh <- sams
}
@ -161,7 +161,7 @@ func (s *server) getFilePaths(dirName string) ([]string, error) {
func (s *server) readSocket() {
// Loop, and wait for new connections.
for {
conn, err := s.StewardSocket.Accept()
conn, err := s.ctrlSocket.Accept()
if err != nil {
er := fmt.Errorf("error: failed to accept conn on socket: %v", err)
s.errorKernel.errSend(s.processInitial, Message{}, er, logError)
@ -211,7 +211,8 @@ func (s *server) readSocket() {
}
// Send the SAM struct to be picked up by the ring buffer.
s.toRingBufferCh <- sams
s.samToSendCh <- sams
s.auditLogCh <- sams
}(conn)
}
@ -289,7 +290,8 @@ func (s *server) readFolder() {
}
// Send the SAM struct to be picked up by the ring buffer.
s.toRingBufferCh <- sams
s.samToSendCh <- sams
s.auditLogCh <- sams
// Delete the file.
err = os.Remove(event.Name)
@ -366,22 +368,23 @@ func (s *server) readTCPListener() {
readBytes = bytes.Trim(readBytes, "\x00")
// unmarshal the JSON into a struct
sam, err := s.convertBytesToSAMs(readBytes)
sams, err := s.convertBytesToSAMs(readBytes)
if err != nil {
er := fmt.Errorf("error: malformed json received on tcp listener: %v", err)
s.errorKernel.errSend(s.processInitial, Message{}, er, logWarning)
return
}
for i := range sam {
for i := range sams {
// Fill in the value for the FromNode field, so the receiver
// can check this field to know where it came from.
sam[i].Message.FromNode = Node(s.nodeName)
sams[i].Message.FromNode = Node(s.nodeName)
}
// Send the SAM struct to be picked up by the ring buffer.
s.toRingBufferCh <- sam
s.samToSendCh <- sams
s.auditLogCh <- sams
}(conn)
}
@ -410,22 +413,23 @@ func (s *server) readHTTPlistenerHandler(w http.ResponseWriter, r *http.Request)
readBytes = bytes.Trim(readBytes, "\x00")
// unmarshal the JSON into a struct
sam, err := s.convertBytesToSAMs(readBytes)
sams, err := s.convertBytesToSAMs(readBytes)
if err != nil {
er := fmt.Errorf("error: malformed json received on HTTPListener: %v", err)
s.errorKernel.errSend(s.processInitial, Message{}, er, logWarning)
return
}
for i := range sam {
for i := range sams {
// Fill in the value for the FromNode field, so the receiver
// can check this field to know where it came from.
sam[i].Message.FromNode = Node(s.nodeName)
sams[i].Message.FromNode = Node(s.nodeName)
}
// Send the SAM struct to be picked up by the ring buffer.
s.toRingBufferCh <- sam
s.samToSendCh <- sams
s.auditLogCh <- sams
}
@ -557,7 +561,6 @@ func newSubjectAndMessage(m Message) (subjectAndMessage, error) {
sub := Subject{
ToNode: string(m.ToNode),
Event: tmpH.getKind(),
Method: m.Method,
messageCh: make(chan Message),
}

36
metrics.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
import (
"fmt"
@ -73,100 +73,100 @@ func newMetrics(hostAndPort string) *metrics {
}
m.promVersion = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "steward_build_version",
Help: "Build version of steward",
Name: "ctrl_build_version",
Help: "Build version of ctrl",
}, []string{"version"},
)
m.promRegistry.MustRegister(m.promVersion)
m.promProcessesTotal = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "steward_processes_total",
Name: "ctrl_processes_total",
Help: "The current number of total running processes",
})
m.promRegistry.MustRegister(m.promProcessesTotal)
m.promProcessesAllRunning = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "steward_processes_all_running",
Name: "ctrl_processes_all_running",
Help: "Name of the running processes",
}, []string{"processName"},
)
m.promRegistry.MustRegister(m.promProcessesAllRunning)
m.promHelloNodesTotal = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "steward_hello_nodes_total",
Name: "ctrl_hello_nodes_total",
Help: "The current number of total nodes who have said hello",
})
m.promRegistry.MustRegister(m.promHelloNodesTotal)
m.promHelloNodesContactLast = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "steward_hello_node_contact_last",
Name: "ctrl_hello_node_contact_last",
Help: "Name of the nodes who have said hello",
}, []string{"nodeName"})
m.promRegistry.MustRegister(m.promHelloNodesContactLast)
m.promMessagesProcessedIDLast = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "steward_messages_processed_id_last",
Name: "ctrl_messages_processed_id_last",
Help: "The last processed id in key value/store db",
})
m.promRegistry.MustRegister(m.promMessagesProcessedIDLast)
m.promRingbufferStalledMessagesTotal = prometheus.NewCounter(prometheus.CounterOpts{
Name: "steward_ringbuffer_stalled_messages_total",
Name: "ctrl_ringbuffer_stalled_messages_total",
Help: "Number of stalled messages in ringbuffer",
})
m.promRegistry.MustRegister(m.promRingbufferStalledMessagesTotal)
m.promInMemoryBufferMessagesCurrent = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "steward_in_memory_buffer_messages_current",
Name: "ctrl_in_memory_buffer_messages_current",
Help: "The current value of messages in memory buffer",
})
m.promRegistry.MustRegister(m.promInMemoryBufferMessagesCurrent)
// Register som metrics for messages delivered by users into the system.
m.promUserMessagesTotal = prometheus.NewCounter(prometheus.CounterOpts{
Name: "steward_user_messages_total",
Name: "ctrl_user_messages_total",
Help: "Number of total messages delivered by users into the system",
})
m.promRegistry.MustRegister(m.promUserMessagesTotal)
m.promNatsDeliveredTotal = prometheus.NewCounter(prometheus.CounterOpts{
Name: "steward_nats_delivered_total",
Name: "ctrl_nats_delivered_total",
Help: "Number of total messages delivered by nats",
})
m.promRegistry.MustRegister(m.promNatsDeliveredTotal)
m.promNatsMessagesFailedACKsTotal = prometheus.NewCounter(prometheus.CounterOpts{
Name: "steward_nats_messages_failed_acks_total",
Name: "ctrl_nats_messages_failed_acks_total",
Help: "Number of messages that never received an ack total",
})
m.promRegistry.MustRegister(m.promNatsMessagesFailedACKsTotal)
m.promNatsMessagesMissedACKsTotal = prometheus.NewCounter(prometheus.CounterOpts{
Name: "steward_nats_messages_missed_acks_total",
Name: "ctrl_nats_messages_missed_acks_total",
Help: "Number of messages missed receiving an ack total",
})
m.promRegistry.MustRegister(m.promNatsMessagesMissedACKsTotal)
m.promErrorMessagesReceivedTotal = prometheus.NewCounter(prometheus.CounterOpts{
Name: "steward_error_messages_received_total",
Name: "ctrl_error_messages_received_total",
Help: "Number of error messages received total",
})
m.promRegistry.MustRegister(m.promErrorMessagesReceivedTotal)
m.promErrorMessagesSentTotal = prometheus.NewCounter(prometheus.CounterOpts{
Name: "steward_error_messages_sent_total",
Name: "ctrl_error_messages_sent_total",
Help: "Number of error messages sent total",
})
m.promRegistry.MustRegister(m.promErrorMessagesSentTotal)
m.promInfoMessagesSentTotal = prometheus.NewCounter(prometheus.CounterOpts{
Name: "steward_info_messages_sent_total",
Name: "ctrl_info_messages_sent_total",
Help: "Number of info messages sent total",
})
m.promRegistry.MustRegister(m.promInfoMessagesSentTotal)
m.promDBMessagesCurrent = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "steward_db_messages_current",
Name: "ctrl_db_messages_current",
Help: "The current value messages in database",
})
m.promRegistry.MustRegister(m.promDBMessagesCurrent)

2
nkey_from_ssh_ed25519_key.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
import (
"bytes"

4
node_auth.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
import (
"crypto/ed25519"
@ -13,7 +13,7 @@ import (
)
// nodeAuth is the structure that holds both keys and acl's
// that the running steward node shall use for authorization.
// that the running ctrl node shall use for authorization.
// It holds a mutex to use when interacting with the map.
type nodeAuth struct {
// ACL that defines where a node is allowed to recieve from.

51
process.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
import (
"bytes"
@ -144,7 +144,7 @@ func newProcess(ctx context.Context, server *server, subject Subject, processKin
processID: pid,
processKind: processKind,
methodsAvailable: method.GetMethodsAvailable(),
toRingbufferCh: server.toRingBufferCh,
toRingbufferCh: server.samToSendCh,
configuration: server.configuration,
processes: server.processes,
natsConn: server.natsConn,
@ -271,7 +271,7 @@ func (p process) startSubscriber() {
}
var (
ErrACKSubscribeRetry = errors.New("steward: retrying to subscribe for ack message")
ErrACKSubscribeRetry = errors.New("ctrl: retrying to subscribe for ack message")
)
// messageDeliverNats will create the Nats message with headers and payload.
@ -316,13 +316,6 @@ func (p process) messageDeliverNats(natsMsgPayload []byte, natsMsgHeader nats.He
}
p.metrics.promNatsDeliveredTotal.Inc()
//err = natsConn.Flush()
//if err != nil {
// er := fmt.Errorf("error: nats publish flush failed: %v", err)
// log.Printf("%v\n", er)
// return
//}
// The remaining logic is for handling ACK messages, so we return here
// since it was a NACK message, and all or now done.
@ -415,7 +408,7 @@ func (p process) messageDeliverNats(natsMsgPayload []byte, natsMsgHeader nats.He
return er
default:
er := fmt.Errorf("error: ack receive failed: the error was not defined, check if nats client have been updated with new error values, and update steward to handle the new error type: subject=%v: %v", p.subject.name(), err)
er := fmt.Errorf("error: ack receive failed: the error was not defined, check if nats client have been updated with new error values, and update ctrl to handle the new error type: subject=%v: %v", p.subject.name(), err)
p.errorKernel.logDebug(er, p.configuration)
return er
@ -557,14 +550,14 @@ func (p process) messageSubscriberHandler(natsConn *nats.Conn, thisNode string,
// Check if it is an ACK or NACK message, and do the appropriate action accordingly.
//
// With ACK messages Steward will keep the state of the message delivery, and try to
// With ACK messages ctrl will keep the state of the message delivery, and try to
// resend the message if an ACK is not received within the timeout/retries specified
// in the message.
// When a process sends an ACK message, it will stop and wait for the nats-reply message
// for the time specified in the replyTimeout value. If no reply message is received
// within the given timeout the publishing process will try to resend the message for
// number of times specified in the retries field of the Steward message.
// When receiving a Steward-message with ACK enabled we send a message back the the
// number of times specified in the retries field of the ctrl message.
// When receiving a ctrl-message with ACK enabled we send a message back the the
// node where the message originated using the msg.Reply subject field of the nats-message.
//
// With NACK messages we do not send a nats reply message, so the message will only be
@ -581,9 +574,9 @@ func (p process) messageSubscriberHandler(natsConn *nats.Conn, thisNode string,
if p.handler == nil {
// Look up the method handler for the specified method.
mh, ok := p.methodsAvailable.CheckIfExists(message.Method)
p.handler = mh.handler
p.handler = mh
if !ok {
er := fmt.Errorf("error: subscriberHandler: no such method type: %v", p.subject.Event)
er := fmt.Errorf("error: subscriberHandler: no such method type: %v", p.subject.Method)
p.errorKernel.errSend(p, message, er, logWarning)
}
}
@ -606,9 +599,9 @@ func (p process) messageSubscriberHandler(natsConn *nats.Conn, thisNode string,
if p.handler == nil {
// Look up the method handler for the specified method.
mh, ok := p.methodsAvailable.CheckIfExists(message.Method)
p.handler = mh.handler
p.handler = mh
if !ok {
er := fmt.Errorf("error: subscriberHandler: no such method type: %v", p.subject.Event)
er := fmt.Errorf("error: subscriberHandler: no such method type: %v", p.subject.Method)
p.errorKernel.errSend(p, message, er, logWarning)
}
}
@ -617,7 +610,7 @@ func (p process) messageSubscriberHandler(natsConn *nats.Conn, thisNode string,
_ = p.callHandler(message, thisNode)
default:
er := fmt.Errorf("info: did not find that specific type of event: %#v", p.subject.Event)
er := fmt.Errorf("info: did not find that specific type of event: %#v", p.subject.Method)
p.errorKernel.infoSend(p, message, er)
}
@ -834,11 +827,6 @@ func (p process) publishMessages(natsConn *nats.Conn) {
}
// Loop and handle 1 message at a time. If some part of the code
// fails in the loop we should throw an error and use `continue`
// to jump back here to the beginning of the loop and continue
// with the next message.
// Adding a timer that will be used for when to remove the sub process
// publisher. The timer is reset each time a message is published with
// the process, so the sub process publisher will not be removed until
@ -940,11 +928,6 @@ func (p process) publishAMessage(m Message, zEnc *zstd.Encoder, once *sync.Once,
// processes map, and increment the message counter.
pn := processNameGet(p.subject.name(), processKindPublisher)
// NB: REMOVED: It doesn't really make sense to get the message id
// from the process. Implemented so this is picked up from the id
// used in the ringbuffer.
// m.ID = p.messageID
// The compressed value of the nats message payload. The content
// can either be compressed or in it's original form depening on
// the outcome of the switch below, and if compression were chosen
@ -1005,16 +988,6 @@ func (p process) publishAMessage(m Message, zEnc *zstd.Encoder, once *sync.Once,
// sending of the message.
p.messageDeliverNats(natsMsgPayloadCompressed, natsMsgHeader, natsConn, m)
if p.configuration.RingBufferPersistStore {
select {
case m.done <- struct{}{}:
// Signaling back to the ringbuffer that we are done with the
// current message, and it can remove it from the ringbuffer.
case <-p.ctx.Done():
return
}
}
// Increment the counter for the next message to be sent.
p.messageID++

17
processes.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
import (
"context"
@ -30,8 +30,6 @@ type processes struct {
metrics *metrics
// Waitgroup to keep track of all the processes started.
wg sync.WaitGroup
// tui
tui *tui
// errorKernel
errorKernel *errorKernel
// configuration
@ -47,7 +45,6 @@ func newProcesses(ctx context.Context, server *server) *processes {
p := processes{
server: server,
active: *newProcsMap(),
tui: server.tui,
errorKernel: server.errorKernel,
configuration: server.configuration,
nodeAuth: server.nodeAuth,
@ -163,14 +160,6 @@ func (p *processes) Start(proc process) {
proc.startup.subscriber(proc, REQErrorLog, nil)
}
if proc.configuration.StartSubREQPing {
proc.startup.subscriber(proc, REQPing, nil)
}
if proc.configuration.StartSubREQPong {
proc.startup.subscriber(proc, REQPong, nil)
}
if proc.configuration.StartSubREQCliCommand {
proc.startup.subscriber(proc, REQCliCommand, nil)
}
@ -179,10 +168,6 @@ func (p *processes) Start(proc process) {
proc.startup.subscriber(proc, REQToConsole, nil)
}
if proc.configuration.EnableTUI {
proc.startup.subscriber(proc, REQTuiToConsole, nil)
}
if proc.configuration.StartPubREQHello != 0 {
pf := func(ctx context.Context, procFuncCh chan Message) error {
ticker := time.NewTicker(time.Second * time.Duration(p.configuration.StartPubREQHello))

223
requests.go
View file

@ -37,7 +37,7 @@
//
// Check out the existing code below for more examples.
package steward
package ctrl
import (
"context"
@ -81,8 +81,6 @@ const (
// The data field is a slice of strings where the first string
// value should be the command, and the following the arguments.
REQToConsole Method = "REQToConsole"
// REQTuiToConsole
REQTuiToConsole Method = "REQTuiToConsole"
// Send text logging to some host by appending the output to a
// file, if the file do not exist we create it.
// A file with the full subject+hostName will be created on
@ -112,12 +110,6 @@ const (
REQHello Method = "REQHello"
// Error log methods to centralError node.
REQErrorLog Method = "REQErrorLog"
// Echo request will ask the subscriber for a
// reply generated as a new message, and sent back to where
// the initial request was made.
REQPing Method = "REQPing"
// Will generate a reply for a ECHORequest
REQPong Method = "REQPong"
// Http Get
REQHttpGet Method = "REQHttpGet"
// Http Get Scheduled
@ -171,14 +163,10 @@ const (
REQAclImport = "REQAclImport"
)
type HandlerFunc func(proc process, message Message, node string) ([]byte, error)
// The mapping of all the method constants specified, what type
// it references, and the kind if it is an Event or Command, and
// if it is ACK or NACK.
//
// Allowed values for the Event field are:
// - EventACK
// - EventNack
//
// it references.
// The primary use of this table is that messages are not able to
// pass the actual type of the request since it is sent as a string,
// so we use the below table to find the actual type based on that
@ -186,150 +174,57 @@ const (
func (m Method) GetMethodsAvailable() MethodsAvailable {
ma := MethodsAvailable{
Methodhandlers: map[Method]methodHandler{
REQInitial: methodREQInitial{
event: EventACK,
},
REQOpProcessList: methodREQOpProcessList{
event: EventACK,
},
REQOpProcessStart: methodREQOpProcessStart{
event: EventACK,
},
REQOpProcessStop: methodREQOpProcessStop{
event: EventACK,
},
REQCliCommand: methodREQCliCommand{
event: EventACK,
},
REQCliCommandCont: methodREQCliCommandCont{
event: EventACK,
},
REQToConsole: methodREQToConsole{
event: EventACK,
},
REQTuiToConsole: methodREQTuiToConsole{
event: EventACK,
},
REQToFileAppend: methodREQToFileAppend{
event: EventACK,
},
REQToFile: methodREQToFile{
event: EventACK,
},
REQToFileNACK: methodREQToFile{
event: EventNACK,
},
REQCopySrc: methodREQCopySrc{
event: EventACK,
},
REQCopyDst: methodREQCopyDst{
event: EventACK,
},
REQSUBCopySrc: methodREQSUB{
event: EventACK,
},
REQSUBCopyDst: methodREQSUB{
event: EventACK,
},
REQHello: methodREQHello{
event: EventNACK,
},
REQErrorLog: methodREQErrorLog{
event: EventACK,
},
REQPing: methodREQPing{
event: EventACK,
},
REQPong: methodREQPong{
event: EventACK,
},
REQHttpGet: methodREQHttpGet{
event: EventACK,
},
REQHttpGetScheduled: methodREQHttpGetScheduled{
event: EventACK,
},
REQTailFile: methodREQTailFile{
event: EventACK,
},
REQPublicKey: methodREQPublicKey{
event: EventACK,
},
REQKeysRequestUpdate: methodREQKeysRequestUpdate{
event: EventNACK,
},
REQKeysDeliverUpdate: methodREQKeysDeliverUpdate{
event: EventNACK,
},
REQKeysAllow: methodREQKeysAllow{
event: EventACK,
},
REQKeysDelete: methodREQKeysDelete{
event: EventACK,
},
Methodhandlers: map[Method]HandlerFunc{
REQInitial: HandlerFunc(methodREQInitial),
REQOpProcessList: HandlerFunc(methodREQOpProcessList),
REQOpProcessStart: HandlerFunc(methodREQOpProcessStart),
REQOpProcessStop: HandlerFunc(methodREQOpProcessStop),
REQCliCommand: HandlerFunc(methodREQCliCommand),
REQCliCommandCont: HandlerFunc(methodREQCliCommandCont),
REQToConsole: HandlerFunc(methodREQToConsole),
REQToFileAppend: HandlerFunc(methodREQToFileAppend),
REQToFile: HandlerFunc(methodREQToFile),
REQToFileNACK: HandlerFunc(methodREQToFile),
REQCopySrc: HandlerFunc(methodREQCopySrc),
REQCopyDst: HandlerFunc(methodREQCopyDst),
REQSUBCopySrc: HandlerFunc(methodREQSUB),
REQSUBCopyDst: HandlerFunc(methodREQSUB),
REQHello: HandlerFunc(methodREQHello),
REQErrorLog: HandlerFunc(methodREQErrorLog),
REQHttpGet: HandlerFunc(methodREQHttpGet),
REQHttpGetScheduled: HandlerFunc(methodREQHttpGetScheduled),
REQTailFile: HandlerFunc(methodREQTailFile),
REQPublicKey: HandlerFunc(methodREQPublicKey),
REQKeysRequestUpdate: HandlerFunc(methodREQKeysRequestUpdate),
REQKeysDeliverUpdate: HandlerFunc(methodREQKeysDeliverUpdate),
REQKeysAllow: HandlerFunc(methodREQKeysAllow),
REQKeysDelete: HandlerFunc(methodREQKeysDelete),
REQAclRequestUpdate: methodREQAclRequestUpdate{
event: EventNACK,
},
REQAclDeliverUpdate: methodREQAclDeliverUpdate{
event: EventNACK,
},
REQAclRequestUpdate: HandlerFunc(methodREQAclRequestUpdate),
REQAclDeliverUpdate: HandlerFunc(methodREQAclDeliverUpdate),
REQAclAddCommand: methodREQAclAddCommand{
event: EventACK,
},
REQAclDeleteCommand: methodREQAclDeleteCommand{
event: EventACK,
},
REQAclDeleteSource: methodREQAclDeleteSource{
event: EventACK,
},
REQAclGroupNodesAddNode: methodREQAclGroupNodesAddNode{
event: EventACK,
},
REQAclGroupNodesDeleteNode: methodREQAclGroupNodesDeleteNode{
event: EventACK,
},
REQAclGroupNodesDeleteGroup: methodREQAclGroupNodesDeleteGroup{
event: EventACK,
},
REQAclGroupCommandsAddCommand: methodREQAclGroupCommandsAddCommand{
event: EventACK,
},
REQAclGroupCommandsDeleteCommand: methodREQAclGroupCommandsDeleteCommand{
event: EventACK,
},
REQAclGroupCommandsDeleteGroup: methodREQAclGroupCommandsDeleteGroup{
event: EventACK,
},
REQAclExport: methodREQAclExport{
event: EventACK,
},
REQAclImport: methodREQAclImport{
event: EventACK,
},
REQTest: methodREQTest{
event: EventACK,
},
REQAclAddCommand: HandlerFunc(methodREQAclAddCommand),
REQAclDeleteCommand: HandlerFunc(methodREQAclDeleteCommand),
REQAclDeleteSource: HandlerFunc(methodREQAclDeleteSource),
REQAclGroupNodesAddNode: HandlerFunc(methodREQAclGroupNodesAddNode),
REQAclGroupNodesDeleteNode: HandlerFunc(methodREQAclGroupNodesDeleteNode),
REQAclGroupNodesDeleteGroup: HandlerFunc(methodREQAclGroupNodesDeleteGroup),
REQAclGroupCommandsAddCommand: HandlerFunc(methodREQAclGroupCommandsAddCommand),
REQAclGroupCommandsDeleteCommand: HandlerFunc(methodREQAclGroupCommandsDeleteCommand),
REQAclGroupCommandsDeleteGroup: HandlerFunc(methodREQAclGroupCommandsDeleteGroup),
REQAclExport: HandlerFunc(methodREQAclExport),
REQAclImport: HandlerFunc(methodREQAclImport),
REQTest: HandlerFunc(methodREQTest),
},
}
return ma
}
// Reply methods. The slice generated here is primarily used within
// the Stew client for knowing what of the req types are generally
// used as reply methods.
func (m Method) GetReplyMethods() []Method {
rm := []Method{REQToConsole, REQTuiToConsole, REQCliCommand, REQCliCommandCont, REQToFile, REQToFileAppend, REQNone}
return rm
}
// getHandler will check the methodsAvailable map, and return the
// method handler for the method given
// as input argument.
func (m Method) getHandler(method Method) methodHandler {
func (m Method) getHandler(method Method) HandlerFunc {
ma := m.GetMethodsAvailable()
mh, _ := ma.CheckIfExists(method)
// mh := ma.Methodhandlers[method]
@ -354,15 +249,7 @@ func getContextForMethodTimeout(ctx context.Context, message Message) (context.C
// ----
// Initial parent method used to start other processes.
type methodREQInitial struct {
event Event
}
func (m methodREQInitial) getKind() Event {
return m.event
}
func (m methodREQInitial) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQInitial(proc process, message Message, node string) ([]byte, error) {
// proc.procFuncCh <- message
ackMsg := []byte("confirmed from: " + node + ": " + fmt.Sprint(message.ID))
return ackMsg, nil
@ -374,15 +261,7 @@ func (m methodREQInitial) handler(proc process, message Message, node string) ([
// Methods used in sub processes are defined within the the requests
// they are spawned in, so this type is primarily for us to use the
// same logic with sub process requests as we do with normal requests.
type methodREQSUB struct {
event Event
}
func (m methodREQSUB) getKind() Event {
return m.event
}
func (m methodREQSUB) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQSUB(proc process, message Message, node string) ([]byte, error) {
// proc.procFuncCh <- message
ackMsg := []byte("confirmed from: " + node + ": " + fmt.Sprint(message.ID))
return ackMsg, nil
@ -393,13 +272,13 @@ func (m methodREQSUB) handler(proc process, message Message, node string) ([]byt
// MethodsAvailable holds a map of all the different method types and the
// associated handler to that method type.
type MethodsAvailable struct {
Methodhandlers map[Method]methodHandler
Methodhandlers map[Method]HandlerFunc
}
// Check if exists will check if the Method is defined. If true the bool
// value will be set to true, and the methodHandler function for that type
// will be returned.
func (ma MethodsAvailable) CheckIfExists(m Method) (methodHandler, bool) {
func (ma MethodsAvailable) CheckIfExists(m Method) (HandlerFunc, bool) {
// First check if it is a sub process.
if strings.HasPrefix(string(m), "REQSUB") {
// Strip of the uuid after the method name.
@ -531,9 +410,3 @@ func selectFileNaming(message Message, proc process) (string, string) {
// ------------------------------------------------------------
// Subscriber method handlers
// ------------------------------------------------------------
// The methodHandler interface.
type methodHandler interface {
handler(proc process, message Message, node string) ([]byte, error)
getKind() Event
}

132
requests_acl.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
import (
"bytes"
@ -10,16 +10,8 @@ import (
// ----
type methodREQAclRequestUpdate struct {
event Event
}
func (m methodREQAclRequestUpdate) getKind() Event {
return m.event
}
// Handler to get all acl's from a central server.
func (m methodREQAclRequestUpdate) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQAclRequestUpdate(proc process, message Message, node string) ([]byte, error) {
inf := fmt.Errorf("<--- subscriber methodREQAclRequestUpdate received from: %v, hash data = %v", message.FromNode, message.Data)
proc.errorKernel.logDebug(inf, proc.configuration)
@ -98,16 +90,8 @@ func (m methodREQAclRequestUpdate) handler(proc process, message Message, node s
// ----
type methodREQAclDeliverUpdate struct {
event Event
}
func (m methodREQAclDeliverUpdate) getKind() Event {
return m.event
}
// Handler to receive the acls from a central server.
func (m methodREQAclDeliverUpdate) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQAclDeliverUpdate(proc process, message Message, node string) ([]byte, error) {
inf := fmt.Errorf("<--- subscriber methodREQAclDeliverUpdate received from: %v, containing: %v", message.FromNode, message.Data)
proc.errorKernel.logDebug(inf, proc.configuration)
@ -184,15 +168,7 @@ func (m methodREQAclDeliverUpdate) handler(proc process, message Message, node s
// ---
type methodREQAclAddCommand struct {
event Event
}
func (m methodREQAclAddCommand) getKind() Event {
return m.event
}
func (m methodREQAclAddCommand) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQAclAddCommand(proc process, message Message, node string) ([]byte, error) {
inf := fmt.Errorf("<--- methodREQAclAddCommand received from: %v, containing: %v", message.FromNode, message.MethodArgs)
proc.errorKernel.logDebug(inf, proc.configuration)
@ -254,15 +230,7 @@ func (m methodREQAclAddCommand) handler(proc process, message Message, node stri
// ---
type methodREQAclDeleteCommand struct {
event Event
}
func (m methodREQAclDeleteCommand) getKind() Event {
return m.event
}
func (m methodREQAclDeleteCommand) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQAclDeleteCommand(proc process, message Message, node string) ([]byte, error) {
inf := fmt.Errorf("<--- methodREQAclDeleteCommand received from: %v, containing: %v", message.FromNode, message.MethodArgs)
proc.errorKernel.logDebug(inf, proc.configuration)
@ -324,15 +292,7 @@ func (m methodREQAclDeleteCommand) handler(proc process, message Message, node s
// ---
type methodREQAclDeleteSource struct {
event Event
}
func (m methodREQAclDeleteSource) getKind() Event {
return m.event
}
func (m methodREQAclDeleteSource) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQAclDeleteSource(proc process, message Message, node string) ([]byte, error) {
inf := fmt.Errorf("<--- methodREQAclDeleteSource received from: %v, containing: %v", message.FromNode, message.MethodArgs)
proc.errorKernel.logDebug(inf, proc.configuration)
@ -393,15 +353,7 @@ func (m methodREQAclDeleteSource) handler(proc process, message Message, node st
// ---
type methodREQAclGroupNodesAddNode struct {
event Event
}
func (m methodREQAclGroupNodesAddNode) getKind() Event {
return m.event
}
func (m methodREQAclGroupNodesAddNode) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQAclGroupNodesAddNode(proc process, message Message, node string) ([]byte, error) {
inf := fmt.Errorf("<--- methodREQAclGroupNodesAddNode received from: %v, containing: %v", message.FromNode, message.MethodArgs)
proc.errorKernel.logDebug(inf, proc.configuration)
@ -462,15 +414,7 @@ func (m methodREQAclGroupNodesAddNode) handler(proc process, message Message, no
// ---
type methodREQAclGroupNodesDeleteNode struct {
event Event
}
func (m methodREQAclGroupNodesDeleteNode) getKind() Event {
return m.event
}
func (m methodREQAclGroupNodesDeleteNode) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQAclGroupNodesDeleteNode(proc process, message Message, node string) ([]byte, error) {
inf := fmt.Errorf("<--- methodREQAclGroupNodesDeleteNode received from: %v, containing: %v", message.FromNode, message.MethodArgs)
proc.errorKernel.logDebug(inf, proc.configuration)
@ -531,15 +475,7 @@ func (m methodREQAclGroupNodesDeleteNode) handler(proc process, message Message,
// ---
type methodREQAclGroupNodesDeleteGroup struct {
event Event
}
func (m methodREQAclGroupNodesDeleteGroup) getKind() Event {
return m.event
}
func (m methodREQAclGroupNodesDeleteGroup) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQAclGroupNodesDeleteGroup(proc process, message Message, node string) ([]byte, error) {
inf := fmt.Errorf("<--- methodREQAclGroupNodesDeleteGroup received from: %v, containing: %v", message.FromNode, message.MethodArgs)
proc.errorKernel.logDebug(inf, proc.configuration)
@ -599,15 +535,7 @@ func (m methodREQAclGroupNodesDeleteGroup) handler(proc process, message Message
// ---
type methodREQAclGroupCommandsAddCommand struct {
event Event
}
func (m methodREQAclGroupCommandsAddCommand) getKind() Event {
return m.event
}
func (m methodREQAclGroupCommandsAddCommand) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQAclGroupCommandsAddCommand(proc process, message Message, node string) ([]byte, error) {
inf := fmt.Errorf("<--- methodREQAclGroupCommandsAddCommand received from: %v, containing: %v", message.FromNode, message.MethodArgs)
proc.errorKernel.logDebug(inf, proc.configuration)
@ -668,15 +596,7 @@ func (m methodREQAclGroupCommandsAddCommand) handler(proc process, message Messa
// ---
type methodREQAclGroupCommandsDeleteCommand struct {
event Event
}
func (m methodREQAclGroupCommandsDeleteCommand) getKind() Event {
return m.event
}
func (m methodREQAclGroupCommandsDeleteCommand) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQAclGroupCommandsDeleteCommand(proc process, message Message, node string) ([]byte, error) {
inf := fmt.Errorf("<--- methodREQAclGroupCommandsDeleteCommand received from: %v, containing: %v", message.FromNode, message.MethodArgs)
proc.errorKernel.logDebug(inf, proc.configuration)
@ -737,15 +657,7 @@ func (m methodREQAclGroupCommandsDeleteCommand) handler(proc process, message Me
// ---
type methodREQAclGroupCommandsDeleteGroup struct {
event Event
}
func (m methodREQAclGroupCommandsDeleteGroup) getKind() Event {
return m.event
}
func (m methodREQAclGroupCommandsDeleteGroup) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQAclGroupCommandsDeleteGroup(proc process, message Message, node string) ([]byte, error) {
inf := fmt.Errorf("<--- methodREQAclGroupCommandsDeleteGroup received from: %v, containing: %v", message.FromNode, message.MethodArgs)
proc.errorKernel.logDebug(inf, proc.configuration)
@ -805,15 +717,7 @@ func (m methodREQAclGroupCommandsDeleteGroup) handler(proc process, message Mess
// ---
type methodREQAclExport struct {
event Event
}
func (m methodREQAclExport) getKind() Event {
return m.event
}
func (m methodREQAclExport) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQAclExport(proc process, message Message, node string) ([]byte, error) {
inf := fmt.Errorf("<--- methodREQAclExport received from: %v, containing: %v", message.FromNode, message.MethodArgs)
proc.errorKernel.logDebug(inf, proc.configuration)
@ -869,15 +773,7 @@ func (m methodREQAclExport) handler(proc process, message Message, node string)
// ---
type methodREQAclImport struct {
event Event
}
func (m methodREQAclImport) getKind() Event {
return m.event
}
func (m methodREQAclImport) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQAclImport(proc process, message Message, node string) ([]byte, error) {
inf := fmt.Errorf("<--- methodREQAclImport received from: %v, containing: %v", message.FromNode, message.MethodArgs)
proc.errorKernel.logDebug(inf, proc.configuration)

36
requests_cli.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
import (
"bufio"
@ -6,20 +6,13 @@ import (
"fmt"
"os/exec"
"strings"
"time"
)
type methodREQCliCommand struct {
event Event
}
func (m methodREQCliCommand) getKind() Event {
return m.event
}
// handler to run a CLI command with timeout context. The handler will
// return the output of the command run back to the calling publisher
// as a new message.
func (m methodREQCliCommand) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQCliCommand(proc process, message Message, node string) ([]byte, error) {
inf := fmt.Errorf("<--- CLICommandREQUEST received from: %v, containing: %v", message.FromNode, message.MethodArgs)
proc.errorKernel.logDebug(inf, proc.configuration)
@ -63,10 +56,10 @@ func (m methodREQCliCommand) handler(proc process, message Message, node string)
var foundEnvData bool
var envData string
for i, v := range message.MethodArgs {
if strings.Contains(v, "{{STEWARD_DATA}}") {
if strings.Contains(v, "{{CTRL_DATA}}") {
foundEnvData = true
// Replace the found env variable placeholder with an actual env variable
message.MethodArgs[i] = strings.Replace(message.MethodArgs[i], "{{STEWARD_DATA}}", "$STEWARD_DATA", -1)
message.MethodArgs[i] = strings.Replace(message.MethodArgs[i], "{{CTRL_DATA}}", "$CTRL_DATA", -1)
// Put all the data which is a slice of string into a single
// string so we can put it in a single env variable.
@ -76,9 +69,9 @@ func (m methodREQCliCommand) handler(proc process, message Message, node string)
cmd := exec.CommandContext(ctx, c, a...)
// Check for the use of env variable for STEWARD_DATA, and set env if found.
// Check for the use of env variable for CTRL_DATA, and set env if found.
if foundEnvData {
envData = fmt.Sprintf("STEWARD_DATA=%v", envData)
envData = fmt.Sprintf("CTRL_DATA=%v", envData)
cmd.Env = append(cmd.Env, envData)
}
@ -87,6 +80,7 @@ func (m methodREQCliCommand) handler(proc process, message Message, node string)
cmd.Stdout = &out
cmd.Stderr = &stderr
cmd.WaitDelay = time.Second * 5
err := cmd.Run()
if err != nil {
er := fmt.Errorf("error: methodREQCliCommand: cmd.Run failed : %v, methodArgs: %v, error_output: %v", err, message.MethodArgs, stderr.String())
@ -110,9 +104,6 @@ func (m methodREQCliCommand) handler(proc process, message Message, node string)
case out := <-outCh:
cancel()
// NB: Not quite sure what is the best way to handle the below
// isReply right now. Implementing as send to central for now.
//
// If this is this a reply message swap the toNode and fromNode
// fields so the output of the command are sent to central node.
if message.IsReply {
@ -132,19 +123,11 @@ func (m methodREQCliCommand) handler(proc process, message Message, node string)
// ---
type methodREQCliCommandCont struct {
event Event
}
func (m methodREQCliCommandCont) getKind() Event {
return m.event
}
// Handler to run REQCliCommandCont, which is the same as normal
// Cli command, but can be used when running a command that will take
// longer time and you want to send the output of the command continually
// back as it is generated, and not just when the command is finished.
func (m methodREQCliCommandCont) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQCliCommandCont(proc process, message Message, node string) ([]byte, error) {
inf := fmt.Errorf("<--- CLInCommandCont REQUEST received from: %v, containing: %v", message.FromNode, message.Data)
proc.errorKernel.logDebug(inf, proc.configuration)
@ -208,6 +191,7 @@ func (m methodREQCliCommandCont) handler(proc process, message Message, node str
newReplyMessage(proc, msgForErrors, []byte(er.Error()))
}
cmd.WaitDelay = time.Second * 5
if err := cmd.Start(); err != nil {
er := fmt.Errorf("error: methodREQCliCommandCont: cmd.Start failed : %v, methodArgs: %v", err, message.MethodArgs)
proc.errorKernel.errSend(proc, message, er, logWarning)

38
requests_copy.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
import (
"context"
@ -32,14 +32,6 @@ type copyInitialData struct {
FolderPermission uint64
}
type methodREQCopySrc struct {
event Event
}
func (m methodREQCopySrc) getKind() Event {
return m.event
}
// methodREQCopySrc are handles the initial and first part of
// the message flow for a copy to destination request.
// It's main role is to start up a sub process for the destination
@ -88,7 +80,7 @@ func (m methodREQCopySrc) getKind() Event {
// -----------------------------------------------------
// Handle writing to a file. Will truncate any existing data if the file did already
// exist.
func (m methodREQCopySrc) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQCopySrc(proc process, message Message, node string) ([]byte, error) {
// If the toNode field is not the same as nodeName of the receiving node
// we should forward the message to that specified toNode. This will allow
@ -297,19 +289,11 @@ func newSubProcess(ctx context.Context, server *server, subject Subject, process
// ----
type methodREQCopyDst struct {
event Event
}
func (m methodREQCopyDst) getKind() Event {
return m.event
}
// methodREQCopyDst are handles the initial and first part of
// the message flow for a copy to destination request.
// It's main role is to start up a sub process for the destination
// in which all the actual file copying is done.
func (m methodREQCopyDst) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQCopyDst(proc process, message Message, node string) ([]byte, error) {
var subProcessName string
proc.processes.wg.Add(1)
@ -459,10 +443,11 @@ func copySrcSubProcFunc(proc process, cia copyInitialData, cancel context.Cancel
// Check the file is a unix socket, and if it is we write the
// data to the socket instead of writing it to a normal file.
// We don't care about the error.
fi, _ := os.Stat(file)
// if err != nil {
// fmt.Printf(" ** DEBUG: STAT ERROR: %v\n", err)
// }
fi, err := os.Stat(file)
if err != nil {
fmt.Printf(" ** DEBUG: STAT ERROR: %v\n", err)
fmt.Printf(" ** DEBUG: fi: %#v\n", fi)
}
// We want to be able to send the reply message when the copying is done,
// and also for any eventual errors within the subProcFunc. We want to
@ -471,7 +456,12 @@ func copySrcSubProcFunc(proc process, cia copyInitialData, cancel context.Cancel
// individual files.
msgForSubReplies := initialMessage
msgForSubErrors := initialMessage
if fi.Mode().Type() != fs.ModeSocket {
if fi != nil {
if fi.Mode().Type() != fs.ModeSocket {
msgForSubReplies.FileName = msgForSubReplies.FileName + ".copyreply"
msgForSubErrors.FileName = msgForSubErrors.FileName + ".copyerror"
}
} else {
msgForSubReplies.FileName = msgForSubReplies.FileName + ".copyreply"
msgForSubErrors.FileName = msgForSubErrors.FileName + ".copyerror"
}

32
requests_file_handling.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
import (
"fmt"
@ -87,16 +87,8 @@ func reqWriteFileOrSocket(isAppend bool, proc process, message Message) error {
return nil
}
type methodREQToFileAppend struct {
event Event
}
func (m methodREQToFileAppend) getKind() Event {
return m.event
}
// Handle appending data to file.
func (m methodREQToFileAppend) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQToFileAppend(proc process, message Message, node string) ([]byte, error) {
err := reqWriteFileOrSocket(true, proc, message)
proc.errorKernel.errSend(proc, message, err, logWarning)
@ -106,17 +98,9 @@ func (m methodREQToFileAppend) handler(proc process, message Message, node strin
// -----
type methodREQToFile struct {
event Event
}
func (m methodREQToFile) getKind() Event {
return m.event
}
// Handle writing to a file. Will truncate any existing data if the file did already
// exist.
func (m methodREQToFile) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQToFile(proc process, message Message, node string) ([]byte, error) {
err := reqWriteFileOrSocket(false, proc, message)
proc.errorKernel.errSend(proc, message, err, logWarning)
@ -126,18 +110,10 @@ func (m methodREQToFile) handler(proc process, message Message, node string) ([]
// --- methodREQTailFile
type methodREQTailFile struct {
event Event
}
func (m methodREQTailFile) getKind() Event {
return m.event
}
// handler to run a tailing of files with timeout context. The handler will
// return the output of the command run back to the calling publisher
// as a new message.
func (m methodREQTailFile) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQTailFile(proc process, message Message, node string) ([]byte, error) {
inf := fmt.Errorf("<--- TailFile REQUEST received from: %v, containing: %v", message.FromNode, message.Data)
proc.errorKernel.logDebug(inf, proc.configuration)

22
requests_http.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
import (
"context"
@ -9,16 +9,8 @@ import (
"time"
)
type methodREQHttpGet struct {
event Event
}
func (m methodREQHttpGet) getKind() Event {
return m.event
}
// handler to do a Http Get.
func (m methodREQHttpGet) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQHttpGet(proc process, message Message, node string) ([]byte, error) {
inf := fmt.Errorf("<--- REQHttpGet received from: %v, containing: %v", message.FromNode, message.Data)
proc.errorKernel.logDebug(inf, proc.configuration)
@ -117,17 +109,9 @@ func (m methodREQHttpGet) handler(proc process, message Message, node string) ([
// ---
type methodREQHttpGetScheduled struct {
event Event
}
func (m methodREQHttpGetScheduled) getKind() Event {
return m.event
}
// handler to do a Http Get Scheduled.
// The second element of the MethodArgs slice holds the timer defined in seconds.
func (m methodREQHttpGetScheduled) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQHttpGetScheduled(proc process, message Message, node string) ([]byte, error) {
inf := fmt.Errorf("<--- REQHttpGetScheduled received from: %v, containing: %v", message.FromNode, message.Data)
proc.errorKernel.logDebug(inf, proc.configuration)

52
requests_keys.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
import (
"bytes"
@ -8,16 +8,8 @@ import (
// ---
type methodREQPublicKey struct {
event Event
}
func (m methodREQPublicKey) getKind() Event {
return m.event
}
// Handler to get the public ed25519 key from a node.
func (m methodREQPublicKey) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQPublicKey(proc process, message Message, node string) ([]byte, error) {
// Get a context with the timeout specified in message.MethodTimeout.
ctx, _ := getContextForMethodTimeout(proc.ctx, message)
@ -55,16 +47,8 @@ func (m methodREQPublicKey) handler(proc process, message Message, node string)
// ----
type methodREQKeysRequestUpdate struct {
event Event
}
func (m methodREQKeysRequestUpdate) getKind() Event {
return m.event
}
// Handler to get all the public ed25519 keys from a central server.
func (m methodREQKeysRequestUpdate) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQKeysRequestUpdate(proc process, message Message, node string) ([]byte, error) {
// Get a context with the timeout specified in message.MethodTimeout.
// TODO:
@ -137,16 +121,8 @@ func (m methodREQKeysRequestUpdate) handler(proc process, message Message, node
// ----
type methodREQKeysDeliverUpdate struct {
event Event
}
func (m methodREQKeysDeliverUpdate) getKind() Event {
return m.event
}
// Handler to receive the public keys from a central server.
func (m methodREQKeysDeliverUpdate) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQKeysDeliverUpdate(proc process, message Message, node string) ([]byte, error) {
// Get a context with the timeout specified in message.MethodTimeout.
// TODO:
@ -226,20 +202,12 @@ func (m methodREQKeysDeliverUpdate) handler(proc process, message Message, node
// ----
type methodREQKeysAllow struct {
event Event
}
func (m methodREQKeysAllow) getKind() Event {
return m.event
}
// Handler to allow new public keys into the database on central auth.
// Nodes will send the public key in the REQHello messages. When they
// are recived on the central server they will be put into a temp key
// map, and we need to acknowledge them before they are moved into the
// main key map, and then allowed to be sent out to other nodes.
func (m methodREQKeysAllow) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQKeysAllow(proc process, message Message, node string) ([]byte, error) {
// Get a context with the timeout specified in message.MethodTimeout.
ctx, _ := getContextForMethodTimeout(proc.ctx, message)
@ -423,15 +391,7 @@ func pushKeys(proc process, message Message, nodes []Node) error {
}
type methodREQKeysDelete struct {
event Event
}
func (m methodREQKeysDelete) getKind() Event {
return m.event
}
func (m methodREQKeysDelete) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQKeysDelete(proc process, message Message, node string) ([]byte, error) {
inf := fmt.Errorf("<--- methodREQKeysDelete received from: %v, containing: %v", message.FromNode, message.MethodArgs)
proc.errorKernel.logDebug(inf, proc.configuration)

31
requests_operator.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
import (
"fmt"
@ -8,16 +8,9 @@ import (
)
// --- OpProcessList
type methodREQOpProcessList struct {
event Event
}
func (m methodREQOpProcessList) getKind() Event {
return m.event
}
// Handle Op Process List
func (m methodREQOpProcessList) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQOpProcessList(proc process, message Message, node string) ([]byte, error) {
proc.processes.wg.Add(1)
go func() {
@ -46,16 +39,8 @@ func (m methodREQOpProcessList) handler(proc process, message Message, node stri
// --- OpProcessStart
type methodREQOpProcessStart struct {
event Event
}
func (m methodREQOpProcessStart) getKind() Event {
return m.event
}
// Handle Op Process Start
func (m methodREQOpProcessStart) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQOpProcessStart(proc process, message Message, node string) ([]byte, error) {
proc.processes.wg.Add(1)
go func() {
defer proc.processes.wg.Done()
@ -101,21 +86,13 @@ func (m methodREQOpProcessStart) handler(proc process, message Message, node str
// --- OpProcessStop
type methodREQOpProcessStop struct {
event Event
}
func (m methodREQOpProcessStop) getKind() Event {
return m.event
}
// RecevingNode Node `json:"receivingNode"`
// Method Method `json:"method"`
// Kind processKind `json:"kind"`
// ID int `json:"id"`
// Handle Op Process Start
func (m methodREQOpProcessStop) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQOpProcessStop(proc process, message Message, node string) ([]byte, error) {
proc.processes.wg.Add(1)
go func() {
defer proc.processes.wg.Done()

195
requests_std.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
import (
"fmt"
@ -10,16 +10,8 @@ import (
// -----
type methodREQHello struct {
event Event
}
func (m methodREQHello) getKind() Event {
return m.event
}
// Handler for receiving hello messages.
func (m methodREQHello) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQHello(proc process, message Message, node string) ([]byte, error) {
data := fmt.Sprintf("%v, Received hello from %#v\n", time.Now().Format("Mon Jan _2 15:04:05 2006"), message.FromNode)
fileName := message.FileName
@ -66,16 +58,8 @@ func (m methodREQHello) handler(proc process, message Message, node string) ([]b
// ---
type methodREQErrorLog struct {
event Event
}
func (m methodREQErrorLog) getKind() Event {
return m.event
}
// Handle the writing of error logs.
func (m methodREQErrorLog) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQErrorLog(proc process, message Message, node string) ([]byte, error) {
proc.metrics.promErrorMessagesReceivedTotal.Inc()
// If it was a request type message we want to check what the initial messages
@ -115,145 +99,11 @@ func (m methodREQErrorLog) handler(proc process, message Message, node string) (
// ---
type methodREQPing struct {
event Event
}
func (m methodREQPing) getKind() Event {
return m.event
}
// Handle receving a ping.
func (m methodREQPing) handler(proc process, message Message, node string) ([]byte, error) {
// Write to file that we received a ping
// If it was a request type message we want to check what the initial messages
// method, so we can use that in creating the file name to store the data.
fileName, folderTree := selectFileNaming(message, proc)
// Check if folder structure exist, if not create it.
if _, err := os.Stat(folderTree); os.IsNotExist(err) {
err := os.MkdirAll(folderTree, 0770)
if err != nil {
er := fmt.Errorf("error: methodREQPing.handler: failed to create toFile directory tree: %v, %v", folderTree, err)
proc.errorKernel.errSend(proc, message, er, logWarning)
return nil, er
}
er := fmt.Errorf("info: Creating subscribers data folder at %v", folderTree)
proc.errorKernel.logDebug(er, proc.configuration)
}
// Open file.
file := filepath.Join(folderTree, fileName)
f, err := os.OpenFile(file, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0755)
if err != nil {
er := fmt.Errorf("error: methodREQPing.handler: failed to open file, check that you've specified a value for fileName in the message: directory: %v, fileName: %v, %v", message.Directory, message.FileName, err)
proc.errorKernel.errSend(proc, message, er, logWarning)
return nil, err
}
defer f.Close()
// And write the data
d := fmt.Sprintf("%v, ping received from %v\n", time.Now().Format("Mon Jan _2 15:04:05 2006"), message.FromNode)
_, err = f.Write([]byte(d))
f.Sync()
if err != nil {
er := fmt.Errorf("error: methodREQPing.handler: failed to write to file: directory: %v, fileName: %v, %v", message.Directory, message.FileName, err)
proc.errorKernel.errSend(proc, message, er, logWarning)
}
proc.processes.wg.Add(1)
go func() {
defer proc.processes.wg.Done()
newReplyMessage(proc, message, nil)
}()
ackMsg := []byte("confirmed from: " + node + ": " + fmt.Sprint(message.ID))
return ackMsg, nil
}
// ---
type methodREQPong struct {
event Event
}
func (m methodREQPong) getKind() Event {
return m.event
}
// Handle receiving a pong.
func (m methodREQPong) handler(proc process, message Message, node string) ([]byte, error) {
// Write to file that we received a pong
// If it was a request type message we want to check what the initial messages
// method, so we can use that in creating the file name to store the data.
fileName, folderTree := selectFileNaming(message, proc)
// Check if folder structure exist, if not create it.
if _, err := os.Stat(folderTree); os.IsNotExist(err) {
err := os.MkdirAll(folderTree, 0770)
if err != nil {
er := fmt.Errorf("error: methodREQPong.handler: failed to create toFile directory tree %v: %v", folderTree, err)
proc.errorKernel.errSend(proc, message, er, logWarning)
return nil, er
}
er := fmt.Errorf("info: Creating subscribers data folder at %v", folderTree)
proc.errorKernel.logDebug(er, proc.configuration)
}
// Open file.
file := filepath.Join(folderTree, fileName)
f, err := os.OpenFile(file, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0755)
if err != nil {
er := fmt.Errorf("error: methodREQPong.handler: failed to open file, check that you've specified a value for fileName in the message: directory: %v, fileName: %v, %v", message.Directory, message.FileName, err)
proc.errorKernel.errSend(proc, message, er, logWarning)
return nil, err
}
defer f.Close()
// And write the data
d := fmt.Sprintf("%v, pong received from %v\n", time.Now().Format("Mon Jan _2 15:04:05 2006"), message.PreviousMessage.ToNode)
_, err = f.Write([]byte(d))
f.Sync()
if err != nil {
er := fmt.Errorf("error: methodREQPong.handler: failed to write to file: directory: %v, fileName: %v, %v", message.Directory, message.FileName, err)
proc.errorKernel.errSend(proc, message, er, logWarning)
}
ackMsg := []byte("confirmed from: " + node + ": " + fmt.Sprint(message.ID))
return ackMsg, nil
}
// ---
type methodREQToConsole struct {
event Event
}
func (m methodREQToConsole) getKind() Event {
return m.event
}
// Handler to write directly to console.
// This handler handles the writing to console both for TUI and shell clients.
func (m methodREQToConsole) handler(proc process, message Message, node string) ([]byte, error) {
// This handler handles the writing to console.
func methodREQToConsole(proc process, message Message, node string) ([]byte, error) {
switch {
case proc.configuration.EnableTUI:
if proc.processes.tui.toConsoleCh != nil {
proc.processes.tui.toConsoleCh <- message.Data
} else {
er := fmt.Errorf("error: no tui client started")
proc.errorKernel.errSend(proc, message, er, logWarning)
}
case len(message.MethodArgs) > 0 && message.MethodArgs[0] == "stderr":
log.Printf("* DEBUG: MethodArgs: got stderr \n")
fmt.Fprintf(os.Stderr, "%v", string(message.Data))
@ -269,44 +119,11 @@ func (m methodREQToConsole) handler(proc process, message Message, node string)
// ---
type methodREQTuiToConsole struct {
event Event
}
func (m methodREQTuiToConsole) getKind() Event {
return m.event
}
// Handler to write directly to console.
// DEPRECATED
func (m methodREQTuiToConsole) handler(proc process, message Message, node string) ([]byte, error) {
if proc.processes.tui.toConsoleCh != nil {
proc.processes.tui.toConsoleCh <- message.Data
} else {
er := fmt.Errorf("error: no tui client started")
proc.errorKernel.errSend(proc, message, er, logWarning)
}
ackMsg := []byte("confirmed from: " + node + ": " + fmt.Sprint(message.ID))
return ackMsg, nil
}
// ---
type methodREQTest struct {
event Event
}
func (m methodREQTest) getKind() Event {
return m.event
}
// handler to be used as a reply method when testing requests.
// We can then within the test listen on the testCh for received
// data and validate it.
// If no test is listening the data will be dropped.
func (m methodREQTest) handler(proc process, message Message, node string) ([]byte, error) {
func methodREQTest(proc process, message Message, node string) ([]byte, error) {
go func() {
// Try to send the received message data on the test channel. If we

2
requests_template.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
// ---- Template that can be used for creating request methods

32
requests_test.go
View file

@ -1,4 +1,4 @@
package steward
package ctrl
import (
"bytes"
@ -58,11 +58,11 @@ func TestMain(m *testing.M) {
func newServerForTesting(addressAndPort string, testFolder string) (*server, *Configuration) {
// Start Steward instance
// Start ctrl instance
// ---------------------------------------
// tempdir := t.TempDir()
// Create the config to run a steward instance.
// Create the config to run a ctrl instance.
//tempdir := "./tmp"
conf := newConfigurationDefaults()
if *logging {
@ -81,12 +81,12 @@ func newServerForTesting(addressAndPort string, testFolder string) (*server, *Co
conf.EnableDebug = false
conf.LogLevel = "none"
stewardServer, err := NewServer(&conf, "test")
ctrlServer, err := NewServer(&conf, "test")
if err != nil {
log.Fatalf(" * failed: could not start the Steward instance %v\n", err)
log.Fatalf(" * failed: could not start the ctrl instance %v\n", err)
}
return stewardServer, &conf
return ctrlServer, &conf
}
// Start up the nats-server message broker for testing purposes.
@ -112,7 +112,7 @@ func writeMsgsToSocketTest(conf *Configuration, messages []Message, t *testing.T
t.Fatalf("writeMsgsToSocketTest: %v\n ", err)
}
socket, err := net.Dial("unix", filepath.Join(conf.SocketFolder, "steward.sock"))
socket, err := net.Dial("unix", filepath.Join(conf.SocketFolder, "ctrl.sock"))
if err != nil {
t.Fatalf(" * failed: could to open socket file for writing: %v\n", err)
}
@ -273,7 +273,7 @@ func TestRequest(t *testing.T) {
MethodArgs: []string{},
MethodTimeout: 5,
ReplyMethod: REQTest,
}, want: []byte("central.REQHttpGet.EventACK"),
}, want: []byte("central.REQHttpGet"),
containsOrEquals: REQTestContains,
viaSocketOrCh: viaCh,
},
@ -288,7 +288,7 @@ func TestRequest(t *testing.T) {
t.Fatalf("newSubjectAndMessage failed: %v\n", err)
}
tstSrv.toRingBufferCh <- []subjectAndMessage{sam}
tstSrv.samToSendCh <- []subjectAndMessage{sam}
case viaSocket:
msgs := []Message{tt.message}
@ -341,7 +341,7 @@ func TestRequest(t *testing.T) {
}
// Check the tailing of files type.
func checkREQTailFileTest(stewardServer *server, conf *Configuration, t *testing.T, tmpDir string) error {
func checkREQTailFileTest(ctrlServer *server, conf *Configuration, t *testing.T, tmpDir string) error {
// Create a file with some content.
fp := filepath.Join(tmpDir, "test.file")
fh, err := os.OpenFile(fp, os.O_APPEND|os.O_RDWR|os.O_CREATE|os.O_SYNC, 0660)
@ -417,7 +417,7 @@ func checkREQTailFileTest(stewardServer *server, conf *Configuration, t *testing
}
// Check the file copier.
func checkREQCopySrc(stewardServer *server, conf *Configuration, t *testing.T, tmpDir string) error {
func checkREQCopySrc(ctrlServer *server, conf *Configuration, t *testing.T, tmpDir string) error {
testFiles := 5
for i := 1; i <= testFiles; i++ {
@ -474,8 +474,8 @@ func checkREQCopySrc(stewardServer *server, conf *Configuration, t *testing.T, t
return nil
}
func checkMetricValuesTest(stewardServer *server, conf *Configuration, t *testing.T, tempDir string) error {
mfs, err := stewardServer.metrics.promRegistry.Gather()
func checkMetricValuesTest(ctrlServer *server, conf *Configuration, t *testing.T, tempDir string) error {
mfs, err := ctrlServer.metrics.promRegistry.Gather()
if err != nil {
return fmt.Errorf("error: promRegistry.gathering: %v", mfs)
}
@ -486,7 +486,7 @@ func checkMetricValuesTest(stewardServer *server, conf *Configuration, t *testin
found := false
for _, mf := range mfs {
if mf.GetName() == "steward_processes_total" {
if mf.GetName() == "ctrl_processes_total" {
found = true
m := mf.GetMetric()
@ -507,7 +507,7 @@ func checkMetricValuesTest(stewardServer *server, conf *Configuration, t *testin
}
// Check errorKernel
func checkErrorKernelMalformedJSONtest(stewardServer *server, conf *Configuration, t *testing.T, tempDir string) error {
func checkErrorKernelMalformedJSONtest(ctrlServer *server, conf *Configuration, t *testing.T, tempDir string) error {
// JSON message with error, missing brace.
m := `[
@ -641,7 +641,7 @@ func findStringInFileTest(want string, fileName string, conf *Configuration, t *
// Write message to socket for testing purposes.
func writeToSocketTest(conf *Configuration, messageText string, t *testing.T) {
socket, err := net.Dial("unix", filepath.Join(conf.SocketFolder, "steward.sock"))
socket, err := net.Dial("unix", filepath.Join(conf.SocketFolder, "ctrl.sock"))
if err != nil {
t.Fatalf(" * failed: could to open socket file for writing: %v\n", err)
}

570
ringbuffer.go
View file

@ -1,570 +0,0 @@
// Info: The idea about the ring buffer is that we have a FIFO
// buffer where we store all incomming messages requested by
// operators.
// Each message in process or waiting to be processed will be
// stored in a DB. When the processing of a given message is
// done it will be removed from the state db, and an entry will
// made in the persistent message log.
package steward
import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"sort"
"strconv"
"sync"
"time"
copier "github.com/jinzhu/copier"
bolt "go.etcd.io/bbolt"
)
// samValue represents one message with a subject. This
// struct type is used when storing and retreiving from
// db.
type samDBValue struct {
ID int
SAM subjectAndMessage
}
// ringBuffer holds the data of the buffer,
type ringBuffer struct {
// In memory buffer for the messages.
bufData chan samDBValue
// The database to use.
db *bolt.DB
samValueBucket string
indexValueBucket string
// The current number of items in the database.
totalMessagesIndex int
mu sync.Mutex
// The channel to send messages that have been processed,
// and we want to store it in the permanent message log.
permStore chan string
// Name of node.
nodeName Node
// ringBufferBulkInCh from *server are also implemented here,
// so the ringbuffer can send it's error messages the same
// way as all messages are handled.
ringBufferBulkInCh chan []subjectAndMessage
metrics *metrics
configuration *Configuration
errorKernel *errorKernel
processInitial process
}
// newringBuffer returns a push/pop storage for values.
func newringBuffer(ctx context.Context, metrics *metrics, configuration *Configuration, size int, dbFileName string, nodeName Node, ringBufferBulkInCh chan []subjectAndMessage, samValueBucket string, indexValueBucket string, errorKernel *errorKernel, processInitial process) *ringBuffer {
r := ringBuffer{}
// Check if socket folder exists, if not create it
if _, err := os.Stat(configuration.DatabaseFolder); os.IsNotExist(err) {
err := os.MkdirAll(configuration.DatabaseFolder, 0770)
if err != nil {
log.Printf("error: failed to create database directory %v: %v\n", configuration.DatabaseFolder, err)
os.Exit(1)
}
}
DatabaseFilepath := filepath.Join(configuration.DatabaseFolder, dbFileName)
// ---
var db *bolt.DB
if configuration.RingBufferPersistStore {
var err error
db, err = bolt.Open(DatabaseFilepath, 0660, nil)
if err != nil {
log.Printf("error: failed to open db: %v\n", err)
os.Exit(1)
}
}
r.bufData = make(chan samDBValue, size)
r.db = db
r.samValueBucket = samValueBucket
r.indexValueBucket = indexValueBucket
r.permStore = make(chan string)
r.nodeName = nodeName
r.ringBufferBulkInCh = ringBufferBulkInCh
r.metrics = metrics
r.configuration = configuration
r.processInitial = processInitial
return &r
}
// start will process incomming messages through the inCh,
// put the messages on a buffered channel
// and deliver messages out when requested on the outCh.
func (r *ringBuffer) start(ctx context.Context, ringBufferInCh chan subjectAndMessage, ringBufferOutCh chan samDBValueAndDelivered) {
// Starting both writing and reading in separate go routines so we
// can write and read concurrently.
r.totalMessagesIndex = 0
if r.configuration.RingBufferPersistStore {
r.totalMessagesIndex = r.getIndexValue()
}
// Fill the buffer when new data arrives into the system
go r.fillBuffer(ctx, ringBufferInCh)
// Start the process to permanently store done messages.
go r.startPermanentStore(ctx)
// Start the process that will handle messages present in the ringbuffer.
go r.processBufferMessages(ctx, ringBufferOutCh)
go func() {
ticker := time.NewTicker(time.Second * 5)
defer ticker.Stop()
for {
select {
case <-ticker.C:
if r.configuration.RingBufferPersistStore {
r.dbUpdateMetrics(r.samValueBucket)
}
case <-ctx.Done():
return
}
}
}()
}
// fillBuffer will fill the buffer in the ringbuffer reading from the inchannel.
// It will also store the messages in a K/V DB while being processed.
func (r *ringBuffer) fillBuffer(ctx context.Context, ringBufferInCh chan subjectAndMessage) {
// At startup get all the values that might be in the K/V store so we can
// put them into the buffer before we start to fill up with new incomming
// messages to the system.
// This is needed when the program have been restarted, and we need to check
// if there where previously unhandled messages that need to be handled first.
if r.configuration.RingBufferPersistStore {
func() {
s, err := r.dumpBucket(r.samValueBucket)
if err != nil {
er := fmt.Errorf("info: fillBuffer: retreival of values from k/v store failed, probaly empty database, and no previous entries in db to process: %v", err)
log.Printf("%v\n", er)
return
}
for _, v := range s {
r.bufData <- v
}
}()
}
// Check for incomming messages. These are typically comming from
// the go routine who reads the socket.
for {
select {
case sam := <-ringBufferInCh:
// Check if default message values for timers are set, and if
// not then set default message values.
if sam.Message.ACKTimeout < 1 {
sam.Subject.Event = EventNACK
}
if sam.Message.ACKTimeout >= 1 {
sam.Subject.Event = EventNACK
}
switch {
case sam.Message.Retries < 0:
sam.Message.Retries = r.configuration.DefaultMessageRetries
}
if sam.Message.MethodTimeout < 1 && sam.Message.MethodTimeout != -1 {
sam.Message.MethodTimeout = r.configuration.DefaultMethodTimeout
}
// --- Store the incomming message in the k/v store ---
// Get a unique number for the message to use when storing
// it in the databases, and also use when further processing.
r.mu.Lock()
dbID := r.totalMessagesIndex
r.mu.Unlock()
// Create a structure for JSON marshaling.
samV := samDBValue{
ID: dbID,
SAM: sam,
}
if r.configuration.RingBufferPersistStore {
js, err := json.Marshal(samV)
if err != nil {
er := fmt.Errorf("error:fillBuffer: json marshaling: %v", err)
r.errorKernel.errSend(r.processInitial, Message{}, er, logError)
}
// Store the incomming message in key/value store
err = r.dbUpdate(r.db, r.samValueBucket, strconv.Itoa(dbID), js)
if err != nil {
er := fmt.Errorf("error: dbUpdate samValue failed: %v", err)
r.errorKernel.errSend(r.processInitial, Message{}, er, logError)
}
}
// Put the message on the inmemory buffer.
r.bufData <- samV
// Increment index, and store the new value to the database.
r.mu.Lock()
r.totalMessagesIndex++
if r.configuration.RingBufferPersistStore {
r.dbUpdate(r.db, r.indexValueBucket, "index", []byte(strconv.Itoa(r.totalMessagesIndex)))
}
r.mu.Unlock()
case <-ctx.Done():
// When done close the buffer channel
close(r.bufData)
return
}
}
}
// processBufferMessages will pick messages from the buffer, and process them
// one by one. The messages will be delivered on the outCh, and it will wait
// until a signal is received on the done channel before it continues with the
// next message.
func (r *ringBuffer) processBufferMessages(ctx context.Context, ringBufferOutCh chan samDBValueAndDelivered) {
// Range over the buffer of messages to pass on to processes.
for {
select {
case samDBv := <-r.bufData:
r.metrics.promInMemoryBufferMessagesCurrent.Set(float64(len(r.bufData)))
samDBv.SAM.ID = samDBv.ID
// // Create a done channel per message. A process started by the
// // spawnProcess function will handle incomming messages sequentaly.
// // So in the spawnProcess function we put a struct{} value when a
// // message is processed on the "done" channel and an ack is received
// // for a message, and we wait here for the "done" to be received.
// We start the actual processing of an individual message here within
// it's own go routine. Reason is that we don't want to block other
// messages to be processed while waiting for the done signal, or if an
// error with an individual message occurs.
go func(v samDBValue) {
// Create a copy of the message that we can use to write to the
// perm store without causing a race since the REQ handler for the
// message might not yet be done when message is written to the
// perm store.
// We also need a copy to be able to remove the data from the message
// when writing it to the store, so we don't mess up to actual data
// that might be in use in the handler.
msgForPermStore := Message{}
copier.Copy(&msgForPermStore, v.SAM.Message)
// Remove the content of the data field.
msgForPermStore.Data = nil
v.SAM.Message.done = make(chan struct{})
delivredCh := make(chan struct{})
// Prepare the structure with the data, and a function that can
// be called when the data is received for signaling back.
sd := samDBValueAndDelivered{
samDBValue: v,
delivered: func() {
delivredCh <- struct{}{}
},
}
// Create a ticker that will kick in when a message have been in the
// system for it's maximum time. This will allow us to continue, and
// remove the message if it takes longer than it should to get delivered.
fmt.Printf("DEBUG:::%v\n", v.SAM.ACKTimeout)
if v.SAM.ACKTimeout <= 0 {
v.SAM.ACKTimeout = 1
}
if v.SAM.Retries <= 0 {
v.SAM.Retries = 1
}
ticker := time.NewTicker(time.Duration(v.SAM.ACKTimeout) * time.Duration(v.SAM.Retries) * time.Second)
defer ticker.Stop()
ringBufferOutCh <- sd
// Just to confirm here that the message was picked up, to know if the
// the read process have stalled or not.
// For now it will not do anything,
select {
case <-delivredCh:
// OK.
case <-time.After(time.Second * 5):
// TODO: Check if more logic should be made here if messages are stuck etc.
// Testing with a timeout here to figure out if messages are stuck
// waiting for done signal.
log.Printf("Error: ringBuffer: message %v seems to be stuck, did not receive delivered signal from reading process\n", v.ID)
r.metrics.promRingbufferStalledMessagesTotal.Inc()
}
// Listen on the done channel here , so a go routine handling the
// message will be able to signal back here that the message have
// been processed, and that we then can delete it out of the K/V Store.
// The publishAMessage method should send a done back here, but in some situations
// it seems that that do not happen. Implementing a ticker that is twice the total
// amount of time a message should be allowed to be using for getting published so
// we don't get stuck go routines here.
//
if r.configuration.RingBufferPersistStore {
select {
case <-v.SAM.done:
// fmt.Printf("---\n DONE with\n---\n")
case <-ticker.C:
log.Printf("----------------------------------------------\n")
log.Printf("Error: ringBuffer message id: %v, subject: %v seems to be stuck, did not receive done signal from publishAMessage process, exited on ticker\n", v.SAM.ID, v.SAM.Subject)
log.Printf("----------------------------------------------\n")
}
}
// log.Printf("info: processBufferMessages: done with message, deleting key from bucket, %v\n", v.ID)
r.metrics.promMessagesProcessedIDLast.Set(float64(v.ID))
// Since we are now done with the specific message we can delete
// it out of the K/V Store.
if r.configuration.RingBufferPersistStore {
r.deleteKeyFromBucket(r.samValueBucket, strconv.Itoa(v.ID))
}
js, err := json.Marshal(msgForPermStore)
if err != nil {
er := fmt.Errorf("error:fillBuffer: json marshaling: %v", err)
r.errorKernel.errSend(r.processInitial, Message{}, er, logError)
}
r.permStore <- time.Now().Format("Mon Jan _2 15:04:05 2006") + ", " + string(js) + "\n"
}(samDBv)
case <-ctx.Done():
return
}
}
}
// dumpBucket will dump out all they keys and values in the
// specified bucket, and return a sorted []samDBValue
func (r *ringBuffer) dumpBucket(bucket string) ([]samDBValue, error) {
samDBValues := []samDBValue{}
err := r.db.View(func(tx *bolt.Tx) error {
bu := tx.Bucket([]byte(bucket))
if bu == nil {
return fmt.Errorf("error: ringBuffer.dumpBucket: tx.bucket returned nil")
}
type kv struct {
key []byte
value []byte
}
dbkvs := []kv{}
// Get all the items from the db.
err := bu.ForEach(func(k, v []byte) error {
va := kv{
key: k,
value: v,
}
dbkvs = append(dbkvs, va)
return nil
})
if err != nil {
// Todo: what to return here ?
log.Fatalf("error: ringBuffer: ranging db failed: %v", err)
}
// Range all the values we got from the db, unmarshal each value.
// If the unmarshaling is ok we put it on the samDBValues slice,
// if it fails the value is not of correct format so we we delete
// it from the db, and loop to work on the next value.
for _, dbkv := range dbkvs {
var sdbv samDBValue
err := json.Unmarshal(dbkv.value, &sdbv)
if err != nil {
// If we're unable to unmarshal the value it value of wrong format,
// so we log it, and delete the value from the bucker.
log.Printf("error: ringBuffer.dumpBucket json.Umarshal failed: %v\n", err)
r.deleteKeyFromBucket(r.samValueBucket, string(dbkv.key))
continue
}
samDBValues = append(samDBValues, sdbv)
}
// TODO:
// BoltDB do not automatically shrink in filesize. We should delete the db, and create a new one to shrink the size.
// Sort the order of the slice items based on ID, since they where retreived from a map.
sort.SliceStable(samDBValues, func(i, j int) bool {
return samDBValues[i].ID > samDBValues[j].ID
})
for _, samDBv := range samDBValues {
log.Printf("info: ringBuffer.dumpBucket: k/v store, kvID: %v, message.ID: %v, subject: %v, len(data): %v\n", samDBv.ID, samDBv.SAM.ID, samDBv.SAM.Subject, len(samDBv.SAM.Data))
}
return nil
})
if err != nil {
return nil, err
}
return samDBValues, err
}
// // printBucketContent will print out all they keys and values in the
// // specified bucket.
// func (r *ringBuffer) printBucketContent(bucket string) error {
// err := r.db.View(func(tx *bolt.Tx) error {
// bu := tx.Bucket([]byte(bucket))
//
// bu.ForEach(func(k, v []byte) error {
// var vv samDBValue
// err := json.Unmarshal(v, &vv)
// if err != nil {
// log.Printf("error: printBucketContent json.Umarshal failed: %v\n", err)
// }
// log.Printf("k: %s, v: %v\n", k, vv)
// return nil
// })
//
// return nil
// })
//
// return err
// }
// deleteKeyFromBucket will delete the specified key from the specified
// bucket if it exists.
func (r *ringBuffer) deleteKeyFromBucket(bucket string, key string) error {
err := r.db.Update(func(tx *bolt.Tx) error {
bu := tx.Bucket([]byte(bucket))
err := bu.Delete([]byte(key))
if err != nil {
log.Printf("error: delete key in bucket %v failed: %v\n", bucket, err)
}
return nil
})
return err
}
// db update metrics.
func (r *ringBuffer) dbUpdateMetrics(bucket string) error {
err := r.db.Update(func(tx *bolt.Tx) error {
bu := tx.Bucket([]byte(bucket))
r.metrics.promDBMessagesCurrent.Set(float64(bu.Stats().KeyN))
return nil
})
return err
}
// getIndexValue will get the last index value stored in DB.
func (r *ringBuffer) getIndexValue() int {
const indexKey string = "index"
indexB, err := r.dbView(r.db, r.indexValueBucket, indexKey)
if err != nil {
log.Printf("error: getIndexValue: dbView: %v\n", err)
}
index, err := strconv.Atoi(string(indexB))
if err != nil && string(indexB) == "" {
log.Printf("info: getIndexValue: no index value found, probaly empty database, and no previous entries in db to process : %v\n", err)
}
return index
}
// dbView will look up and return a specific value if it exists for a key in a bucket in a DB.
func (r *ringBuffer) dbView(db *bolt.DB, bucket string, key string) ([]byte, error) {
var value []byte
// View is a help function to get values out of the database.
err := db.View(func(tx *bolt.Tx) error {
//Open a bucket to get key's and values from.
bu := tx.Bucket([]byte(bucket))
if bu == nil {
log.Printf("info: no db bucket exist: %v\n", bucket)
return nil
}
v := bu.Get([]byte(key))
if len(v) == 0 {
log.Printf("info: view: key not found\n")
return nil
}
value = v
return nil
})
return value, err
}
// dbUpdate will update the specified bucket with a key and value.
func (r *ringBuffer) dbUpdate(db *bolt.DB, bucket string, key string, value []byte) error {
err := db.Update(func(tx *bolt.Tx) error {
//Create a bucket
bu, err := tx.CreateBucketIfNotExists([]byte(bucket))
if err != nil {
return fmt.Errorf("error: CreateBuckerIfNotExists failed: %v", err)
}
//Put a value into the bucket.
if err := bu.Put([]byte(key), []byte(value)); err != nil {
return err
}
//If all was ok, we should return a nil for a commit to happen. Any error
// returned will do a rollback.
return nil
})
return err
}
// startPermStore will start the process that will handle writing of
// handled message to a permanent file.
// To store a message in the store, send what to store on the
// ringbuffer.permStore channel.
func (r *ringBuffer) startPermanentStore(ctx context.Context) {
storeFile := filepath.Join(r.configuration.DatabaseFolder, "store.log")
f, err := os.OpenFile(storeFile, os.O_APPEND|os.O_RDWR|os.O_CREATE, 0660)
if err != nil {
log.Printf("error: startPermanentStore: failed to open file: %v\n", err)
}
defer f.Close()
for {
select {
case d := <-r.permStore:
_, err := f.WriteString(d)
if err != nil {
log.Printf("error:failed to write entry: %v\n", err)
}
case <-ctx.Done():
return
}
}
}

2
scripts/clisocket.sh
View file

@ -49,5 +49,5 @@ EOF
for element in "${array[@]}"; do
sendMessage element "$command"
nc -U ./tmp/steward.sock <msg-"$element".yaml
nc -U ./tmp/ctrl.sock <msg-"$element".yaml
done

1
scripts/env.env
View file

@ -30,7 +30,6 @@ COMPRESSION=""
SERIALIZATION=""
SET_BLOCK_PROFILE_RATE=0
ENABLE_SOCKET=true
ENABLE_TUI=false
ENABLE_SIGNATURE_CHECK=false
ENABLE_ACL_CHECK=false
IS_CENTRAL_AUTH=false

1
scripts/steward/create-docker-compose-files/env.env.tpl
View file

@ -26,7 +26,6 @@ COMPRESSION
SERIALIZATION
SET_BLOCK_PROFILE_RATE=0
ENABLE_SOCKET=1
ENABLE_TUI=0
ENABLE_SIGNATURE_CHECK=0
IS_CENTRAL_AUTH=0
ENABLE_DEBUG=0

330
server.go
View file

@ -1,16 +1,19 @@
// Notes:
package steward
package ctrl
import (
"context"
"encoding/json"
"fmt"
"log"
"net"
"net/http"
"os"
"path/filepath"
"sync"
"time"
"github.com/jinzhu/copier"
"github.com/nats-io/nats.go"
"github.com/prometheus/client_golang/prometheus"
)
@ -34,8 +37,8 @@ type server struct {
configuration *Configuration
// The nats connection to the broker
natsConn *nats.Conn
// net listener for communicating via the steward socket
StewardSocket net.Listener
// net listener for communicating via the ctrl socket
ctrlSocket net.Listener
// processes holds all the information about running processes
processes *processes
// The name of the node
@ -45,20 +48,16 @@ type server struct {
//
// In general the ringbuffer will read this
// channel, unfold each slice, and put single messages on the buffer.
toRingBufferCh chan []subjectAndMessage
samToSendCh chan []subjectAndMessage
// directSAMSCh
directSAMSCh chan []subjectAndMessage
samSendLocalCh chan []subjectAndMessage
// errorKernel is doing all the error handling like what to do if
// an error occurs.
errorKernel *errorKernel
// Ring buffer
ringBuffer *ringBuffer
// metric exporter
metrics *metrics
// Version of package
version string
// tui client
tui *tui
// processInitial is the initial process that all other processes are tied to.
processInitial process
@ -70,6 +69,15 @@ type server struct {
helloRegister *helloRegister
// holds the logic for the central auth services
centralAuth *centralAuth
// message ID
messageID messageID
// audit logging
auditLogCh chan []subjectAndMessage
}
type messageID struct {
id int
mu sync.Mutex
}
// newServer will prepare and return a server type
@ -135,7 +143,7 @@ func NewServer(configuration *Configuration, version string) (*server, error) {
log.Printf(" * conn.Opts.ReconnectJitterTLS: %v\n", conn.Opts.ReconnectJitterTLS)
log.Printf(" * conn.Opts.ReconnectJitter: %v\n", conn.Opts.ReconnectJitter)
var stewardSocket net.Listener
var ctrlSocket net.Listener
var err error
// Check if tmp folder for socket exists, if not create it
@ -147,26 +155,15 @@ func NewServer(configuration *Configuration, version string) (*server, error) {
}
}
// Open the steward socket file, and start the listener if enabled.
// Open the ctrl socket file, and start the listener if enabled.
if configuration.EnableSocket {
stewardSocket, err = createSocket(configuration.SocketFolder, "steward.sock")
ctrlSocket, err = createSocket(configuration.SocketFolder, "ctrl.sock")
if err != nil {
cancel()
return nil, err
}
}
// Create the tui client structure if enabled.
var tuiClient *tui
if configuration.EnableTUI {
tuiClient, err = newTui(Node(configuration.NodeName))
if err != nil {
cancel()
return nil, err
}
}
//var nodeAuth *nodeAuth
//if configuration.EnableSignatureCheck {
nodeAuth := newNodeAuth(configuration, errorKernel)
@ -184,16 +181,16 @@ func NewServer(configuration *Configuration, version string) (*server, error) {
configuration: configuration,
nodeName: configuration.NodeName,
natsConn: conn,
StewardSocket: stewardSocket,
toRingBufferCh: make(chan []subjectAndMessage),
directSAMSCh: make(chan []subjectAndMessage),
ctrlSocket: ctrlSocket,
samToSendCh: make(chan []subjectAndMessage),
samSendLocalCh: make(chan []subjectAndMessage),
metrics: metrics,
version: version,
tui: tuiClient,
errorKernel: errorKernel,
nodeAuth: nodeAuth,
helloRegister: newHelloRegister(),
centralAuth: centralAuth,
auditLogCh: make(chan []subjectAndMessage),
}
s.processes = newProcesses(ctx, &s)
@ -231,7 +228,7 @@ func newHelloRegister() *helloRegister {
// communicate with that socket.
func createSocket(socketFolder string, socketFileName string) (net.Listener, error) {
// Just as an extra check we eventually delete any existing Steward socket files if found.
// Just as an extra check we eventually delete any existing ctrl socket files if found.
socketFilepath := filepath.Join(socketFolder, socketFileName)
if _, err := os.Stat(socketFilepath); !os.IsNotExist(err) {
err = os.Remove(socketFilepath)
@ -256,11 +253,11 @@ func createSocket(socketFolder string, socketFileName string) (net.Listener, err
// if there is publisher process for a given message subject, and
// if it does not exist it will spawn one.
func (s *server) Start() {
log.Printf("Starting steward, version=%+v\n", s.version)
log.Printf("Starting ctrl, version=%+v\n", s.version)
s.metrics.promVersion.With(prometheus.Labels{"version": string(s.version)})
go func() {
err := s.errorKernel.start(s.toRingBufferCh)
err := s.errorKernel.start(s.samToSendCh)
if err != nil {
log.Printf("%v\n", err)
}
@ -295,6 +292,9 @@ func (s *server) Start() {
go s.readHttpListener()
}
// Start audit logger.
go s.startAuditLog(s.ctx)
// Start up the predefined subscribers.
//
// Since all the logic to handle processes are tied to the process
@ -315,20 +315,10 @@ func (s *server) Start() {
go s.exposeDataFolder(s.ctx)
}
if s.configuration.EnableTUI {
go func() {
err := s.tui.Start(s.ctx, s.toRingBufferCh)
if err != nil {
log.Printf("%v\n", err)
os.Exit(1)
}
}()
}
// Start the processing of new messages from an input channel.
// NB: We might need to create a sub context for the ringbuffer here
// so we can cancel this context last, and not use the server.
s.routeMessagesToProcess("./incomingBuffer.db")
s.routeMessagesToProcess()
// Start reading the channel for injecting direct messages that should
// not be sent via the message broker.
@ -339,6 +329,45 @@ func (s *server) Start() {
}
// startAuditLog will start up the logging of all messages to audit file
func (s *server) startAuditLog(ctx context.Context) {
storeFile := filepath.Join(s.configuration.DatabaseFolder, "store.log")
f, err := os.OpenFile(storeFile, os.O_APPEND|os.O_RDWR|os.O_CREATE, 0660)
if err != nil {
log.Printf("error: startPermanentStore: failed to open file: %v\n", err)
}
defer f.Close()
for {
select {
case sams := <-s.auditLogCh:
for _, sam := range sams {
msgForPermStore := Message{}
copier.Copy(&msgForPermStore, sam.Message)
// Remove the content of the data field.
msgForPermStore.Data = nil
js, err := json.Marshal(msgForPermStore)
if err != nil {
er := fmt.Errorf("error:fillBuffer: json marshaling: %v", err)
s.errorKernel.errSend(s.processInitial, Message{}, er, logError)
}
d := time.Now().Format("Mon Jan _2 15:04:05 2006") + ", " + string(js) + "\n"
_, err = f.WriteString(d)
if err != nil {
log.Printf("error:failed to write entry: %v\n", err)
}
}
case <-ctx.Done():
return
}
}
}
// directSAMSChRead for injecting messages directly in to the local system
// without sending them via the message broker.
func (s *server) directSAMSChRead() {
@ -348,7 +377,7 @@ func (s *server) directSAMSChRead() {
case <-s.ctx.Done():
log.Printf("info: stopped the directSAMSCh reader\n\n")
return
case sams := <-s.directSAMSCh:
case sams := <-s.samSendLocalCh:
// fmt.Printf(" * DEBUG: directSAMSChRead: <- sams = %v\n", sams)
// Range over all the sams, find the process, check if the method exists, and
// handle the message by starting the correct method handler.
@ -361,12 +390,12 @@ func (s *server) directSAMSChRead() {
mh, ok := p.methodsAvailable.CheckIfExists(sams[i].Message.Method)
if !ok {
er := fmt.Errorf("error: subscriberHandler: method type not available: %v", p.subject.Event)
er := fmt.Errorf("error: subscriberHandler: method type not available: %v", p.subject.Method)
p.errorKernel.errSend(p, sams[i].Message, er, logError)
continue
}
p.handler = mh.handler
p.handler = mh
go executeHandler(p, sams[i].Message, s.nodeName)
}
@ -389,8 +418,8 @@ func (s *server) Stop() {
s.cancel()
log.Printf("info: stopped the main context\n")
// Delete the steward socket file when the program exits.
socketFilepath := filepath.Join(s.configuration.SocketFolder, "steward.sock")
// Delete the ctrl socket file when the program exits.
socketFilepath := filepath.Join(s.configuration.SocketFolder, "ctrl.sock")
if _, err := os.Stat(socketFilepath); !os.IsNotExist(err) {
err = os.Remove(socketFilepath)
@ -402,14 +431,6 @@ func (s *server) Stop() {
}
// samDBValueAndDelivered Contains the sam value as it is used in the
// state DB, and also a delivered function to be called when this message
// is picked up, so we can control if messages gets stale at some point.
type samDBValueAndDelivered struct {
samDBValue samDBValue
delivered func()
}
// routeMessagesToProcess takes a database name it's input argument.
// The database will be used as the persistent k/v store for the work
// queue which is implemented as a ring buffer.
@ -419,32 +440,9 @@ type samDBValueAndDelivered struct {
// worker process.
// It will also handle the process of spawning more worker processes
// for publisher subjects if it does not exist.
func (s *server) routeMessagesToProcess(dbFileName string) {
// Prepare and start a new ring buffer
var bufferSize int = s.configuration.RingBufferSize
const samValueBucket string = "samValueBucket"
const indexValueBucket string = "indexValueBucket"
s.ringBuffer = newringBuffer(s.ctx, s.metrics, s.configuration, bufferSize, dbFileName, Node(s.nodeName), s.toRingBufferCh, samValueBucket, indexValueBucket, s.errorKernel, s.processInitial)
ringBufferInCh := make(chan subjectAndMessage)
ringBufferOutCh := make(chan samDBValueAndDelivered)
// start the ringbuffer.
s.ringBuffer.start(s.ctx, ringBufferInCh, ringBufferOutCh)
func (s *server) routeMessagesToProcess() {
// Start reading new fresh messages received on the incomming message
// pipe/file requested, and fill them into the buffer.
// Since the new messages comming into the system is a []subjectAndMessage
// we loop here, unfold the slice, and put single subjectAndMessages's on
// the channel to the ringbuffer.
go func() {
for sams := range s.toRingBufferCh {
for _, sam := range sams {
ringBufferInCh <- sam
}
}
close(ringBufferInCh)
}()
// pipe/file.
// Process the messages that are in the ring buffer. Check and
// send if there are a specific subject for it, and if no subject
@ -454,99 +452,115 @@ func (s *server) routeMessagesToProcess(dbFileName string) {
methodsAvailable := method.GetMethodsAvailable()
go func() {
for samDBVal := range ringBufferOutCh {
go func(samDBVal samDBValueAndDelivered) {
// Signal back to the ringbuffer that message have been picked up.
samDBVal.delivered()
for samSlice := range s.samToSendCh {
for _, sam := range samSlice {
sam := samDBVal.samDBValue.SAM
// Check if the format of the message is correct.
if _, ok := methodsAvailable.CheckIfExists(sam.Message.Method); !ok {
er := fmt.Errorf("error: routeMessagesToProcess: the method do not exist, message dropped: %v", sam.Message.Method)
s.errorKernel.errSend(s.processInitial, sam.Message, er, logError)
return
}
go func(sam subjectAndMessage) {
s.messageID.mu.Lock()
s.messageID.id++
sam.Message.ID = s.messageID.id
s.messageID.mu.Unlock()
m := sam.Message
s.metrics.promMessagesProcessedIDLast.Set(float64(sam.Message.ID))
subjName := sam.Subject.name()
pn := processNameGet(subjName, processKindPublisher)
sendOK := func() bool {
var ctxCanceled bool
s.processes.active.mu.Lock()
defer s.processes.active.mu.Unlock()
// Check if the process exist, if it do not exist return false so a
// new publisher process will be created.
proc, ok := s.processes.active.procNames[pn]
if !ok {
return false
// Check if the format of the message is correct.
if _, ok := methodsAvailable.CheckIfExists(sam.Message.Method); !ok {
er := fmt.Errorf("error: routeMessagesToProcess: the method do not exist, message dropped: %v", sam.Message.Method)
s.errorKernel.errSend(s.processInitial, sam.Message, er, logError)
return
}
if proc.ctx.Err() != nil {
ctxCanceled = true
switch {
case sam.Message.Retries < 0:
sam.Message.Retries = s.configuration.DefaultMessageRetries
}
if ok && ctxCanceled {
er := fmt.Errorf(" ** routeMessagesToProcess: context is already ended for process %v, will not try to reuse existing publisher, deleting it, and creating a new publisher !!! ", proc.processName)
s.errorKernel.logDebug(er, s.configuration)
delete(proc.processes.active.procNames, proc.processName)
return false
if sam.Message.MethodTimeout < 1 && sam.Message.MethodTimeout != -1 {
sam.Message.MethodTimeout = s.configuration.DefaultMethodTimeout
}
// If found in map above, and go routine for publishing is running,
// put the message on that processes incomming message channel.
if ok && !ctxCanceled {
select {
case proc.subject.messageCh <- m:
er := fmt.Errorf(" ** routeMessagesToProcess: passed message: %v to existing process: %v", m.ID, proc.processName)
s.errorKernel.logDebug(er, s.configuration)
case <-proc.ctx.Done():
er := fmt.Errorf(" ** routeMessagesToProcess: got ctx.done for process %v", proc.processName)
s.errorKernel.logDebug(er, s.configuration)
// ---
m := sam.Message
subjName := sam.Subject.name()
pn := processNameGet(subjName, processKindPublisher)
sendOK := func() bool {
var ctxCanceled bool
s.processes.active.mu.Lock()
defer s.processes.active.mu.Unlock()
// Check if the process exist, if it do not exist return false so a
// new publisher process will be created.
proc, ok := s.processes.active.procNames[pn]
if !ok {
return false
}
return true
if proc.ctx.Err() != nil {
ctxCanceled = true
}
if ok && ctxCanceled {
er := fmt.Errorf(" ** routeMessagesToProcess: context is already ended for process %v, will not try to reuse existing publisher, deleting it, and creating a new publisher !!! ", proc.processName)
s.errorKernel.logDebug(er, s.configuration)
delete(proc.processes.active.procNames, proc.processName)
return false
}
// If found in map above, and go routine for publishing is running,
// put the message on that processes incomming message channel.
if ok && !ctxCanceled {
select {
case proc.subject.messageCh <- m:
er := fmt.Errorf(" ** routeMessagesToProcess: passed message: %v to existing process: %v", m.ID, proc.processName)
s.errorKernel.logDebug(er, s.configuration)
case <-proc.ctx.Done():
er := fmt.Errorf(" ** routeMessagesToProcess: got ctx.done for process %v", proc.processName)
s.errorKernel.logDebug(er, s.configuration)
}
return true
}
// The process was not found, so we return false here so a new publisher
// process will be created later.
return false
}()
if sendOK {
return
}
// The process was not found, so we return false here so a new publisher
// process will be created later.
return false
}()
if sendOK {
return
}
er := fmt.Errorf("info: processNewMessages: did not find publisher process for subject %v, starting new", subjName)
s.errorKernel.logDebug(er, s.configuration)
sub := newSubject(sam.Subject.Method, sam.Subject.ToNode)
var proc process
switch {
case m.IsSubPublishedMsg:
proc = newSubProcess(s.ctx, s, sub, processKindPublisher, nil)
default:
proc = newProcess(s.ctx, s, sub, processKindPublisher, nil)
}
proc.spawnWorker()
er = fmt.Errorf("info: processNewMessages: new process started, subject: %v, processID: %v", subjName, proc.processID)
s.errorKernel.logDebug(er, s.configuration)
// Now when the process is spawned we continue,
// and send the message to that new process.
select {
case proc.subject.messageCh <- m:
er := fmt.Errorf(" ** routeMessagesToProcess: passed message: %v to the new process: %v", m.ID, proc.processName)
er := fmt.Errorf("info: processNewMessages: did not find publisher process for subject %v, starting new", subjName)
s.errorKernel.logDebug(er, s.configuration)
case <-proc.ctx.Done():
er := fmt.Errorf(" ** routeMessagesToProcess: got ctx.done for process %v", proc.processName)
s.errorKernel.logDebug(er, s.configuration)
}
}(samDBVal)
sub := newSubject(sam.Subject.Method, sam.Subject.ToNode)
var proc process
switch {
case m.IsSubPublishedMsg:
proc = newSubProcess(s.ctx, s, sub, processKindPublisher, nil)
default:
proc = newProcess(s.ctx, s, sub, processKindPublisher, nil)
}
proc.spawnWorker()
er = fmt.Errorf("info: processNewMessages: new process started, subject: %v, processID: %v", subjName, proc.processID)
s.errorKernel.logDebug(er, s.configuration)
// Now when the process is spawned we continue,
// and send the message to that new process.
select {
case proc.subject.messageCh <- m:
er := fmt.Errorf(" ** routeMessagesToProcess: passed message: %v to the new process: %v", m.ID, proc.processName)
s.errorKernel.logDebug(er, s.configuration)
case <-proc.ctx.Done():
er := fmt.Errorf(" ** routeMessagesToProcess: got ctx.done for process %v", proc.processName)
s.errorKernel.logDebug(er, s.configuration)
}
}(sam)
}
}
}()
}

937
tui.go
View file

@ -1,937 +0,0 @@
package steward
import (
"bufio"
"context"
"fmt"
"io"
"log"
"os"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
"time"
"gopkg.in/yaml.v3"
"github.com/gdamore/tcell/v2"
"github.com/rivo/tview"
)
// ---------------------------------------------------------------------
// Main structure
// ---------------------------------------------------------------------
// tui holds general types used within the tui.
type tui struct {
toConsoleCh chan []byte
toRingbufferCh chan []subjectAndMessage
ctx context.Context
nodeName Node
}
// newTui returns a new tui.
func newTui(nodeName Node) (*tui, error) {
ch := make(chan []byte)
s := tui{
toConsoleCh: ch,
nodeName: nodeName,
}
return &s, nil
}
// slide holds the information about a slide
type slide struct {
name string
key tcell.Key
primitive tview.Primitive
}
// Start will start the tui.
func (t *tui) Start(ctx context.Context, toRingBufferCh chan []subjectAndMessage) error {
t.ctx = ctx
t.toRingbufferCh = toRingBufferCh
pages := tview.NewPages()
app := tview.NewApplication()
// Check if F key is pressed, and switch slide accordingly.
app.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
switch event.Key() {
case tcell.KeyF1:
pages.SwitchToPage("console")
return nil
case tcell.KeyF2:
pages.SwitchToPage("message")
return nil
case tcell.KeyF3:
pages.SwitchToPage("info")
return nil
case tcell.KeyCtrlC:
app.Stop()
log.Printf("info: detected ctrl+c, stopping TUI\n")
}
return event
})
info := tview.NewTextView().
SetDynamicColors(true).
SetRegions(true).
SetWrap(false)
// The slides to draw, and their name.
// NB: This slice is being looped over further below, to create the menu
// elements. If adding a new slide, make sure that slides are ordered in
// chronological order, so we can auto generate the info menu with it's
// corresponding F key based on the slice index+1.
slides := []slide{
{name: "console", key: tcell.KeyF1, primitive: t.console(app)},
{name: "message", key: tcell.KeyF2, primitive: t.messageSlide(app)},
{name: "info", key: tcell.KeyF3, primitive: t.infoSlide(app)},
}
// Add a page for each slide.
for i, v := range slides {
if i == 0 {
pages.AddPage(v.name, v.primitive, true, true)
fmt.Fprintf(info, " F%v:%v ", i+1, v.name)
continue
}
pages.AddPage(v.name, v.primitive, true, false)
fmt.Fprintf(info, " F%v:%v ", i+1, v.name)
}
// Create the main layout.
layout := tview.NewFlex()
//layout.SetBorder(true)
layout.SetDirection(tview.FlexRow).
AddItem(pages, 0, 10, true).
AddItem(info, 1, 1, false)
root := app.SetRoot(layout, true)
root.EnableMouse(true)
if err := root.Run(); err != nil {
log.Printf("error: root.Run(): %v\n", err)
os.Exit(1)
}
return nil
}
// ---------------------------------------------------------------------
// Slides
// ---------------------------------------------------------------------
func (t *tui) infoSlide(app *tview.Application) tview.Primitive {
flex := tview.NewFlex()
flex.SetTitle("info")
flex.SetBorder(true)
textView := tview.NewTextView()
flex.AddItem(textView, 0, 1, false)
fmt.Fprintf(textView, "Information page for Steward TUI.\n")
return flex
}
// drawMessageInputFields
//
// Draw all the message input field with values on the screen.
//
// Loop trough all the fields of the Message struct, and create
// a an input field or dropdown selector for each field.
// If a field of the struct is not defined below, it will be
// created a "no defenition" element, so it we can easily spot
// Message fields who miss an item in the form.
//
// INFO: The reason that reflect are being used here is to have
// a simple way of detecting that we are creating form fields
// for all the fields in the struct. If we have forgot'en one
// it will create a "no case" field in the console, to easily
// detect that a struct field are missing a defenition below.
func drawMessageInputFields(p slideMessageEdit, m tuiMessage) {
fieldWidth := 0
mRefVal := reflect.ValueOf(m)
for i := 0; i < mRefVal.NumField(); i++ {
fieldName := mRefVal.Type().Field(i).Name
switch fieldName {
case "_":
case "ToNode":
// Get nodes from file.
values, err := getNodeNames("nodeslist.cfg")
if err != nil {
log.Printf("error: unable to open file: %v\n", err)
}
if m.ToNode != nil && *m.ToNode != "" {
tmp := []string{string(*m.ToNode)}
tmp = append(tmp, values...)
values = tmp
}
p.inputForm.AddDropDown(fieldName, values, 0, nil).SetItemPadding(1)
//c.msgForm.AddDropDown(mRefVal.Type().Field(i).Name, values, 0, nil).SetItemPadding(1)
case "ToNodes":
if m.ToNodes == nil {
p.inputForm.AddInputField(fieldName, "", fieldWidth, nil, nil)
continue
}
if len(*m.ToNodes) != 0 {
val1 := *m.ToNodes
var val2 string
for i, v := range val1 {
if i == 0 {
val2 = fmt.Sprintf("\"%v\"", v)
continue
}
val2 = fmt.Sprintf("%v,\"%v\"", val2, v)
}
p.inputForm.AddInputField(fieldName, val2, fieldWidth, nil, nil)
}
case "Method":
var v Method
ma := v.GetMethodsAvailable()
values := []string{}
for k := range ma.Methodhandlers {
values = append(values, string(k))
}
if m.Method != nil && *m.Method != "" {
tmp := []string{string(*m.Method)}
tmp = append(tmp, values...)
values = tmp
}
p.inputForm.AddDropDown(fieldName, values, 0, nil).SetItemPadding(1)
case "MethodArgs":
if m.MethodArgs == nil {
p.inputForm.AddInputField(fieldName, "", 0, nil, nil)
continue
}
if len(*m.MethodArgs) != 0 {
val1 := *m.MethodArgs
var val2 string
for i, v := range val1 {
if i == 0 {
val2 = fmt.Sprintf("\"%v\"", v)
continue
}
val2 = fmt.Sprintf("%v,\"%v\"", val2, v)
}
p.inputForm.AddInputField(fieldName, val2, 0, nil, nil)
}
case "ReplyMethod":
var v Method
rm := v.GetReplyMethods()
values := []string{}
for _, k := range rm {
values = append(values, string(k))
}
if m.ReplyMethod != nil && *m.ReplyMethod != "" {
tmp := []string{string(*m.ReplyMethod)}
tmp = append(tmp, values...)
values = tmp
}
p.inputForm.AddDropDown(fieldName, values, 0, nil).SetItemPadding(1)
case "ReplyMethodArgs":
if m.ReplyMethodArgs == nil {
p.inputForm.AddInputField(fieldName, "", fieldWidth, nil, nil)
continue
}
if len(*m.ReplyMethodArgs) != 0 {
val1 := *m.ReplyMethodArgs
var val2 string
for i, v := range val1 {
if i == 0 {
val2 = fmt.Sprintf("\"%v\"", v)
continue
}
val2 = fmt.Sprintf("%v,\"%v\"", val2, v)
}
p.inputForm.AddInputField(fieldName, val2, fieldWidth, nil, nil)
}
case "ACKTimeout":
value := 30
if m.ACKTimeout != nil && *m.ACKTimeout != 0 {
value = *m.ACKTimeout
}
p.inputForm.AddInputField(fieldName, fmt.Sprintf("%d", value), fieldWidth, validateInteger, nil)
case "Retries":
value := 1
if m.Retries != nil && *m.Retries != 0 {
value = *m.Retries
}
p.inputForm.AddInputField(fieldName, fmt.Sprintf("%d", value), fieldWidth, validateInteger, nil)
case "ReplyACKTimeout":
value := 30
if m.ReplyACKTimeout != nil && *m.ReplyACKTimeout != 0 {
value = *m.ReplyACKTimeout
}
p.inputForm.AddInputField(fieldName, fmt.Sprintf("%d", value), fieldWidth, validateInteger, nil)
case "ReplyRetries":
value := 1
if m.ReplyRetries != nil && *m.ReplyRetries != 0 {
value = *m.ReplyRetries
}
p.inputForm.AddInputField(fieldName, fmt.Sprintf("%d", value), fieldWidth, validateInteger, nil)
case "MethodTimeout":
value := 120
if m.MethodTimeout != nil && *m.MethodTimeout != 0 {
value = *m.MethodTimeout
}
p.inputForm.AddInputField(fieldName, fmt.Sprintf("%d", value), fieldWidth, validateInteger, nil)
case "ReplyMethodTimeout":
value := 120
if m.ReplyMethodTimeout != nil && *m.ReplyMethodTimeout != 0 {
value = *m.ReplyMethodTimeout
}
p.inputForm.AddInputField(fieldName, fmt.Sprintf("%d", value), fieldWidth, validateInteger, nil)
case "Directory":
value := "/some-dir/"
if m.Directory != nil && *m.Directory != "" {
value = *m.Directory
}
p.inputForm.AddInputField(fieldName, value, fieldWidth, nil, nil)
case "FileName":
value := ".log"
if m.FileName != nil && *m.FileName != "" {
value = *m.FileName
}
p.inputForm.AddInputField(fieldName, value, fieldWidth, nil, nil)
default:
// Add a no definition fields to the form if a a field within the
// struct were missing an action above, so we can easily detect
// if there is missing a case action for one of the struct fields.
p.inputForm.AddDropDown("error: no case for: "+fieldName, []string{"1", "2"}, 0, nil).SetItemPadding(1)
}
}
}
// pageMessage is a struct for holding all the main forms and
// views used in the message slide, so we can easily reference
// them later in the code.
type slideMessageEdit struct {
flex *tview.Flex
selectMessage *tview.Form
inputForm *tview.Form
outputForm *tview.TextView
logForm *tview.TextView
saveForm *tview.Form
}
// messageSlide is the main function for setting up the slides.
func (t *tui) messageSlide(app *tview.Application) tview.Primitive {
p := slideMessageEdit{}
p.selectMessage = tview.NewForm()
p.selectMessage.SetBorder(true).SetTitle("Select Message").SetTitleAlign(tview.AlignLeft)
p.inputForm = tview.NewForm()
p.inputForm.SetBorder(true).SetTitle("Message input").SetTitleAlign(tview.AlignLeft)
p.outputForm = tview.NewTextView()
p.outputForm.SetBorder(true).SetTitle("Message output").SetTitleAlign(tview.AlignLeft)
p.outputForm.SetChangedFunc(func() {
// Will cause the log window to be redrawn as soon as
// new output are detected.
app.Draw()
})
p.logForm = tview.NewTextView()
p.logForm.SetBorder(true).SetTitle("Log/Status").SetTitleAlign(tview.AlignLeft)
p.logForm.SetChangedFunc(func() {
// Will cause the log window to be redrawn as soon as
// new output are detected.
app.Draw()
})
p.saveForm = tview.NewForm()
p.saveForm.SetBorder(true).SetTitle("Save message").SetTitleAlign(tview.AlignLeft)
// Create a flex layout.
//
// Create the outer flex layout.
p.flex = tview.NewFlex().SetDirection(tview.FlexRow).
// Add a flex for the top windows with columns.
AddItem(tview.NewFlex().SetDirection(tview.FlexColumn).
// Add a new flex for splitting output form horizontally.
AddItem(tview.NewFlex().SetDirection(tview.FlexRow).
// Add the select message form.
AddItem(p.selectMessage, 5, 0, false).
// Add the input message form.
AddItem(p.inputForm, 0, 10, false),
0, 10, false).
// Add a new flex for splitting output form horizontally.
AddItem(tview.NewFlex().SetDirection(tview.FlexRow).
// Add the message output form.
AddItem(p.outputForm, 0, 10, false).
// Add the save message form.
AddItem(p.saveForm, 0, 2, false),
0, 10, false),
0, 10, false).
// Add a flex for the bottom log window.
AddItem(tview.NewFlex().
// Add the log form.
AddItem(p.logForm, 0, 1, false),
0, 1, false)
m := tuiMessage{}
// ---
// Add a dropdown menu to select message files to use.
msgsValues := t.getMessageNames(p.logForm)
msgDropdownFunc := func(msgFileName string, index int) {
filePath := filepath.Join("messages", msgFileName)
fh, err := os.Open(filePath)
if err != nil {
fmt.Fprintf(p.logForm, "error: failed to open message file: %v\n", err)
return
}
defer fh.Close()
fileContent, err := io.ReadAll(fh)
if err != nil {
fmt.Fprintf(p.logForm, "error: failed to read message file: %v\n", err)
return
}
var msgs []tuiMessage
err = yaml.Unmarshal(fileContent, &msgs)
if err != nil {
fmt.Fprintf(p.logForm, "error: yaml unmarshal of file content failed: %v\n", err)
return
}
m = msgs[0]
// Clear the form.
p.inputForm.Clear(false)
// Draw all the message input field with values on the screen.
p.inputForm.Clear(false)
drawMessageInputFields(p, m)
}
messageDropdown := tview.NewDropDown()
messageDropdown.SetLabelColor(tcell.ColorIndianRed)
messageDropdown.SetLabel("message").SetOptions(msgsValues, msgDropdownFunc)
// Clear the form.
p.inputForm.Clear(false)
drawMessageInputFields(p, m)
p.selectMessage.AddFormItem(messageDropdown)
p.inputForm.AddButton("update message dropdown menu", func() {
messageMessageValues := t.getMessageNames(p.logForm)
messageDropdown.SetLabel("message").SetOptions(messageMessageValues, msgDropdownFunc)
})
// ---
// Variable to hold the last output created when the generate button have
// been pushed.
var lastGeneratedMessage []byte
var saveFileName string
// Add Buttons below the message fields. Like Generate and Exit.
p.inputForm.
// Add a generate button, which when pressed will loop through all the
// message form items, and if found fill the value into a msg struct,
// and at last write it to a file.
AddButton("generate to console", func() {
p.outputForm.Clear()
m := tuiMessage{}
// Loop trough all the form fields, check the value of each
// form field, and add the value to m.
for i := 0; i < p.inputForm.GetFormItemCount(); i++ {
fi := p.inputForm.GetFormItem(i)
label, value := getLabelAndValue(fi)
switch label {
case "message":
case "ToNode":
v := Node(value)
m.ToNode = &v
case "ToNodes":
slice, err := stringToNode(value)
if err != nil {
fmt.Fprintf(p.logForm, "%v : error: ToNodes missing or malformed format, should be \"arg0\",\"arg1\",\"arg2\", %v\n", time.Now().Format("Mon Jan _2 15:04:05 2006"), err)
return
}
m.ToNodes = slice
case "Method":
v := Method(value)
m.Method = &v
case "MethodArgs":
slice, err := stringToSlice(value)
if err != nil {
fmt.Fprintf(p.logForm, "%v : error: MethodArgs missing or malformed format, should be \"arg0\",\"arg1\",\"arg2\", %v\n", time.Now().Format("Mon Jan _2 15:04:05 2006"), err)
return
}
m.MethodArgs = slice
case "ReplyMethod":
v := Method(value)
m.ReplyMethod = &v
case "ReplyMethodArgs":
slice, err := stringToSlice(value)
if err != nil {
fmt.Fprintf(p.logForm, "%v : error: ReplyMethodArgs missing or malformed format, should be \"arg0\",\"arg1\",\"arg2\", %v\n", time.Now().Format("Mon Jan _2 15:04:05 2006"), err)
return
}
m.ReplyMethodArgs = slice
case "ACKTimeout":
v, _ := strconv.Atoi(value)
m.ACKTimeout = &v
case "Retries":
v, _ := strconv.Atoi(value)
m.Retries = &v
case "ReplyACKTimeout":
v, _ := strconv.Atoi(value)
m.ReplyACKTimeout = &v
case "ReplyRetries":
v, _ := strconv.Atoi(value)
m.ReplyRetries = &v
case "MethodTimeout":
v, _ := strconv.Atoi(value)
m.MethodTimeout = &v
case "ReplyMethodTimeout":
v, _ := strconv.Atoi(value)
m.ReplyMethodTimeout = &v
case "Directory":
m.Directory = &value
case "FileName":
m.FileName = &value
default:
fmt.Fprintf(p.logForm, "%v : error: did not find case definition for how to handle the \"%v\" within the switch statement\n", time.Now().Format("Mon Jan _2 15:04:05 2006"), label)
return
}
}
msgs := []tuiMessage{}
msgs = append(msgs, m)
// // msgsIndented, err := json.MarshalIndent(msgs, "", " ")
// buf := new(bytes.Buffer)
// enc := json.NewEncoder(buf)
// enc.SetEscapeHTML(false)
// enc.SetIndent("", " ")
// err := enc.Encode(msgs)
// if err != nil {
// fmt.Fprintf(p.logForm, "%v : error: jsonIndent failed: %v\n", time.Now().Format("Mon Jan _2 15:04:05 2006"), err)
// }
bs, err := yaml.Marshal(msgs)
if err != nil {
fmt.Fprintf(p.logForm, "%v : error: yaml marshal failed: %v\n", time.Now().Format("Mon Jan _2 15:04:05 2006"), err)
}
// Copy the message to a variable outside this scope so we can use
// the content for example if we want to save the message to file.
lastGeneratedMessage = bs
_, err = p.outputForm.Write(bs)
if err != nil {
fmt.Fprintf(p.logForm, "%v : error: write to fh failed: %v\n", time.Now().Format("Mon Jan _2 15:04:05 2006"), err)
}
}).
// Add exit button.
AddButton("exit", func() {
app.Stop()
})
// app.SetFocus(p.inputForm)
p.saveForm.
AddInputField("FileName", "", 40, nil, func(text string) {
saveFileName = text
}).
AddButton("save", func() {
messageFolder := "messages"
if saveFileName == "" {
fmt.Fprintf(p.logForm, "error: missing filename\n")
return
}
if _, err := os.Stat(messageFolder); os.IsNotExist(err) {
err := os.MkdirAll(messageFolder, 0770)
if err != nil {
fmt.Fprintf(p.logForm, "error: failed to create messages folder: %v\n", err)
return
}
}
file := filepath.Join(messageFolder, saveFileName)
fh, err := os.OpenFile(file, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0755)
if err != nil {
fmt.Fprintf(p.logForm, "error: opening file for writing: %v\n", err)
return
}
defer fh.Close()
_, err = fh.Write([]byte(lastGeneratedMessage))
if err != nil {
fmt.Fprintf(p.logForm, "error: writing message to file: %v\n", err)
return
}
fmt.Fprintf(p.logForm, "info: succesfully wrote message to file: %v\n", file)
// update the select message dropdown
messageMessageValues := t.getMessageNames(p.logForm)
messageDropdown.SetLabel("message").SetOptions(messageMessageValues, msgDropdownFunc)
// p.inputForm.Clear(false)
})
return p.flex
}
func (t *tui) console(app *tview.Application) tview.Primitive {
// pageMessage is a struct for holding all the main forms and
// views used in the message slide, so we can easily reference
// them later in the code.
type slideConsole struct {
flex *tview.Flex
selectForm *tview.Form
outputForm *tview.TextView
}
p := slideConsole{}
p.selectForm = tview.NewForm()
p.selectForm.SetBorder(true).SetTitle("select").SetTitleAlign(tview.AlignLeft)
p.selectForm.SetButtonsAlign(tview.AlignCenter)
p.selectForm.SetHorizontal(false)
p.outputForm = tview.NewTextView()
p.outputForm.SetBorder(true).SetTitle("output").SetTitleAlign(tview.AlignLeft)
p.outputForm.SetChangedFunc(func() {
// Will cause the log window to be redrawn as soon as
// new output are detected.
app.Draw()
})
// Create a flex layout.
//
// Create the outer flex layout.
p.flex = tview.NewFlex().SetDirection(tview.FlexRow).
// Add a flex for the top windows with columns.
AddItem(tview.NewFlex().SetDirection(tview.FlexColumn).
AddItem(p.selectForm, 0, 3, false).
// Add the message output form.
AddItem(p.outputForm, 0, 10, false),
0, 10, false)
// Add a flex for the bottom log window.
// Add items.
// Create nodes dropdown field.
nodesList, err := getNodeNames("nodeslist.cfg")
if err != nil {
fmt.Fprintf(p.outputForm, "error: failed to open nodeslist.cfg file\n")
}
nodesDropdown := tview.NewDropDown()
nodesDropdown.SetLabelColor(tcell.ColorIndianRed)
nodesDropdown.SetLabel("nodes").SetOptions(nodesList, nil)
p.selectForm.AddFormItem(nodesDropdown)
msgsValues := t.getMessageNames(p.outputForm)
messageDropdown := tview.NewDropDown()
messageDropdown.SetLabelColor(tcell.ColorIndianRed)
messageDropdown.SetLabel("message").SetOptions(msgsValues, nil)
p.selectForm.AddFormItem(messageDropdown)
// Add button for manually updating dropdown menus.
p.selectForm.AddButton("update", func() {
nodesList, err := getNodeNames("nodeslist.cfg")
if err != nil {
fmt.Fprintf(p.outputForm, "error: failed to open nodeslist.cfg file\n")
}
nodesDropdown.SetLabel("nodes").SetOptions(nodesList, nil)
msgsValues := t.getMessageNames(p.outputForm)
messageDropdown.SetLabel("message").SetOptions(msgsValues, nil)
})
// Add button for clearing the output form.
p.selectForm.AddButton("clear", func() {
p.outputForm.Clear()
})
// Update the dropdown menus when the flex view gets focus.
p.flex.SetFocusFunc(func() {
nodesList, err := getNodeNames("nodeslist.cfg")
if err != nil {
fmt.Fprintf(p.outputForm, "error: failed to open nodeslist.cfg file\n")
}
nodesDropdown.SetLabel("nodes").SetOptions(nodesList, nil)
messageValues := t.getMessageNames(p.outputForm)
messageDropdown.SetLabel("message").SetOptions(messageValues, nil)
})
p.selectForm.AddButton("send message", func() {
// here........
nr, msgFileName := messageDropdown.GetCurrentOption()
if nr < 1 {
fmt.Fprintf(p.outputForm, "info: please select a message from the dropdown: %v\n", msgFileName)
return
}
nr, toNode := nodesDropdown.GetCurrentOption()
if nr < 1 {
fmt.Fprintf(p.outputForm, "info: please select a message from the dropdown: %v\n", msgFileName)
return
}
filePath := filepath.Join("messages", msgFileName)
fh, err := os.Open(filePath)
if err != nil {
fmt.Fprintf(p.outputForm, "error: failed to open message file: %v\n", err)
return
}
defer fh.Close()
fileContent, err := io.ReadAll(fh)
if err != nil {
fmt.Fprintf(p.outputForm, "error: failed to read message file: %v\n", err)
return
}
var msgs []Message
err = yaml.Unmarshal(fileContent, &msgs)
if err != nil {
fmt.Fprintf(p.outputForm, "error: yaml unmarshal of file content failed: %v\n", err)
return
}
msg := msgs[0]
msg.FromNode = t.nodeName
msg.ToNode = Node(toNode)
// fmt.Fprintf(p.outputForm, "%#v\n", msg)
sam, err := newSubjectAndMessage(msg)
if err != nil {
fmt.Fprintf(p.outputForm, "error: newSubjectAndMessage failed: %v\n", err)
return
}
sams := []subjectAndMessage{sam}
t.toRingbufferCh <- sams
})
go func() {
for {
select {
case messageData := <-t.toConsoleCh:
for _, v := range messageData {
fmt.Fprintf(p.outputForm, "%v", string(v))
}
case <-t.ctx.Done():
log.Printf("info: stopped tui toConsole worker\n")
return
}
}
}()
return p.flex
}
// ---------------------------------------------------------------------
// Helper functions
// ---------------------------------------------------------------------
// getMessageNames will get the names of all the messages in
// the messages folder.
func (t *tui) getMessageNames(outputForm *tview.TextView) []string {
// Create messages dropdown field.
fInfo, err := os.ReadDir("messages")
if err != nil {
fmt.Fprintf(outputForm, "error: failed to read files from messages dir\n")
}
msgsValues := []string{}
msgsValues = append(msgsValues, "")
for _, v := range fInfo {
msgsValues = append(msgsValues, v.Name())
}
return msgsValues
}
// stringToSlice will Split the comma separated string
// into a and remove the start and end ampersand.
func stringToSlice(s string) (*[]string, error) {
if s == "" {
return nil, nil
}
var stringSlice []string
sp := strings.Split(s, ",")
for _, v := range sp {
// Check if format is correct, return if not.
pre := strings.HasPrefix(v, "\"")
suf := strings.HasSuffix(v, "\"")
if !pre || !suf {
return nil, fmt.Errorf("stringToSlice: missing leading or ending ampersand")
}
// Remove leading and ending ampersand.
v = v[1:]
v = strings.TrimSuffix(v, "\"")
stringSlice = append(stringSlice, v)
}
return &stringSlice, nil
}
// stringToNodes will Split the comma separated slice
// of nodes, and remove the start and end ampersand.
func stringToNode(s string) (*[]Node, error) {
if s == "" {
return nil, nil
}
var nodeSlice []Node
sp := strings.Split(s, ",")
for _, v := range sp {
// Check if format is correct, return if not.
pre := strings.HasPrefix(v, "\"")
suf := strings.HasSuffix(v, "\"")
if !pre || !suf {
return nil, fmt.Errorf("stringToSlice: missing leading or ending ampersand")
}
// Remove leading and ending ampersand.
v = v[1:]
v = strings.TrimSuffix(v, "\"")
nodeSlice = append(nodeSlice, Node(v))
}
return &nodeSlice, nil
}
// Will return the Label And the text Value of an input or dropdown form field.
func getLabelAndValue(fi tview.FormItem) (string, string) {
var label string
var value string
switch v := fi.(type) {
case *tview.InputField:
value = v.GetText()
label = v.GetLabel()
case *tview.DropDown:
label = v.GetLabel()
_, value = v.GetCurrentOption()
}
return label, value
}
// Check if number is int.
func validateInteger(text string, ch rune) bool {
if text == "-" {
return true
}
_, err := strconv.Atoi(text)
return err == nil
}
// getNodes will load all the node names from a file, and return a slice of
// string values, each representing a unique node.
func getNodeNames(fileName string) ([]string, error) {
dirPath, err := os.Getwd()
if err != nil {
return nil, fmt.Errorf("error: tui: unable to get working directory: %v", err)
}
filePath := filepath.Join(dirPath, fileName)
fh, err := os.Open(filePath)
if err != nil {
return nil, fmt.Errorf("error: tui: you should create a file named nodeslist.cfg with all your nodes : %v", err)
}
defer fh.Close()
nodes := []string{}
// append a blank node at the beginning of the slice, so the dropdown
// can be set to blank
nodes = append(nodes, "")
scanner := bufio.NewScanner(fh)
for scanner.Scan() {
node := scanner.Text()
nodes = append(nodes, node)
}
sort.Strings(nodes)
return nodes, nil
}

18
tui_msg.go
View file

@ -1,18 +0,0 @@
package steward
type tuiMessage struct {
ToNode *Node `json:"toNode,omitempty" yaml:"toNode,omitempty"`
ToNodes *[]Node `json:"toNodes,omitempty" yaml:"toNodes,omitempty"`
Method *Method `json:"method,omitempty" yaml:"method,omitempty"`
MethodArgs *[]string `json:"methodArgs,omitempty" yaml:"methodArgs,omitempty"`
ReplyMethod *Method `json:"replyMethod,omitempty" yaml:"replyMethod,omitempty"`
ReplyMethodArgs *[]string `json:"replyMethodArgs,omitempty" yaml:"replyMethodArgs,omitempty"`
ACKTimeout *int `json:"ACKTimeout,omitempty" yaml:"ACKTimeout,omitempty"`
Retries *int `json:"retries,omitempty" yaml:"retries,omitempty"`
ReplyACKTimeout *int `json:"replyACKTimeout,omitempty" yaml:"replyACKTimeout,omitempty"`
ReplyRetries *int `json:"replyRetries,omitempty" yaml:"replyRetries,omitempty"`
MethodTimeout *int `json:"methodTimeout,omitempty" yaml:"methodTimeout,omitempty"`
ReplyMethodTimeout *int `json:"replyMethodTimeout,omitempty" yaml:"replyMethodTimeout,omitempty"`
Directory *string `json:"directory,omitempty" yaml:"directory,omitempty"`
FileName *string `json:"fileName,omitempty" yaml:"fileName,omitempty"`
}