Skip to content

Commit c49c511

Browse files
KbayeroKbayero
committed
Write pipelines using installer, inject geolocation and plugins to event processor image
1 parent 1a8b73c commit c49c511

11 files changed

Lines changed: 89 additions & 115 deletions

‎event_processor.Dockerfile‎

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,16 @@ ARG BASE_IMAGE
22

33
FROM ${BASE_IMAGE}
44

5-
COPY ./pipelines/ /build-data/pipeline/
6-
COPY ./geolocation/ /build-data/geolocation/
7-
COPY ./plugins/alerts/com.utmstack.alerts.plugin /build-data/plugins/
8-
COPY ./plugins/aws/com.utmstack.aws.plugin /build-data/plugins/
9-
COPY ./plugins/azure/com.utmstack.azure.plugin /build-data/plugins/
10-
COPY ./plugins/bitdefender/com.utmstack.bitdefender.plugin /build-data/plugins/
11-
COPY ./plugins/config/com.utmstack.config.plugin /build-data/plugins/
12-
COPY ./plugins/events/com.utmstack.events.plugin /build-data/plugins/
13-
COPY ./plugins/gcp/com.utmstack.gcp.plugin /build-data/plugins/
14-
COPY ./plugins/geolocation/com.utmstack.geolocation.plugin /build-data/plugins/
15-
COPY ./plugins/inputs/com.utmstack.inputs.plugin /build-data/plugins/
16-
COPY ./plugins/o365/com.utmstack.o365.plugin /build-data/plugins/
17-
COPY ./plugins/sophos/com.utmstack.sophos.plugin /build-data/plugins/
18-
COPY ./plugins/stats/com.utmstack.stats.plugin /build-data/plugins/
5+
COPY ./geolocation/ /workdir/geolocation/
6+
COPY ./plugins/alerts/com.utmstack.alerts.plugin /workdir/plugins/utmstack/
7+
COPY ./plugins/aws/com.utmstack.aws.plugin /workdir/plugins/utmstack/
8+
COPY ./plugins/azure/com.utmstack.azure.plugin /workdir/plugins/utmstack/
9+
COPY ./plugins/bitdefender/com.utmstack.bitdefender.plugin /workdir/plugins/utmstack/
10+
COPY ./plugins/config/com.utmstack.config.plugin /workdir/plugins/utmstack/
11+
COPY ./plugins/events/com.utmstack.events.plugin /workdir/plugins/utmstack/
12+
COPY ./plugins/gcp/com.utmstack.gcp.plugin /workdir/plugins/utmstack/
13+
COPY ./plugins/geolocation/com.utmstack.geolocation.plugin /workdir/plugins/utmstack/
14+
COPY ./plugins/inputs/com.utmstack.inputs.plugin /workdir/plugins/utmstack/
15+
COPY ./plugins/o365/com.utmstack.o365.plugin /workdir/plugins/utmstack/
16+
COPY ./plugins/sophos/com.utmstack.sophos.plugin /workdir/plugins/utmstack/
17+
COPY ./plugins/stats/com.utmstack.stats.plugin /workdir/plugins/utmstack/

‎installer/config/const.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ import (
77

88
const (
99
RegisterInstanceEndpoint = "/api/v1/instances/register"
10-
GetInstanceDetailsEndpoint = "/api/v1/instances/"
11-
GetUpdatesInfoEndpoint = "/api/v1/updates/"
10+
GetInstanceDetailsEndpoint = "/api/v1/instances"
11+
GetUpdatesInfoEndpoint = "/api/v1/updates"
1212

1313
CMAlpha = "https://customermanager.utmstack.com/alpha"
1414
CMBeta = "https://customermanager.utmstack.com/beta"

‎installer/docker/compose.go‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,7 @@ func (c *Compose) Encode() ([]byte, error) {
8686
}
8787

8888
func (c *Compose) Populate(conf *config.Config, stack *StackConfig) error {
89-
var pluginsConfig PluginsConfig
90-
err := pluginsConfig.Set(conf, stack)
89+
err := SetPluginsConfigs(conf, stack)
9190
if err != nil {
9291
return err
9392
}
@@ -304,11 +303,10 @@ func (c *Compose) Populate(conf *config.Config, stack *StackConfig) error {
304303
"8000:8000",
305304
},
306305
Volumes: []string{
307-
utils.MakeDir(0777, stack.EventsEngineWorkdir, "build-data") + ":/build-data",
306+
// utils.MakeDir(0777, stack.EventsEngineWorkdir, "geolocation") + ":/workdir/geolocation",
307+
// utils.MakeDir(0777, stack.EventsEngineWorkdir, "plugins") + ":/workdir/plugins/utmstack",
308308
utils.MakeDir(0777, stack.EventsEngineWorkdir, "pipeline") + ":/workdir/pipeline",
309-
utils.MakeDir(0777, stack.EventsEngineWorkdir, "geolocation") + ":/workdir/geolocation",
310309
utils.MakeDir(0777, stack.EventsEngineWorkdir, "rules") + ":/workdir/rules/utmstack",
311-
utils.MakeDir(0777, stack.EventsEngineWorkdir, "plugins") + ":/workdir/plugins/utmstack",
312310
utils.MakeDir(0777, stack.EventsEngineWorkdir, "logs") + ":/workdir/logs",
313311
stack.Cert + ":/cert",
314312
conf.UpdatesFolder + ":/updates",

‎installer/docker/plugins.go‎

Lines changed: 61 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,29 @@
11
package docker
22

33
import (
4-
"os"
4+
"fmt"
55
"path/filepath"
66

77
"github.com/utmstack/UTMStack/installer/config"
88
"github.com/utmstack/UTMStack/installer/utils"
9-
"gopkg.in/yaml.v3"
109
)
1110

1211
type PluginsConfig struct {
1312
Plugins map[string]PluginConfig `yaml:"plugins"`
1413
}
1514

1615
type PluginConfig struct {
17-
RulesFolder string `yaml:"rulesFolder"`
18-
GeoIPFolder string `yaml:"geoipFolder"`
19-
Elasticsearch string `yaml:"elasticsearch"`
20-
PostgreSQL PostgreConfig `yaml:"postgresql"`
21-
ServerName string `yaml:"serverName"`
22-
InternalKey string `yaml:"internalKey"`
23-
AgentManager string `yaml:"agentManager"`
24-
Backend string `yaml:"backend"`
25-
CertsFolder string `yaml:"certsFolder"`
26-
BdgzPort string `yaml:"bdgzPort"`
16+
Order []string `yaml:"order,omitempty"`
17+
Port int `yaml:"port,omitempty"`
18+
RulesFolder string `yaml:"rulesFolder,omitempty"`
19+
GeoIPFolder string `yaml:"geoipFolder,omitempty"`
20+
Elasticsearch string `yaml:"elasticsearch,omitempty"`
21+
PostgreSQL PostgreConfig `yaml:"postgresql,omitempty"`
22+
ServerName string `yaml:"serverName,omitempty"`
23+
InternalKey string `yaml:"internalKey,omitempty"`
24+
AgentManager string `yaml:"agentManager,omitempty"`
25+
Backend string `yaml:"backend,omitempty"`
26+
CertsFolder string `yaml:"certsFolder,omitempty"`
2727
}
2828

2929
type PostgreConfig struct {
@@ -34,10 +34,37 @@ type PostgreConfig struct {
3434
Database string `yaml:"database"`
3535
}
3636

37-
func (c *PluginsConfig) Set(conf *config.Config, stack *StackConfig) error {
38-
c.Plugins = make(map[string]PluginConfig)
37+
func SetPluginsConfigs(conf *config.Config, stack *StackConfig) error {
38+
analysisPipeline := PluginsConfig{}
39+
analysisPipeline.Plugins = make(map[string]PluginConfig)
40+
analysisPipeline.Plugins["analysis"] = PluginConfig{
41+
Order: []string{"com.utmstack.events", "cel"},
42+
}
43+
44+
correlationPipeline := PluginsConfig{}
45+
correlationPipeline.Plugins = make(map[string]PluginConfig)
46+
correlationPipeline.Plugins["correlation"] = PluginConfig{
47+
Order: []string{"com.utmstack.events"},
48+
}
49+
50+
inputPipeline := PluginsConfig{}
51+
inputPipeline.Plugins = make(map[string]PluginConfig)
52+
inputPipeline.Plugins["http-input"] = PluginConfig{
53+
Port: 8082,
54+
}
55+
inputPipeline.Plugins["grpc-input"] = PluginConfig{
56+
Port: 8083,
57+
}
3958

40-
c.Plugins["com.utmstack"] = PluginConfig{
59+
notificationPipeline := PluginsConfig{}
60+
notificationPipeline.Plugins = make(map[string]PluginConfig)
61+
notificationPipeline.Plugins["notification"] = PluginConfig{
62+
Order: []string{"com.utmstack.stats"},
63+
}
64+
65+
utmstackPipeline := PluginsConfig{}
66+
utmstackPipeline.Plugins = make(map[string]PluginConfig)
67+
utmstackPipeline.Plugins["com.utmstack"] = PluginConfig{
4168
RulesFolder: "/workdir/rules",
4269
GeoIPFolder: "/workdir/geolocation",
4370
Elasticsearch: "http://node1:9200",
@@ -55,23 +82,33 @@ func (c *PluginsConfig) Set(conf *config.Config, stack *StackConfig) error {
5582
CertsFolder: "/cert",
5683
}
5784

58-
config, err := yaml.Marshal(c)
85+
pipelineDir := filepath.Join(stack.EventsEngineWorkdir, "pipeline")
86+
utils.CreatePathIfNotExist(pipelineDir)
87+
88+
err := utils.WriteYAML(filepath.Join(pipelineDir, "system_plugins_analysis.yaml"), analysisPipeline)
5989
if err != nil {
60-
return err
90+
return fmt.Errorf("error writing analysis pipeline config: %w", err)
6191
}
6292

63-
pipelineDir := filepath.Join(stack.EventsEngineWorkdir, "pipeline")
93+
err = utils.WriteYAML(filepath.Join(pipelineDir, "system_plugins_correlation.yaml"), correlationPipeline)
94+
if err != nil {
95+
return fmt.Errorf("error writing correlation pipeline config: %w", err)
96+
}
6497

65-
utils.CreatePathIfNotExist(pipelineDir)
98+
err = utils.WriteYAML(filepath.Join(pipelineDir, "system_plugins_input.yaml"), inputPipeline)
99+
if err != nil {
100+
return fmt.Errorf("error writing input pipeline config: %w", err)
101+
}
66102

67-
err = os.WriteFile(filepath.Join(pipelineDir, "utmstack_plugins.yaml"), config, 0644)
103+
err = utils.WriteYAML(filepath.Join(pipelineDir, "system_plugins_notification.yaml"), notificationPipeline)
68104
if err != nil {
69-
return err
105+
return fmt.Errorf("error writing notification pipeline config: %w", err)
70106
}
71107

72-
return nil
73-
}
108+
err = utils.WriteYAML(filepath.Join(pipelineDir, "utmstack_plugins.yaml"), utmstackPipeline)
109+
if err != nil {
110+
return fmt.Errorf("error writing UTMStack pipeline config: %w", err)
111+
}
74112

75-
func (c *PluginsConfig) UpdatePlugings() error {
76113
return nil
77114
}

‎installer/docker/stack.go‎

Lines changed: 1 addition & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ func StackUP(tag string) error {
9999
return err
100100
}
101101

102-
err = os.WriteFile("compose.yml", d, 0644)
102+
err = os.WriteFile(filepath.Join(utils.GetMyPath(), "compose.yml"), d, 0644)
103103
if err != nil {
104104
return err
105105
}
@@ -119,30 +119,6 @@ func StackUP(tag string) error {
119119
return err
120120
}
121121

122-
err = utils.MoveFolderContents(
123-
filepath.Join(GetStackConfig().EventsEngineWorkdir, "build-data", "plugins"),
124-
filepath.Join(GetStackConfig().EventsEngineWorkdir, "plugins"),
125-
)
126-
if err != nil {
127-
return fmt.Errorf("error copying plugins: %w", err)
128-
}
129-
130-
err = utils.MoveFolderContents(
131-
filepath.Join(GetStackConfig().EventsEngineWorkdir, "build-data", "pipeline"),
132-
filepath.Join(GetStackConfig().EventsEngineWorkdir, "pipeline"),
133-
)
134-
if err != nil {
135-
return fmt.Errorf("error copying pipeline: %w", err)
136-
}
137-
138-
err = utils.MoveFolderContents(
139-
filepath.Join(GetStackConfig().EventsEngineWorkdir, "build-data", "geolocation"),
140-
filepath.Join(GetStackConfig().EventsEngineWorkdir, "geolocation"),
141-
)
142-
if err != nil {
143-
return fmt.Errorf("error copying geolocation: %w", err)
144-
}
145-
146122
return nil
147123
}
148124

‎installer/utils/files.go‎

Lines changed: 0 additions & 27 deletions
This file was deleted.

‎installer/utils/os.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,15 @@ import (
1111
"gopkg.in/yaml.v3"
1212
)
1313

14+
func GetMyPath() string {
15+
ex, err := os.Executable()
16+
if err != nil {
17+
return ""
18+
}
19+
exPath := filepath.Dir(ex)
20+
return exPath
21+
}
22+
1423
func RunEnvCmd(env []string, command string, arg ...string) error {
1524
cmd := exec.Command(command, arg...)
1625
cmd.Env = append(os.Environ(), env...)

‎pipelines/system_plugins_analysis.yaml‎

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎pipelines/system_plugins_correlation.yaml‎

Lines changed: 0 additions & 4 deletions
This file was deleted.

‎pipelines/system_plugins_input.yaml‎

Lines changed: 0 additions & 5 deletions
This file was deleted.

0 commit comments

Comments
 (0)